From 8cac1bb09f55ff2f285914e349507472ff86f4d7 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Mon, 28 Mar 2016 10:36:59 -0700 Subject: [PATCH 01/75] HADOOP-12954. Add a way to change hadoop.security.token.service.use_ip (rkanter) --- .../apache/hadoop/security/SecurityUtil.java | 53 +++++++++++-------- .../hadoop/security/TestSecurityUtil.java | 16 ++++-- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index 61cd516fef6..42abe0ed3bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -73,16 +73,38 @@ public class SecurityUtil { @VisibleForTesting static HostResolver hostResolver; + private static boolean logSlowLookups; + private static int slowLookupThresholdMs; + static { - Configuration conf = new Configuration(); + setConfigurationInternal(new Configuration()); + } + + @InterfaceAudience.Public + @InterfaceStability.Evolving + public static void setConfiguration(Configuration conf) { + LOG.info("Updating Configuration"); + setConfigurationInternal(conf); + } + + private static void setConfigurationInternal(Configuration conf) { boolean useIp = conf.getBoolean( CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT); setTokenServiceUseIp(useIp); - } - private static boolean logSlowLookups = getLogSlowLookupsEnabled(); - private static int slowLookupThresholdMs = getSlowLookupThresholdMs(); + logSlowLookups = conf.getBoolean( + CommonConfigurationKeys + .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY, + CommonConfigurationKeys + .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT); + + slowLookupThresholdMs = conf.getInt( + CommonConfigurationKeys + .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY, + CommonConfigurationKeys + .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT); + } /** * For use only by tests and initialization @@ -90,6 +112,11 @@ public class SecurityUtil { @InterfaceAudience.Private @VisibleForTesting public static void setTokenServiceUseIp(boolean flag) { + if (LOG.isDebugEnabled()) { + LOG.debug("Setting " + + CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP + + " to " + flag); + } useIpForTokenService = flag; hostResolver = !useIpForTokenService ? new QualifiedHostResolver() @@ -485,24 +512,6 @@ public class SecurityUtil { } } - private static boolean getLogSlowLookupsEnabled() { - Configuration conf = new Configuration(); - - return conf.getBoolean(CommonConfigurationKeys - .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY, - CommonConfigurationKeys - .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT); - } - - private static int getSlowLookupThresholdMs() { - Configuration conf = new Configuration(); - - return conf.getInt(CommonConfigurationKeys - .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY, - CommonConfigurationKeys - .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT); - } - /** * Resolves a host subject to the security requirements determined by * hadoop.security.token.service.use_ip. Optionally logs slow resolutions. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java index 14f9091ed9a..29932d160b4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java @@ -29,6 +29,7 @@ import java.net.URI; import javax.security.auth.kerberos.KerberosPrincipal; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.token.Token; @@ -145,7 +146,10 @@ public class TestSecurityUtil { @Test public void testBuildDTServiceName() { - SecurityUtil.setTokenServiceUseIp(true); + Configuration conf = new Configuration(false); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, true); + SecurityUtil.setConfiguration(conf); assertEquals("127.0.0.1:123", SecurityUtil.buildDTServiceName(URI.create("test://LocalHost"), 123) ); @@ -162,7 +166,10 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { - SecurityUtil.setTokenServiceUseIp(true); + Configuration conf = new Configuration(false); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, true); + SecurityUtil.setConfiguration(conf); assertEquals("127.0.0.1:123", SecurityUtil.buildTokenService(new InetSocketAddress("LocalHost", 123)).toString() ); @@ -261,7 +268,10 @@ public class TestSecurityUtil { verifyTokenService(InetSocketAddress addr, String host, String ip, int port, boolean useIp) { //LOG.info("address:"+addr+" host:"+host+" ip:"+ip+" port:"+port); - SecurityUtil.setTokenServiceUseIp(useIp); + Configuration conf = new Configuration(false); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, useIp); + SecurityUtil.setConfiguration(conf); String serviceHost = useIp ? ip : StringUtils.toLowerCase(host); Token token = new Token(); From 524bc3c33aff301c1a8d60ed8e6a3b240e305045 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 28 Mar 2016 11:12:33 -0700 Subject: [PATCH 02/75] YARN-998. Keep NM resource updated through dynamic resource config for RM/NM restart. Contributed by Junping Du --- .../server/resourcemanager/AdminService.java | 38 +++++---- .../ResourceTrackerService.java | 80 +++++++++++++++---- .../DynamicResourceConfiguration.java | 13 ++- .../resourcemanager/TestRMAdminService.java | 78 +++++++++++++++++- 4 files changed, 165 insertions(+), 44 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index fbc2d6f068b..fc530e38094 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -639,34 +639,32 @@ public class AdminService extends CompositeService implements try { Configuration conf = getConfig(); Configuration configuration = new Configuration(conf); - DynamicResourceConfiguration newconf; + DynamicResourceConfiguration newConf; - InputStream DRInputStream = - this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(configuration, - YarnConfiguration.DR_CONFIGURATION_FILE); - if (DRInputStream != null) { - configuration.addResource(DRInputStream); - newconf = new DynamicResourceConfiguration(configuration, false); + InputStream drInputStream = + this.rmContext.getConfigurationProvider().getConfigurationInputStream( + configuration, YarnConfiguration.DR_CONFIGURATION_FILE); + + if (drInputStream != null) { + newConf = new DynamicResourceConfiguration(configuration, + drInputStream); } else { - newconf = new DynamicResourceConfiguration(configuration, true); + newConf = new DynamicResourceConfiguration(configuration); } - if (newconf.getNodes() == null || newconf.getNodes().length == 0) { - RMAuditLogger.logSuccess(user.getShortUserName(), argName, - "AdminService"); - return response; - } else { + if (newConf.getNodes() != null && newConf.getNodes().length != 0) { Map nodeResourceMap = - newconf.getNodeResourceMap(); - + newConf.getNodeResourceMap(); UpdateNodeResourceRequest updateRequest = - UpdateNodeResourceRequest.newInstance(nodeResourceMap); + UpdateNodeResourceRequest.newInstance(nodeResourceMap); updateNodeResource(updateRequest); - RMAuditLogger.logSuccess(user.getShortUserName(), argName, - "AdminService"); - return response; } + // refresh dynamic resource in ResourceTrackerService + this.rmContext.getResourceTrackerService(). + updateDynamicResourceConfiguration(newConf); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, + "AdminService"); + return response; } catch (IOException ioe) { throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 902244b3be2..b0bc565e6c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -62,6 +63,7 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; +import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; @@ -105,6 +107,7 @@ public class ResourceTrackerService extends AbstractService implements private boolean isDistributedNodeLabelsConf; private boolean isDelegatedCentralizedNodeLabelsConf; + private volatile DynamicResourceConfiguration drConf; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -139,11 +142,11 @@ public class ResourceTrackerService extends AbstractService implements } minAllocMb = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); minAllocVcores = conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, @@ -156,9 +159,42 @@ public class ResourceTrackerService extends AbstractService implements YarnConfiguration.isDelegatedCentralizedNodeLabelConfiguration(conf); } + loadDynamicResourceConfiguration(conf); + super.serviceInit(conf); } + /** + * Load DynamicResourceConfiguration from dynamic-resources.xml. + * @param conf + * @throws IOException + */ + public void loadDynamicResourceConfiguration(Configuration conf) + throws IOException { + try { + // load dynamic-resources.xml + InputStream drInputStream = this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.DR_CONFIGURATION_FILE); + if (drInputStream != null) { + this.drConf = new DynamicResourceConfiguration(conf, drInputStream); + } else { + this.drConf = new DynamicResourceConfiguration(conf); + } + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Update DynamicResourceConfiguration with new configuration. + * @param conf + */ + public void updateDynamicResourceConfiguration( + DynamicResourceConfiguration conf) { + this.drConf = conf; + } + @Override protected void serviceStart() throws Exception { super.serviceStart(); @@ -166,15 +202,14 @@ public class ResourceTrackerService extends AbstractService implements // security is enabled, so no secretManager. Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); - this.server = - rpc.getServer(ResourceTracker.class, this, resourceTrackerAddress, - conf, null, - conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT)); - + this.server = rpc.getServer( + ResourceTracker.class, this, resourceTrackerAddress, conf, null, + conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT)); + // Enable service authorization? if (conf.getBoolean( - CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() @@ -185,12 +220,12 @@ public class ResourceTrackerService extends AbstractService implements } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } - + this.server.start(); conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, - YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, - server.getListenerAddress()); + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + server.getListenerAddress()); } @Override @@ -295,6 +330,19 @@ public class ResourceTrackerService extends AbstractService implements return response; } + // check if node's capacity is load from dynamic-resources.xml + String[] nodes = this.drConf.getNodes(); + String nid = nodeId.toString(); + + if (nodes != null && Arrays.asList(nodes).contains(nid)) { + capability.setMemory(this.drConf.getMemoryPerNode(nid)); + capability.setVirtualCores(this.drConf.getVcoresPerNode(nid)); + if (LOG.isDebugEnabled()) { + LOG.debug("Resource for node: " + nid + " is adjusted to " + + capability + " due to settings in dynamic-resources.xml."); + } + } + // Check if this node has minimum allocations if (capability.getMemory() < minAllocMb || capability.getVirtualCores() < minAllocVcores) { @@ -311,7 +359,7 @@ public class ResourceTrackerService extends AbstractService implements response.setContainerTokenMasterKey(containerTokenSecretManager .getCurrentKey()); response.setNMTokenMasterKey(nmTokenSecretManager - .getCurrentKey()); + .getCurrentKey()); RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, resolve(host), capability, nodeManagerVersion); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java index dd37801a241..045c7bdc957 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource; +import java.io.InputStream; + import java.util.HashMap; import java.util.Map; @@ -38,8 +40,6 @@ public class DynamicResourceConfiguration extends Configuration { private static final Log LOG = LogFactory.getLog(DynamicResourceConfiguration.class); - private static final String DR_CONFIGURATION_FILE = "dynamic-resources.xml"; - @Private public static final String PREFIX = "yarn.resource.dynamic."; @@ -63,15 +63,14 @@ public class DynamicResourceConfiguration extends Configuration { } public DynamicResourceConfiguration(Configuration configuration) { - this(configuration, true); + super(configuration); + addResource(YarnConfiguration.DR_CONFIGURATION_FILE); } public DynamicResourceConfiguration(Configuration configuration, - boolean useLocalConfigurationProvider) { + InputStream drInputStream) { super(configuration); - if (useLocalConfigurationProvider) { - addResource(DR_CONFIGURATION_FILE); - } + addResource(drInputStream); } private String getNodePrefix(String node) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 639b95586a1..4513cbb337a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -27,7 +27,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -202,7 +204,7 @@ public class TestRMAdminService { } @Test - public void testAdminRefreshNodesResourcesWithFileSystemBasedConfigurationProvider() + public void testRefreshNodesResourceWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); @@ -239,6 +241,75 @@ public class TestRMAdminService { Assert.assertEquals("", resourceAfter.toString()); } + @Test + public void testResourcePersistentForNMRegistrationWithNewResource() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + + //upload default configurations + uploadDefaultConfiguration(); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + rm.registerNode("h1:1234", 5120); + } catch(Exception ex) { + fail("Should not get any exceptions"); + } + + NodeId nid = ConverterUtils.toNodeId("h1:1234"); + RMNode ni = rm.getRMContext().getRMNodes().get(nid); + Resource resource = ni.getTotalCapability(); + Assert.assertEquals("", resource.toString()); + + DynamicResourceConfiguration drConf = + new DynamicResourceConfiguration(); + drConf.set("yarn.resource.dynamic.nodes", "h1:1234"); + drConf.set("yarn.resource.dynamic.h1:1234.vcores", "4"); + drConf.set("yarn.resource.dynamic.h1:1234.memory", "4096"); + uploadConfiguration(drConf, "dynamic-resources.xml"); + + rm.adminService.refreshNodesResources( + RefreshNodesResourcesRequest.newInstance()); + + try { + // register the same node again with a different resource. + // validate this won't work as resource cached in RM side. + rm.registerNode("h1:1234", 8192, 8); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + RMNode niAfter = rm.getRMContext().getRMNodes().get(nid); + Resource resourceAfter = niAfter.getTotalCapability(); + Assert.assertEquals("", resourceAfter.toString()); + + // Replace original dr file with an empty dr file, and validate node + // registration with new resources will take effective now. + deleteOnRemoteFileSystem("dynamic-resources.xml"); + DynamicResourceConfiguration emptyDRConf = + new DynamicResourceConfiguration(); + + uploadConfiguration(emptyDRConf, "dynamic-resources.xml"); + rm.adminService.refreshNodesResources( + RefreshNodesResourcesRequest.newInstance()); + try { + // register the same node third time, this time the register resource + // should work. + rm.registerNode("h1:1234", 8192, 8); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + niAfter = rm.getRMContext().getRMNodes().get(nid); + resourceAfter = niAfter.getTotalCapability(); + // new resource in registration should take effective as we empty + // dynamic resource file already. + Assert.assertEquals("", resourceAfter.toString()); + } + @Test public void testAdminAclsWithLocalConfigurationProvider() { rm = new MockRM(configuration); @@ -1006,6 +1077,11 @@ public class TestRMAdminService { uploadToRemoteFileSystem(new Path(csConfFile)); } + private void deleteOnRemoteFileSystem(String fileName) + throws IOException { + fs.delete(new Path(workingPath, fileName)); + } + private void uploadDefaultConfiguration() throws IOException { Configuration conf = new Configuration(); uploadConfiguration(conf, "core-site.xml"); From 8bfaa80037365c0790083313a905d1e7d88b0682 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 28 Mar 2016 14:13:48 -0700 Subject: [PATCH 03/75] HADOOP-10965. Print fully qualified path in CommandWithDestination error messages. Contributed by John Zhuge. --- .../org/apache/hadoop/fs/PathIOException.java | 9 ++ .../fs/shell/CommandWithDestination.java | 3 +- .../org/apache/hadoop/fs/shell/Touch.java | 3 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 52 +++++++++-- .../apache/hadoop/fs/TestFsShellTouch.java | 88 +++++++++++++++++++ 5 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java index 459a83669aa..deb3880ee41 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java @@ -33,6 +33,7 @@ public class PathIOException extends IOException { // uris with no authority private String operation; private String path; + private String fullyQualifiedPath; private String targetPath; /** @@ -68,6 +69,11 @@ public class PathIOException extends IOException { this.path = path; } + public PathIOException withFullyQualifiedPath(String fqPath) { + fullyQualifiedPath = fqPath; + return this; + } + /** Format: * cmd: {operation} `path' {to `target'}: error string */ @@ -85,6 +91,9 @@ public class PathIOException extends IOException { if (getCause() != null) { message.append(": " + getCause().getMessage()); } + if (fullyQualifiedPath != null && !fullyQualifiedPath.equals(path)) { + message.append(": ").append(formatPath(fullyQualifiedPath)); + } return message.toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index 4b8d3bc11ac..5fcfdf88eff 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -220,7 +220,8 @@ abstract class CommandWithDestination extends FsCommand { throw new PathExistsException(dst.toString()); } } else if (!dst.parentExists()) { - throw new PathNotFoundException(dst.toString()); + throw new PathNotFoundException(dst.toString()) + .withFullyQualifiedPath(dst.path.toUri().toString()); } super.processArguments(args); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java index 72a463af7ba..a6c751ea6f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java @@ -72,7 +72,8 @@ class Touch extends FsCommand { @Override protected void processNonexistentPath(PathData item) throws IOException { if (!item.parentExists()) { - throw new PathNotFoundException(item.toString()); + throw new PathNotFoundException(item.toString()) + .withFullyQualifiedPath(item.path.toUri().toString()); } touchz(item); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 6b5de745c0a..c00ed66d6fa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -18,18 +18,28 @@ package org.apache.hadoop.fs; -import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -public class TestFsShellCopy { +public class TestFsShellCopy { + static final Log LOG = LogFactory.getLog(TestFsShellCopy.class); + static Configuration conf; static FsShell shell; static LocalFileSystem lfs; @@ -42,9 +52,10 @@ public class TestFsShellCopy { lfs = FileSystem.getLocal(conf); testRootDir = lfs.makeQualified(new Path( System.getProperty("test.build.data","test/build/data"), - "testShellCopy")); + "testFsShellCopy")); - lfs.mkdirs(testRootDir); + lfs.mkdirs(testRootDir); + lfs.setWorkingDirectory(testRootDir); srcPath = new Path(testRootDir, "srcFile"); dstPath = new Path(testRootDir, "dstFile"); } @@ -62,6 +73,16 @@ public class TestFsShellCopy { assertTrue(lfs.exists(lfs.getChecksumFile(srcPath))); } + private void shellRun(int n, String ... args) throws Exception { + assertEquals(n, shell.run(args)); + } + + private int shellRun(String... args) throws Exception { + int exitCode = shell.run(args); + LOG.info("exit " + exitCode + " - " + StringUtils.join(" ", args)); + return exitCode; + } + @Test public void testCopyNoCrc() throws Exception { shellRun(0, "-get", srcPath.toString(), dstPath.toString()); @@ -95,10 +116,6 @@ public class TestFsShellCopy { assertEquals(expectChecksum, hasChecksum); } - private void shellRun(int n, String ... args) throws Exception { - assertEquals(n, shell.run(args)); - } - @Test public void testCopyFileFromLocal() throws Exception { Path testRoot = new Path(testRootDir, "testPutFile"); @@ -571,4 +588,23 @@ public class TestFsShellCopy { String s = (p == null) ? Path.CUR_DIR : p.toString(); return s.isEmpty() ? Path.CUR_DIR : s; } + + /** + * Test copy to a path with non-existent parent directory. + */ + @Test + public void testCopyNoParent() throws Exception { + final String noDirName = "noDir"; + final Path noDir = new Path(noDirName); + lfs.delete(noDir, true); + assertThat(lfs.exists(noDir), is(false)); + + assertThat("Expected failed put to a path without parent directory", + shellRun("-put", srcPath.toString(), noDirName + "/foo"), is(not(0))); + + // Note the trailing '/' in the target path. + assertThat("Expected failed copyFromLocal to a non-existent directory", + shellRun("-copyFromLocal", srcPath.toString(), noDirName + "/"), + is(not(0))); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java new file mode 100644 index 00000000000..988a57c2950 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestFsShellTouch { + static final Log LOG = LogFactory.getLog(TestFsShellTouch.class); + + static FsShell shell; + static LocalFileSystem lfs; + static Path testRootDir; + + @BeforeClass + public static void setup() throws Exception { + Configuration conf = new Configuration(); + shell = new FsShell(conf); + lfs = FileSystem.getLocal(conf); + testRootDir = lfs.makeQualified(new Path( + System.getProperty("test.build.data","test/build/data"), + "testFsShell")); + + lfs.mkdirs(testRootDir); + lfs.setWorkingDirectory(testRootDir); + } + + @Before + public void prepFiles() throws Exception { + lfs.setVerifyChecksum(true); + lfs.setWriteChecksum(true); + } + + private int shellRun(String... args) throws Exception { + int exitCode = shell.run(args); + LOG.info("exit " + exitCode + " - " + StringUtils.join(" ", args)); + return exitCode; + } + + @Test + public void testTouchz() throws Exception { + // Ensure newFile does not exist + final String newFileName = "newFile"; + final Path newFile = new Path(newFileName); + lfs.delete(newFile, true); + assertThat(lfs.exists(newFile), is(false)); + + assertThat("Expected successful touchz on a new file", + shellRun("-touchz", newFileName), is(0)); + shellRun("-ls", newFileName); + + assertThat("Expected successful touchz on an existing zero-length file", + shellRun("-touchz", newFileName), is(0)); + + // Ensure noDir does not exist + final String noDirName = "noDir"; + final Path noDir = new Path(noDirName); + lfs.delete(noDir, true); + assertThat(lfs.exists(noDir), is(false)); + + assertThat("Expected failed touchz in a non-existent directory", + shellRun("-touchz", noDirName + "/foo"), is(not(0))); + } +} From 948b75807068c304ffe789e32f2b850c0d653e0a Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 28 Mar 2016 23:00:56 +0000 Subject: [PATCH 04/75] YARN-4773. Log aggregation performs extraneous filesystem operations when rolling log aggregation is disabled. Contributed by Jun Gong --- .../logaggregation/AppLogAggregatorImpl.java | 14 ++++++++++---- .../logaggregation/TestLogAggregationService.java | 12 +++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index da7fc14cb6a..fed4a3bab65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final long rollingMonitorInterval; private final boolean logAggregationInRolling; private final NodeId nodeId; - // This variable is only for testing - private final AtomicBoolean waiting = new AtomicBoolean(false); - // This variable is only for testing + // These variables are only for testing + private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; + private int cleanupOldLogTimes = 0; private boolean renameTemporaryLogFileFailed = false; @@ -365,8 +365,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { // Before upload logs, make sure the number of existing logs // is smaller than the configured NM log aggregation retention size. - if (uploadedLogsInThisCycle) { + if (uploadedLogsInThisCycle && logAggregationInRolling) { cleanOldLogs(); + cleanupOldLogTimes++; } if (writer != null) { @@ -689,4 +690,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { public int getLogAggregationTimes() { return this.logAggregationTimes; } + + @VisibleForTesting + int getCleanupOldLogTimes() { + return this.cleanupOldLogTimes; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 0392b3899dc..0445c7917de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2276,7 +2276,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { Records.newRecord(LogAggregationContext.class); logAggregationContext.setLogAggregationPolicyClassName( FailedOrKilledContainerLogAggregationPolicy.class.getName()); - verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2); + verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0); } @Test (timeout = 20000) @@ -2290,13 +2290,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest { AMOnlyLogAggregationPolicy.class.getName()); contextWithAMOnly.setRolledLogsIncludePattern("sys*"); contextWithAMOnly.setRolledLogsExcludePattern("std_final"); - verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4); + verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1); } private void verifySkipUnnecessaryNNOperations( LogAggregationContext logAggregationContext, - int expectedLogAggregationTimes, int expectedAggregationReportNum) - throws Exception { + int expectedLogAggregationTimes, int expectedAggregationReportNum, + int expectedCleanupOldLogsTimes) throws Exception { LogAggregationService logAggregationService = new LogAggregationService( dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); @@ -2307,7 +2307,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { null, this.acls, logAggregationContext)); // Container finishes - String[] logFiles = new String[] { "stdout" }; + String[] logFiles = new String[] { "sysout" }; finishContainer(appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, 0, logFiles); AppLogAggregatorImpl aggregator = @@ -2327,6 +2327,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { aggregator.getLogAggregationTimes()); assertEquals(expectedAggregationReportNum, this.context.getLogAggregationStatusForApps().size()); + assertEquals(expectedCleanupOldLogsTimes, + aggregator.getCleanupOldLogTimes()); } private int numOfLogsAvailable(LogAggregationService logAggregationService, From 0ef8bbfd8791899c7cfed3dd9c1670182fd87575 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 29 Mar 2016 10:32:21 +0800 Subject: [PATCH 05/75] MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B --- .../v2/hs/TestHistoryFileManager.java | 5 +++-- .../hadoop/mapred/TestFieldSelection.java | 5 +++-- .../mapred/TestMRTimelineEventHandling.java | 19 ++++++++++++------- .../hadoop/mapred/TestMiniMRChildTask.java | 16 ++++++++++------ .../mapred/TestOldCombinerGrouping.java | 5 +++-- .../mapred/lib/aggregate/TestAggregates.java | 7 ++++--- .../mapreduce/TestNewCombinerGrouping.java | 5 +++-- .../aggregate/TestMapReduceAggregates.java | 5 +++-- .../lib/input/TestCombineFileInputFormat.java | 4 +++- 9 files changed, 44 insertions(+), 27 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 18e3e6a38ac..aa2e9796511 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -71,8 +71,9 @@ public class TestHistoryFileManager { Configuration conf = new HdfsConfiguration(); Configuration conf2 = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).build(); - conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2"); + conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get( + MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory()) + + "_2"); dfsCluster2 = new MiniDFSCluster.Builder(conf2).build(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java index 239c239230e..29333b7bfdb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java @@ -44,8 +44,9 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); FileSystem fs = FileSystem.get(conf); int numOfInputLines = 10; - Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test"); - Path INPUT_DIR = new Path("build/test/input_for_field_selection_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index a82007f97c9..40ed9ad1160 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -97,9 +97,10 @@ public class TestMRTimelineEventHandling { + cluster.getApplicationHistoryServer().getPort()); TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -155,8 +156,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); RunningJob job = UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.SUCCEEDED, @@ -195,8 +198,10 @@ public class TestMRTimelineEventHandling { TimelineStore ts = cluster.getApplicationHistoryServer() .getTimelineStore(); - Path inDir = new Path("input"); - Path outDir = new Path("output"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path inDir = new Path(localPathRoot, "input"); + Path outDir = new Path(localPathRoot, "output"); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); RunningJob job = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index 74be8a758db..cbeeccffc7a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -436,8 +436,10 @@ public class TestMiniMRChildTask { JobConf conf = new JobConf(mr.getConfig()); // initialize input, output directories - Path inDir = new Path("input"); - Path outDir = new Path("output"); + Path rootDir = new Path(System.getProperty("test.build.data", + "build/test/data")); + Path inDir = new Path(rootDir, "input"); + Path outDir = new Path(rootDir, "output"); String input = "The input"; // set config to use the ExecutionEnvCheckMapClass map class @@ -462,9 +464,10 @@ public class TestMiniMRChildTask { public void testTaskEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, false); outFs.delete(outDir, true); @@ -485,9 +488,10 @@ public class TestMiniMRChildTask { public void testTaskOldEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); // initialize input, output directories - Path inDir = new Path("testing/wc/input1"); - Path outDir = new Path("testing/wc/output1"); + Path inDir = new Path(baseDir + "/testing/wc/input1"); + Path outDir = new Path(baseDir + "/testing/wc/output1"); FileSystem outFs = outDir.getFileSystem(conf); runTestTaskEnv(conf, inDir, outDir, true); outFs.delete(outDir, true); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java index 80a670209d2..046c2d37eed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java @@ -37,8 +37,9 @@ import java.util.Set; import java.util.UUID; public class TestOldCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map implements Mapper { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java index 6da96ce22bd..e2fdd429c56 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java @@ -46,8 +46,9 @@ public class TestAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); @@ -115,7 +116,7 @@ public class TestAggregates extends TestCase { outdata = outdata.substring(0, expectedOutput.toString().length()); assertEquals(expectedOutput.toString(),outdata); - //fs.delete(OUTPUT_DIR); + fs.delete(OUTPUT_DIR, true); fs.delete(INPUT_DIR, true); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java index ab9ddbb4aaa..c2054f1d4c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java @@ -39,8 +39,9 @@ import java.util.Set; import java.util.UUID; public class TestNewCombinerGrouping { - private static String TEST_ROOT_DIR = - new File("build", UUID.randomUUID().toString()).getAbsolutePath(); + private static String TEST_ROOT_DIR = new File(System.getProperty( + "test.build.data", "build/test/data"), UUID.randomUUID().toString()) + .getAbsolutePath(); public static class Map extends Mapper { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java index f24dffe2655..788ad41ff96 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java @@ -51,8 +51,9 @@ public class TestMapReduceAggregates extends TestCase { FileSystem fs = FileSystem.get(conf); int numOfInputLines = 20; - Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); - Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); + String baseDir = System.getProperty("test.build.data", "build/test/data"); + Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test"); + Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test"); String inputFile = "input.txt"; fs.delete(INPUT_DIR, true); fs.mkdirs(INPUT_DIR); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index b49f2d831ab..1fca5c982bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -1772,7 +1772,9 @@ public class TestCombineFileInputFormat { // default fs path assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString()); // add a local file - Path localPath = new Path("testFile1"); + String localPathRoot = System.getProperty("test.build.data", + "build/test/data"); + Path localPath = new Path(localPathRoot, "testFile1"); FileSystem lfs = FileSystem.getLocal(conf); FSDataOutputStream dos = lfs.create(localPath); dos.writeChars("Local file for CFIF"); From 1f004b3367c57de9e8a67040a57efc31c9ba8ee2 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Tue, 29 Mar 2016 11:25:41 +0800 Subject: [PATCH 06/75] HDFS-9871. "Bytes Being Moved" -ve(-1 B) when cluster was already balanced. (Contributed by Brahma Reddy Battulla) --- .../java/org/apache/hadoop/hdfs/server/balancer/Balancer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 643589767d3..a7600e00913 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -583,7 +583,7 @@ public class Balancer { final long bytesLeftToMove = init(reports); if (bytesLeftToMove == 0) { System.out.println("The cluster is balanced. Exiting..."); - return newResult(ExitStatus.SUCCESS, bytesLeftToMove, -1); + return newResult(ExitStatus.SUCCESS, bytesLeftToMove, 0); } else { LOG.info( "Need to move "+ StringUtils.byteDesc(bytesLeftToMove) + " to make the cluster balanced." ); From 80182809aed76fb9522d25db9bf5e445395dd573 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 28 Mar 2016 22:17:45 -0700 Subject: [PATCH 07/75] YARN-4863. AHS Security login should be in serviceInit() instead of serviceStart(). Contributed by Junping Du --- .../ApplicationHistoryServer.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index cedbd2eb1ae..d241077c4fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -86,7 +86,14 @@ public class ApplicationHistoryServer extends CompositeService { @Override protected void serviceInit(Configuration conf) throws Exception { - // init timeline services first + + // do security login first. + try { + doSecureLogin(conf); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to login", ie); + } + // init timeline services timelineStore = createTimelineStore(conf); addIfService(timelineStore); secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); @@ -111,12 +118,6 @@ public class ApplicationHistoryServer extends CompositeService { @Override protected void serviceStart() throws Exception { - try { - doSecureLogin(getConfig()); - } catch(IOException ie) { - throw new YarnRuntimeException("Failed to login", ie); - } - super.serviceStart(); startWebApp(); } From 0050fa5f1c20087009bd76a0bb2183a479f787f0 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 29 Mar 2016 18:17:52 +0900 Subject: [PATCH 08/75] MAPREDUCE-6543. Migrate MR client test cases part 2. Contributed by Dustin Cote. --- .../java/org/apache/hadoop/fs/DFSCIOTest.java | 8 +-- .../org/apache/hadoop/fs/TestFileSystem.java | 20 +++++-- .../java/org/apache/hadoop/fs/TestJHLA.java | 6 +-- .../io/TestSequenceFileMergeProgress.java | 12 +++-- .../mapred/ClusterMapReduceTestCase.java | 14 ++--- .../apache/hadoop/mapred/TestAuditLogger.java | 9 ++-- .../apache/hadoop/mapred/TestBadRecords.java | 8 ++- .../mapred/TestClusterMapReduceTestCase.java | 10 ++++ .../org/apache/hadoop/mapred/TestCollect.java | 10 ++-- .../mapred/TestCommandLineJobSubmission.java | 9 ++-- .../hadoop/mapred/TestFieldSelection.java | 6 ++- .../mapred/TestFileInputFormatPathFilter.java | 19 ++++--- .../hadoop/mapred/TestGetSplitHosts.java | 7 +-- .../hadoop/mapred/TestIFileStreams.java | 13 ++--- .../apache/hadoop/mapred/TestInputPath.java | 7 +-- .../hadoop/mapred/TestJavaSerialization.java | 10 ++-- .../org/apache/hadoop/mapred/TestJobName.java | 6 +++ .../hadoop/mapred/TestJobSysDirWithDFS.java | 10 ++-- .../mapred/TestKeyValueTextInputFormat.java | 15 +++--- .../apache/hadoop/mapred/TestLazyOutput.java | 7 +-- .../mapred/TestMRCJCFileInputFormat.java | 32 +++++++----- .../mapred/TestMRCJCFileOutputCommitter.java | 28 ++++++---- .../apache/hadoop/mapred/TestMapProgress.java | 9 ++-- .../org/apache/hadoop/mapred/TestMerge.java | 7 +-- .../hadoop/mapred/TestMiniMRBringup.java | 6 ++- .../hadoop/mapred/TestMiniMRDFSCaching.java | 14 +++-- .../mapred/TestMultiFileInputFormat.java | 19 +++---- .../hadoop/mapred/TestMultiFileSplit.java | 10 ++-- .../mapred/TestMultipleLevelCaching.java | 12 +++-- .../mapred/TestMultipleTextOutputFormat.java | 23 ++++---- .../apache/hadoop/mapred/TestReduceFetch.java | 10 ++-- .../mapred/TestReduceFetchFromPartialMem.java | 46 +++++++--------- .../apache/hadoop/mapred/TestReduceTask.java | 18 ++++--- .../TestSequenceFileAsBinaryInputFormat.java | 19 ++++--- .../TestSequenceFileAsBinaryOutputFormat.java | 31 +++++++---- .../TestSequenceFileAsTextInputFormat.java | 27 +++++----- .../mapred/TestSequenceFileInputFilter.java | 32 ++++++------ .../mapred/TestSequenceFileInputFormat.java | 23 ++++---- .../hadoop/mapred/TestSortedRanges.java | 19 ++++--- .../TestSpecialCharactersInOutputPath.java | 21 ++++---- .../mapred/TestStatisticsCollector.java | 10 ++-- .../mapred/TestUserDefinedCounters.java | 24 +++++---- .../hadoop/mapred/TestWritableJobConf.java | 20 ++++--- .../apache/hadoop/mapred/TestYARNRunner.java | 8 +-- .../hadoop/mapred/join/TestDatamerge.java | 42 ++++++++------- .../hadoop/mapred/join/TestTupleWritable.java | 24 ++++++--- .../TestWrappedRecordReaderClassloader.java | 7 +-- .../mapred/lib/TestDelegatingInputFormat.java | 9 ++-- .../mapred/lib/TestLineInputFormat.java | 7 +-- .../hadoop/mapred/lib/TestMultipleInputs.java | 2 - .../mapred/lib/aggregate/TestAggregates.java | 7 +-- .../mapred/lib/db/TestConstructQuery.java | 16 +++--- .../apache/hadoop/mapred/pipes/TestPipes.java | 9 ++-- .../hadoop/mapreduce/TestLocalRunner.java | 34 +++++++----- .../hadoop/mapreduce/TestMRJobClient.java | 49 +++++++++-------- .../mapreduce/TestMapReduceLazyOutput.java | 9 ++-- .../hadoop/mapreduce/TestValueIterReset.java | 8 +-- .../TestYarnClientProtocolProvider.java | 5 +- .../aggregate/TestMapReduceAggregates.java | 23 ++++---- .../mapreduce/lib/db/TestDBOutputFormat.java | 17 +++--- .../mapreduce/lib/db/TestIntegerSplitter.java | 15 ++++-- .../mapreduce/lib/db/TestTextSplitter.java | 18 +++++-- .../lib/fieldsel/TestMRFieldSelection.java | 20 ++++--- ...TestMRSequenceFileAsBinaryInputFormat.java | 21 +++++--- .../TestMRSequenceFileAsTextInputFormat.java | 27 ++++++---- .../input/TestMRSequenceFileInputFilter.java | 39 +++++++------- .../lib/input/TestNLineInputFormat.java | 34 +++++++----- .../mapreduce/lib/join/TestJoinDatamerge.java | 52 +++++++++++-------- .../lib/join/TestJoinProperties.java | 44 ++++++++-------- .../lib/join/TestJoinTupleWritable.java | 24 ++++++--- .../lib/join/TestWrappedRRClassloader.java | 17 ++++-- ...estMRSequenceFileAsBinaryOutputFormat.java | 35 +++++++++---- .../lib/partition/TestBinaryPartitioner.java | 16 ++++-- .../lib/partition/TestKeyFieldHelper.java | 9 +++- .../TestMRKeyFieldBasedPartitioner.java | 6 ++- .../partition/TestTotalOrderPartitioner.java | 11 ++-- .../util/TestMRAsyncDiskService.java | 15 ++++-- .../mapreduce/v2/TestMiniMRProxyUser.java | 30 ++++++----- .../mapreduce/v2/TestNonExistentJob.java | 18 ++++--- .../streaming/TestStreamingBadRecords.java | 9 +++- 80 files changed, 835 insertions(+), 567 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java index 1caa2cdae6c..12bec0869f6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java @@ -28,8 +28,6 @@ import java.io.PrintStream; import java.util.Date; import java.util.StringTokenizer; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -39,8 +37,9 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.mapred.*; import org.junit.Ignore; +import org.junit.Test; -/** + /** * Distributed i/o benchmark. *

* This test writes into or reads from a specified number of files. @@ -68,7 +67,7 @@ import org.junit.Ignore; * */ @Ignore -public class DFSCIOTest extends TestCase { +public class DFSCIOTest { // Constants private static final Log LOG = LogFactory.getLog(DFSCIOTest.class); private static final int TEST_TYPE_READ = 0; @@ -98,6 +97,7 @@ public class DFSCIOTest extends TestCase { * * @throws Exception */ + @Test public void testIOs() throws Exception { testIOs(10, 10); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java index 4146b139c50..f0300b368a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java @@ -34,8 +34,6 @@ import java.util.HashMap; import java.net.InetSocketAddress; import java.net.URI; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -50,8 +48,15 @@ import org.apache.hadoop.mapred.*; import org.apache.hadoop.mapred.lib.LongSumReducer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; -public class TestFileSystem extends TestCase { + +public class TestFileSystem { private static final Log LOG = FileSystem.LOG; private static Configuration conf = new Configuration(); @@ -66,6 +71,7 @@ public class TestFileSystem extends TestCase { private static Path READ_DIR = new Path(ROOT, "fs_read"); private static Path DATA_DIR = new Path(ROOT, "fs_data"); + @Test public void testFs() throws Exception { testFs(10 * MEGA, 100, 0); } @@ -90,7 +96,8 @@ public class TestFileSystem extends TestCase { fs.delete(READ_DIR, true); } - public static void testCommandFormat() throws Exception { + @Test + public void testCommandFormat() throws Exception { // This should go to TestFsShell.java when it is added. CommandFormat cf; cf= new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc"); @@ -488,6 +495,7 @@ public class TestFileSystem extends TestCase { } } + @Test public void testFsCache() throws Exception { { long now = System.currentTimeMillis(); @@ -561,6 +569,7 @@ public class TestFileSystem extends TestCase { + StringUtils.toUpperCase(add.getHostName()) + ":" + add.getPort())); } + @Test public void testFsClose() throws Exception { { Configuration conf = new Configuration(); @@ -569,6 +578,7 @@ public class TestFileSystem extends TestCase { } } + @Test public void testFsShutdownHook() throws Exception { final Set closed = Collections.synchronizedSet(new HashSet()); Configuration conf = new Configuration(); @@ -600,7 +610,7 @@ public class TestFileSystem extends TestCase { assertTrue(closed.contains(fsWithoutAuto)); } - + @Test public void testCacheKeysAreCaseInsensitive() throws Exception { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java index f2bc4edc46d..31950fd6104 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java @@ -23,19 +23,18 @@ import java.io.FileOutputStream; import java.io.OutputStreamWriter; import java.io.File; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.After; import org.junit.Before; +import org.junit.Test; /** * Test Job History Log Analyzer. * * @see JHLogAnalyzer */ -public class TestJHLA extends TestCase { +public class TestJHLA { private static final Log LOG = LogFactory.getLog(JHLogAnalyzer.class); private String historyLog = System.getProperty("test.build.data", "build/test/data") + "/history/test.log"; @@ -133,6 +132,7 @@ public class TestJHLA extends TestCase { /** * Run log analyzer in test mode for file test.log. */ + @Test public void testJHLA() { String[] args = {"-test", historyLog, "-jobDelimiter", ".!!FILE=.*!!"}; JHLogAnalyzer.main(args); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java index 1d7b98a6719..97dfa26acf4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java @@ -32,21 +32,25 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.mapred.*; -import junit.framework.TestCase; import org.apache.commons.logging.*; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -public class TestSequenceFileMergeProgress extends TestCase { +public class TestSequenceFileMergeProgress { private static final Log LOG = FileInputFormat.LOG; private static final int RECORDS = 10000; - + + @Test public void testMergeProgressWithNoCompression() throws IOException { runTest(SequenceFile.CompressionType.NONE); } + @Test public void testMergeProgressWithRecordCompression() throws IOException { runTest(SequenceFile.CompressionType.RECORD); } + @Test public void testMergeProgressWithBlockCompression() throws IOException { runTest(SequenceFile.CompressionType.BLOCK); } @@ -92,7 +96,7 @@ public class TestSequenceFileMergeProgress extends TestCase { count++; } assertEquals(RECORDS, count); - assertEquals(1.0f, rIter.getProgress().get()); + assertEquals(1.0f, rIter.getProgress().get(), 0.0000); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ClusterMapReduceTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ClusterMapReduceTestCase.java index 5bf4ff11b89..8d33b1580a8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ClusterMapReduceTestCase.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ClusterMapReduceTestCase.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.mapred; -import junit.framework.TestCase; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.Map; @@ -41,7 +42,7 @@ import java.util.Properties; *

* The DFS filesystem is formated before the testcase starts and after it ends. */ -public abstract class ClusterMapReduceTestCase extends TestCase { +public abstract class ClusterMapReduceTestCase { private MiniDFSCluster dfsCluster = null; private MiniMRCluster mrCluster = null; @@ -50,9 +51,8 @@ public abstract class ClusterMapReduceTestCase extends TestCase { * * @throws Exception */ - protected void setUp() throws Exception { - super.setUp(); - + @Before + public void setUp() throws Exception { startCluster(true, null); } @@ -139,9 +139,9 @@ public abstract class ClusterMapReduceTestCase extends TestCase { * * @throws Exception */ - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { stopCluster(); - super.tearDown(); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java index 353185b59e3..bc85703bc84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java @@ -28,13 +28,13 @@ import org.apache.hadoop.ipc.TestRPC.TestImpl; import org.apache.hadoop.ipc.TestRPC.TestProtocol; import org.apache.hadoop.mapred.AuditLogger.Keys; import org.apache.hadoop.net.NetUtils; - -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.assertEquals; /** * Tests {@link AuditLogger}. */ -public class TestAuditLogger extends TestCase { +public class TestAuditLogger { private static final String USER = "test"; private static final String OPERATION = "oper"; private static final String TARGET = "tgt"; @@ -44,6 +44,7 @@ public class TestAuditLogger extends TestCase { /** * Test the AuditLog format with key-val pair. */ + @Test public void testKeyValLogFormat() { StringBuilder actLog = new StringBuilder(); StringBuilder expLog = new StringBuilder(); @@ -114,6 +115,7 @@ public class TestAuditLogger extends TestCase { /** * Test {@link AuditLogger} without IP set. */ + @Test public void testAuditLoggerWithoutIP() throws Exception { // test without ip testSuccessLogFormat(false); @@ -137,6 +139,7 @@ public class TestAuditLogger extends TestCase { /** * Test {@link AuditLogger} with IP set. */ + @Test public void testAuditLoggerWithIP() throws Exception { Configuration conf = new Configuration(); // start the IPC server diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java index ea9f3d3f989..c2d6257823e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java @@ -40,6 +40,11 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.util.ReflectionUtils; import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; @Ignore public class TestBadRecords extends ClusterMapReduceTestCase { @@ -206,7 +211,8 @@ public class TestBadRecords extends ClusterMapReduceTestCase { } return processed; } - + + @Test public void testBadMapRed() throws Exception { JobConf conf = createJobConf(); conf.setMapperClass(BadMapper.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java index ada2d0c634b..f04fbd7a29a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java @@ -29,6 +29,12 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; public class TestClusterMapReduceTestCase extends ClusterMapReduceTestCase { public void _testMapReduce(boolean restart) throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); @@ -85,14 +91,17 @@ public class TestClusterMapReduceTestCase extends ClusterMapReduceTestCase { } + @Test public void testMapReduce() throws Exception { _testMapReduce(false); } + @Test public void testMapReduceRestarting() throws Exception { _testMapReduce(true); } + @Test public void testDFSRestart() throws Exception { Path file = new Path(getInputDir(), "text.txt"); OutputStream os = getFileSystem().create(file); @@ -109,6 +118,7 @@ public class TestClusterMapReduceTestCase extends ClusterMapReduceTestCase { } + @Test public void testMRConfig() throws Exception { JobConf conf = createJobConf(); assertNull(conf.get("xyz")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCollect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCollect.java index 4bd20d54ad5..595d09cc2a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCollect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCollect.java @@ -21,15 +21,15 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.UtilsForTests.RandomInputFormat; import org.apache.hadoop.mapreduce.MRConfig; +import org.junit.Test; -import junit.framework.TestCase; import java.io.*; import java.util.*; /** * TestCollect checks if the collect can handle simultaneous invocations. */ -public class TestCollect extends TestCase +public class TestCollect { final static Path OUTPUT_DIR = new Path("build/test/test.collect.output"); static final int NUM_FEEDERS = 10; @@ -127,7 +127,7 @@ public class TestCollect extends TestCase conf.setNumMapTasks(1); conf.setNumReduceTasks(1); } - + @Test public void testCollect() throws IOException { JobConf conf = new JobConf(); configure(conf); @@ -144,9 +144,5 @@ public class TestCollect extends TestCase fs.delete(OUTPUT_DIR, true); } } - - public static void main(String[] args) throws IOException { - new TestCollect().testCollect(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java index 69353871cf4..7cf5e71e1a5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java @@ -21,28 +21,29 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.Ignore; +import org.junit.Test; +import static org.junit.Assert.assertTrue; /** * check for the job submission options of * -libjars -files -archives */ @Ignore -public class TestCommandLineJobSubmission extends TestCase { - // Input output paths for this.. +public class TestCommandLineJobSubmission { + // Input output paths for this.. // these are all dummy and does not test // much in map reduce except for the command line // params static final Path input = new Path("/test/input/"); static final Path output = new Path("/test/output"); File buildDir = new File(System.getProperty("test.build.data", "/tmp")); + @Test public void testJobShell() throws Exception { MiniDFSCluster dfs = null; MiniMRCluster mr = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java index 29333b7bfdb..868896815ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java @@ -23,11 +23,12 @@ import org.apache.hadoop.mapred.lib.*; import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper; import org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -import junit.framework.TestCase; import java.text.NumberFormat; -public class TestFieldSelection extends TestCase { +public class TestFieldSelection { private static NumberFormat idFormat = NumberFormat.getInstance(); static { @@ -35,6 +36,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); idFormat.setGroupingUsed(false); } + @Test public void testFieldSelection() throws Exception { launch(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormatPathFilter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormatPathFilter.java index 1c8be66d084..d87f6fd91a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormatPathFilter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormatPathFilter.java @@ -17,12 +17,14 @@ */ package org.apache.hadoop.mapred; -import junit.framework.TestCase; - import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.io.Writer; @@ -30,7 +32,7 @@ import java.io.OutputStreamWriter; import java.util.Set; import java.util.HashSet; -public class TestFileInputFormatPathFilter extends TestCase { +public class TestFileInputFormatPathFilter { public static class DummyFileInputFormat extends FileInputFormat { @@ -55,12 +57,12 @@ public class TestFileInputFormatPathFilter extends TestCase { new Path(new Path(System.getProperty("test.build.data", "."), "data"), "TestFileInputFormatPathFilter"); - + @Before public void setUp() throws Exception { tearDown(); localFs.mkdirs(workDir); } - + @After public void tearDown() throws Exception { if (localFs.exists(workDir)) { localFs.delete(workDir, true); @@ -129,18 +131,19 @@ public class TestFileInputFormatPathFilter extends TestCase { assertEquals(createdFiles, computedFiles); } + @Test public void testWithoutPathFilterWithoutGlob() throws Exception { _testInputFiles(false, false); } - + @Test public void testWithoutPathFilterWithGlob() throws Exception { _testInputFiles(false, true); } - + @Test public void testWithPathFilterWithoutGlob() throws Exception { _testInputFiles(true, false); } - + @Test public void testWithPathFilterWithGlob() throws Exception { _testInputFiles(true, true); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestGetSplitHosts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestGetSplitHosts.java index 7891bca7990..3d1c2e71bff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestGetSplitHosts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestGetSplitHosts.java @@ -20,10 +20,11 @@ package org.apache.hadoop.mapred; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.net.NetworkTopology; -import junit.framework.TestCase; - -public class TestGetSplitHosts extends TestCase { +import org.junit.Test; +import static org.junit.Assert.assertTrue; +public class TestGetSplitHosts { + @Test public void testGetSplitHosts() throws Exception { int numBlocks = 3; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java index 86431e5c135..2b97d3b95ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java @@ -21,11 +21,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; +import org.junit.Test; +import static org.junit.Assert.fail; +import static org.junit.Assert.assertEquals; -import junit.framework.TestCase; - -public class TestIFileStreams extends TestCase { - +public class TestIFileStreams { + @Test public void testIFileStream() throws Exception { final int DLEN = 100; DataOutputBuffer dob = new DataOutputBuffer(DLEN + 4); @@ -42,7 +43,7 @@ public class TestIFileStreams extends TestCase { } ifis.close(); } - + @Test public void testBadIFileStream() throws Exception { final int DLEN = 100; DataOutputBuffer dob = new DataOutputBuffer(DLEN + 4); @@ -73,7 +74,7 @@ public class TestIFileStreams extends TestCase { } fail("Did not detect bad data in checksum"); } - + @Test public void testBadLength() throws Exception { final int DLEN = 100; DataOutputBuffer dob = new DataOutputBuffer(DLEN + 4); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestInputPath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestInputPath.java index 1398f9e5aaa..0c20c335d89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestInputPath.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestInputPath.java @@ -17,14 +17,15 @@ */ package org.apache.hadoop.mapred; -import junit.framework.TestCase; - import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -public class TestInputPath extends TestCase { +public class TestInputPath { + @Test public void testInputPath() throws Exception { JobConf jobConf = new JobConf(); Path workingDir = jobConf.getWorkingDirectory(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java index 265118a70f6..a787e68c124 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java @@ -26,8 +26,6 @@ import java.io.Writer; import java.util.Iterator; import java.util.StringTokenizer; -import junit.framework.TestCase; - import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -36,8 +34,11 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.serializer.JavaSerializationComparator; import org.apache.hadoop.mapreduce.MRConfig; +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; -public class TestJavaSerialization extends TestCase { +public class TestJavaSerialization { private static String TEST_ROOT_DIR = new File(System.getProperty("test.build.data", "/tmp")).toURI() @@ -90,7 +91,7 @@ public class TestJavaSerialization extends TestCase { wr.write("b a\n"); wr.close(); } - + @Test public void testMapReduceJob() throws Exception { JobConf conf = new JobConf(TestJavaSerialization.class); @@ -149,6 +150,7 @@ public class TestJavaSerialization extends TestCase { * coupled to Writable types, if so, the job will fail. * */ + @Test public void testWriteToSequencefile() throws Exception { JobConf conf = new JobConf(TestJavaSerialization.class); conf.setJobName("JavaSerialization"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java index 4b62b4a1d8e..2659a14a70b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java @@ -29,8 +29,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.lib.IdentityMapper; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + public class TestJobName extends ClusterMapReduceTestCase { + @Test public void testComplexName() throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); @@ -65,6 +70,7 @@ public class TestJobName extends ClusterMapReduceTestCase { reader.close(); } + @Test public void testComplexNameWithRegex() throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java index 109c781c2b0..3dbc5777bd5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java @@ -21,8 +21,6 @@ package org.apache.hadoop.mapred; import java.io.DataOutputStream; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -32,11 +30,15 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; /** * A JUnit test to test Job System Directory with Mini-DFS. */ -public class TestJobSysDirWithDFS extends TestCase { +public class TestJobSysDirWithDFS { private static final Log LOG = LogFactory.getLog(TestJobSysDirWithDFS.class.getName()); @@ -115,7 +117,7 @@ public class TestJobSysDirWithDFS extends TestCase { // between Job Client & Job Tracker assertTrue(result.job.isSuccessful()); } - + @Test public void testWithDFS() throws IOException { MiniDFSCluster dfs = null; MiniMRCluster mr = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java index 27070783e14..bacc196008e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java @@ -20,7 +20,6 @@ package org.apache.hadoop.mapred; import java.io.*; import java.util.*; -import junit.framework.TestCase; import org.apache.commons.logging.*; import org.apache.hadoop.fs.*; @@ -28,8 +27,11 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.io.compress.*; import org.apache.hadoop.util.LineReader; import org.apache.hadoop.util.ReflectionUtils; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -public class TestKeyValueTextInputFormat extends TestCase { +public class TestKeyValueTextInputFormat { private static final Log LOG = LogFactory.getLog(TestKeyValueTextInputFormat.class.getName()); @@ -47,7 +49,7 @@ public class TestKeyValueTextInputFormat extends TestCase { private static Path workDir = new Path(new Path(System.getProperty("test.build.data", "."), "data"), "TestKeyValueTextInputFormat"); - + @Test public void testFormat() throws Exception { JobConf job = new JobConf(); Path file = new Path(workDir, "test.txt"); @@ -134,7 +136,7 @@ public class TestKeyValueTextInputFormat extends TestCase { (str.getBytes("UTF-8")), defaultConf); } - + @Test public void testUTF8() throws Exception { LineReader in = null; @@ -153,7 +155,7 @@ public class TestKeyValueTextInputFormat extends TestCase { } } } - + @Test public void testNewLines() throws Exception { LineReader in = null; try { @@ -219,7 +221,8 @@ public class TestKeyValueTextInputFormat extends TestCase { /** * Test using the gzip codec for reading */ - public static void testGzip() throws IOException { + @Test + public void testGzip() throws IOException { JobConf job = new JobConf(); CompressionCodec gzip = new GzipCodec(); ReflectionUtils.setConf(gzip, job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java index 7412832d5c2..dde9310607f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java @@ -35,14 +35,15 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.lib.LazyOutputFormat; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.assertTrue; /** * A JUnit test to test the Map-Reduce framework's feature to create part * files only if there is an explicit output.collect. This helps in preventing * 0 byte files */ -public class TestLazyOutput extends TestCase { +public class TestLazyOutput { private static final int NUM_HADOOP_SLAVES = 3; private static final int NUM_MAPS_PER_NODE = 2; private static final Path INPUT = new Path("/testlazy/input"); @@ -132,7 +133,7 @@ public class TestLazyOutput extends TestCase { } } - + @Test public void testLazyOutput() throws Exception { MiniDFSCluster dfs = null; MiniMRCluster mr = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java index fb9e8fcce3a..20d0173cc81 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java @@ -17,16 +17,6 @@ */ package org.apache.hadoop.mapred; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataOutputStream; @@ -36,9 +26,21 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.Text; +import org.junit.After; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @SuppressWarnings("deprecation") -public class TestMRCJCFileInputFormat extends TestCase { +public class TestMRCJCFileInputFormat { Configuration conf = new Configuration(); MiniDFSCluster dfs = null; @@ -50,6 +52,7 @@ public class TestMRCJCFileInputFormat extends TestCase { .build(); } + @Test public void testLocality() throws Exception { JobConf job = new JobConf(conf); dfs = newDFSCluster(job); @@ -109,6 +112,7 @@ public class TestMRCJCFileInputFormat extends TestCase { DFSTestUtil.waitReplication(fs, path, replication); } + @Test public void testNumInputs() throws Exception { JobConf job = new JobConf(conf); dfs = newDFSCluster(job); @@ -157,6 +161,7 @@ public class TestMRCJCFileInputFormat extends TestCase { } } + @Test public void testMultiLevelInput() throws Exception { JobConf job = new JobConf(conf); @@ -195,6 +200,7 @@ public class TestMRCJCFileInputFormat extends TestCase { } @SuppressWarnings("rawtypes") + @Test public void testLastInputSplitAtSplitBoundary() throws Exception { FileInputFormat fif = new FileInputFormatForTest(1024l * 1024 * 1024, 128l * 1024 * 1024); @@ -208,6 +214,7 @@ public class TestMRCJCFileInputFormat extends TestCase { } @SuppressWarnings("rawtypes") + @Test public void testLastInputSplitExceedingSplitBoundary() throws Exception { FileInputFormat fif = new FileInputFormatForTest(1027l * 1024 * 1024, 128l * 1024 * 1024); @@ -221,6 +228,7 @@ public class TestMRCJCFileInputFormat extends TestCase { } @SuppressWarnings("rawtypes") + @Test public void testLastInputSplitSingleSplit() throws Exception { FileInputFormat fif = new FileInputFormatForTest(100l * 1024 * 1024, 128l * 1024 * 1024); @@ -305,7 +313,7 @@ public class TestMRCJCFileInputFormat extends TestCase { DFSTestUtil.waitReplication(fileSys, name, replication); } - @Override + @After public void tearDown() throws Exception { if (dfs != null) { dfs.shutdown(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java index 3b86f81cc23..74b6d77f6a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java @@ -18,18 +18,25 @@ package org.apache.hadoop.mapred; -import java.io.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.JobStatus; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.mapred.JobContextImpl; -import org.apache.hadoop.mapred.TaskAttemptContextImpl; -import org.apache.hadoop.mapreduce.JobStatus; - -public class TestMRCJCFileOutputCommitter extends TestCase { +public class TestMRCJCFileOutputCommitter { private static Path outDir = new Path( System.getProperty("test.build.data", "/tmp"), "output"); @@ -67,6 +74,7 @@ public class TestMRCJCFileOutputCommitter extends TestCase { } @SuppressWarnings("unchecked") + @Test public void testCommitter() throws Exception { JobConf job = new JobConf(); setConfForFileOutputCommitter(job); @@ -108,6 +116,7 @@ public class TestMRCJCFileOutputCommitter extends TestCase { FileUtil.fullyDelete(new File(outDir.toString())); } + @Test public void testAbort() throws IOException { JobConf job = new JobConf(); setConfForFileOutputCommitter(job); @@ -161,6 +170,7 @@ public class TestMRCJCFileOutputCommitter extends TestCase { } } + @Test public void testFailAbort() throws IOException { JobConf job = new JobConf(); job.set(FileSystem.FS_DEFAULT_NAME_KEY, "faildel:///"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java index db6348ba440..b8ff016d6af 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java @@ -22,8 +22,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; @@ -40,6 +38,8 @@ import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; import org.apache.hadoop.mapreduce.split.JobSplitWriter; import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader; import org.apache.hadoop.util.ReflectionUtils; +import org.junit.Test; +import static org.junit.Assert.assertTrue; /** * Validates map phase progress. @@ -55,7 +55,7 @@ import org.apache.hadoop.util.ReflectionUtils; * once mapTask.run() is finished. Sort phase progress in map task is not * validated here. */ -public class TestMapProgress extends TestCase { +public class TestMapProgress { public static final Log LOG = LogFactory.getLog(TestMapProgress.class); private static String TEST_ROOT_DIR; static { @@ -234,7 +234,8 @@ public class TestMapProgress extends TestCase { /** * Validates map phase progress after each record is processed by map task * using custom task reporter. - */ + */ + @Test public void testMapProgress() throws Exception { JobConf job = new JobConf(); fs = FileSystem.getLocal(job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java index e19ff589fa4..a9e7f64c0b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java @@ -44,8 +44,8 @@ import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.Serializer; import org.apache.hadoop.mapred.Task.TaskReporter; - -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.assertEquals; @SuppressWarnings(value={"unchecked", "deprecation"}) /** @@ -56,7 +56,7 @@ import junit.framework.TestCase; * framework's merge on the reduce side will merge the partitions created to * generate the final output which is sorted on the key. */ -public class TestMerge extends TestCase { +public class TestMerge { private static final int NUM_HADOOP_DATA_NODES = 2; // Number of input files is same as the number of mappers. private static final int NUM_MAPPERS = 10; @@ -69,6 +69,7 @@ public class TestMerge extends TestCase { // Where output goes. private static final Path OUTPUT = new Path("/testplugin/output"); + @Test public void testMerge() throws Exception { MiniDFSCluster dfsCluster = null; MiniMRClientCluster mrCluster = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java index 8b7b8f51b96..b608d756a49 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java @@ -18,14 +18,16 @@ package org.apache.hadoop.mapred; +import org.junit.Test; + import java.io.IOException; -import junit.framework.TestCase; /** * A Unit-test to test bringup and shutdown of Mini Map-Reduce Cluster. */ -public class TestMiniMRBringup extends TestCase { +public class TestMiniMRBringup { + @Test public void testBringUp() throws IOException { MiniMRCluster mr = null; try { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java index 45879aff623..3f64f7a35b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java @@ -18,20 +18,23 @@ package org.apache.hadoop.mapred; -import java.io.*; -import junit.framework.TestCase; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.MRCaching.TestResult; import org.junit.Ignore; +import org.junit.Test; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; /** * A JUnit test to test caching with DFS * */ @Ignore -public class TestMiniMRDFSCaching extends TestCase { +public class TestMiniMRDFSCaching { + @Test public void testWithDFS() throws IOException { MiniMRCluster mr = null; MiniDFSCluster dfs = null; @@ -70,9 +73,4 @@ public class TestMiniMRDFSCaching extends TestCase { } } } - - public static void main(String[] argv) throws Exception { - TestMiniMRDFSCaching td = new TestMiniMRDFSCaching(); - td.testWithDFS(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java index 49825e99f57..1bd29542fcd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java @@ -21,17 +21,17 @@ import java.io.IOException; import java.util.BitSet; import java.util.HashMap; import java.util.Random; - -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -public class TestMultiFileInputFormat extends TestCase{ +public class TestMultiFileInputFormat { private static JobConf job = new JobConf(); @@ -79,7 +79,8 @@ public class TestMultiFileInputFormat extends TestCase{ FileInputFormat.setInputPaths(job, multiFileDir); return multiFileDir; } - + + @Test public void testFormat() throws IOException { LOG.info("Test started"); LOG.info("Max split count = " + MAX_SPLIT_COUNT); @@ -122,7 +123,8 @@ public class TestMultiFileInputFormat extends TestCase{ } LOG.info("Test Finished"); } - + + @Test public void testFormatWithLessPathsThanSplits() throws Exception { MultiFileInputFormat format = new DummyMultiFileInputFormat(); FileSystem fs = FileSystem.getLocal(job); @@ -135,9 +137,4 @@ public class TestMultiFileInputFormat extends TestCase{ initFiles(fs, 2, 500); assertEquals(2, format.getSplits(job, 4).length); } - - public static void main(String[] args) throws Exception{ - TestMultiFileInputFormat test = new TestMultiFileInputFormat(); - test.testFormat(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java index 16ff6af9271..5bb336e4e81 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java @@ -27,16 +27,19 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; -import junit.framework.TestCase; - import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * * test MultiFileSplit class */ -public class TestMultiFileSplit extends TestCase{ +public class TestMultiFileSplit { + @Test public void testReadWrite() throws Exception { MultiFileSplit split = new MultiFileSplit(new JobConf(), new Path[] {new Path("/test/path/1"), new Path("/test/path/2")}, new long[] {100,200}); @@ -70,6 +73,7 @@ public class TestMultiFileSplit extends TestCase{ * test method getLocations * @throws IOException */ + @Test public void testgetLocations() throws IOException{ JobConf job= new JobConf(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java index 294723a9c87..7e8dfef03f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.mapred; -import java.io.IOException; - -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,12 +28,17 @@ import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.mapreduce.JobCounter; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; /** * This test checks whether the task caches are created and used properly. */ @Ignore -public class TestMultipleLevelCaching extends TestCase { +public class TestMultipleLevelCaching { private static final int MAX_LEVEL = 5; final Path inDir = new Path("/cachetesting"); final Path outputPath = new Path("/output"); @@ -71,6 +72,7 @@ public class TestMultipleLevelCaching extends TestCase { return rack.toString(); } + @Test public void testMultiLevelCaching() throws Exception { for (int i = 1 ; i <= MAX_LEVEL; ++i) { testCachingAtLevel(i); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleTextOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleTextOutputFormat.java index 14c097d77e1..b5047fc8331 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleTextOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleTextOutputFormat.java @@ -18,15 +18,19 @@ package org.apache.hadoop.mapred; -import java.io.*; -import junit.framework.TestCase; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat; +import org.junit.Test; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import java.io.File; +import java.io.IOException; -import org.apache.hadoop.mapred.lib.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; -public class TestMultipleTextOutputFormat extends TestCase { +public class TestMultipleTextOutputFormat { private static JobConf defaultConf = new JobConf(); private static FileSystem localFs = null; @@ -83,7 +87,8 @@ public class TestMultipleTextOutputFormat extends TestCase { writeData(rw); rw.close(null); } - + + @Test public void testFormat() throws Exception { JobConf job = new JobConf(); job.set(JobContext.TASK_ATTEMPT_ID, attempt); @@ -145,8 +150,4 @@ public class TestMultipleTextOutputFormat extends TestCase { //System.out.printf("File_2 output: %s\n", output); assertEquals(output, expectedOutput.toString()); } - - public static void main(String[] args) throws Exception { - new TestMultipleTextOutputFormat().testFormat(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetch.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetch.java index 586df38dcfc..767459f88b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetch.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetch.java @@ -19,17 +19,18 @@ package org.apache.hadoop.mapred; import org.apache.hadoop.mapreduce.TaskCounter; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestReduceFetch extends TestReduceFetchFromPartialMem { - static { - setSuite(TestReduceFetch.class); - } - /** * Verify that all segments are read from disk * @throws Exception might be thrown */ + @Test public void testReduceFromDisk() throws Exception { final int MAP_TASKS = 8; JobConf job = mrCluster.createJobConf(); @@ -53,6 +54,7 @@ public class TestReduceFetch extends TestReduceFetchFromPartialMem { * Verify that no segment hits disk. * @throws Exception might be thrown */ + @Test public void testReduceFromMem() throws Exception { final int MAP_TASKS = 3; JobConf job = mrCluster.createJobConf(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java index 3a1a275ab91..9b04f64ac60 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java @@ -18,10 +18,6 @@ package org.apache.hadoop.mapred; -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -30,7 +26,9 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.mapreduce.TaskCounter; -import org.apache.hadoop.mapreduce.MRConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.DataInput; import java.io.DataOutput; @@ -39,34 +37,27 @@ import java.util.Arrays; import java.util.Formatter; import java.util.Iterator; -public class TestReduceFetchFromPartialMem extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestReduceFetchFromPartialMem { protected static MiniMRCluster mrCluster = null; protected static MiniDFSCluster dfsCluster = null; - protected static TestSuite mySuite; - protected static void setSuite(Class klass) { - mySuite = new TestSuite(klass); + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + mrCluster = new MiniMRCluster(2, + dfsCluster.getFileSystem().getUri().toString(), 1); } - static { - setSuite(TestReduceFetchFromPartialMem.class); - } - - public static Test suite() { - TestSetup setup = new TestSetup(mySuite) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - mrCluster = new MiniMRCluster(2, - dfsCluster.getFileSystem().getUri().toString(), 1); - } - protected void tearDown() throws Exception { - if (dfsCluster != null) { dfsCluster.shutdown(); } - if (mrCluster != null) { mrCluster.shutdown(); } - } - }; - return setup; + @After + public void tearDown() throws Exception { + if (dfsCluster != null) { dfsCluster.shutdown(); } + if (mrCluster != null) { mrCluster.shutdown(); } } private static final String tagfmt = "%04d"; @@ -78,6 +69,7 @@ public class TestReduceFetchFromPartialMem extends TestCase { } /** Verify that at least one segment does not hit disk */ + @Test public void testReduceFromPartialMem() throws Exception { final int MAP_TASKS = 7; JobConf job = mrCluster.createJobConf(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java index 43fd94871a2..69546a6cba2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.mapred; -import java.io.IOException; - -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -30,11 +26,17 @@ import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.util.Progressable; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * This test exercises the ValueIterator. */ -public class TestReduceTask extends TestCase { +public class TestReduceTask { static class NullProgress implements Progressable { public void progress() { } @@ -119,9 +121,10 @@ public class TestReduceTask extends TestCase { } assertEquals(vals.length, i); // make sure we have progress equal to 1.0 - assertEquals(1.0f, rawItr.getProgress().get()); + assertEquals(1.0f, rawItr.getProgress().get(),0.0000); } + @Test public void testValueIterator() throws Exception { Path tmpDir = new Path("build/test/test.reduce.task"); Configuration conf = new Configuration(); @@ -129,7 +132,8 @@ public class TestReduceTask extends TestCase { runValueIterator(tmpDir, testCase, conf, null); } } - + + @Test public void testValueIteratorWithCompression() throws Exception { Path tmpDir = new Path("build/test/test.reduce.task.compression"); Configuration conf = new Configuration(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java index b8be7400070..64b0983a5d6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java @@ -18,19 +18,26 @@ package org.apache.hadoop.mapred; +import org.apache.commons.logging.Log; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.junit.Test; + import java.io.IOException; import java.util.Random; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; -import junit.framework.TestCase; -import org.apache.commons.logging.*; - -public class TestSequenceFileAsBinaryInputFormat extends TestCase { +public class TestSequenceFileAsBinaryInputFormat { private static final Log LOG = FileInputFormat.LOG; private static final int RECORDS = 10000; + @Test public void testBinary() throws IOException { JobConf job = new JobConf(); FileSystem fs = FileSystem.getLocal(job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java index abe21f223ef..03dc6a69003 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java @@ -18,24 +18,35 @@ package org.apache.hadoop.mapred; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.junit.Test; + import java.io.IOException; import java.util.Random; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.io.SequenceFile.CompressionType; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import junit.framework.TestCase; -import org.apache.commons.logging.*; - -public class TestSequenceFileAsBinaryOutputFormat extends TestCase { +public class TestSequenceFileAsBinaryOutputFormat { private static final Log LOG = LogFactory.getLog(TestSequenceFileAsBinaryOutputFormat.class.getName()); - private static final int RECORDS = 10000; // A random task attempt id for testing. private static final String attempt = "attempt_200707121733_0001_m_000000_0"; + @Test public void testBinary() throws IOException { JobConf job = new JobConf(); FileSystem fs = FileSystem.getLocal(job); @@ -129,7 +140,8 @@ public class TestSequenceFileAsBinaryOutputFormat extends TestCase { assertEquals("Some records not found", RECORDS, count); } - public void testSequenceOutputClassDefaultsToMapRedOutputClass() + @Test + public void testSequenceOutputClassDefaultsToMapRedOutputClass() throws IOException { JobConf job = new JobConf(); FileSystem fs = FileSystem.getLocal(job); @@ -163,6 +175,7 @@ public class TestSequenceFileAsBinaryOutputFormat extends TestCase { job)); } + @Test public void testcheckOutputSpecsForbidRecordCompression() throws IOException { JobConf job = new JobConf(); FileSystem fs = FileSystem.getLocal(job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java index 4cfd59af745..d4e5e17e11f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java @@ -18,22 +18,29 @@ package org.apache.hadoop.mapred; -import java.io.*; -import java.util.*; -import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.junit.Test; -import org.apache.commons.logging.*; +import java.util.BitSet; +import java.util.Random; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.conf.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -public class TestSequenceFileAsTextInputFormat extends TestCase { +public class TestSequenceFileAsTextInputFormat { private static final Log LOG = FileInputFormat.LOG; private static int MAX_LENGTH = 10000; private static Configuration conf = new Configuration(); + @Test public void testFormat() throws Exception { JobConf job = new JobConf(conf); FileSystem fs = FileSystem.getLocal(conf); @@ -112,8 +119,4 @@ public class TestSequenceFileAsTextInputFormat extends TestCase { } } - - public static void main(String[] args) throws Exception { - new TestSequenceFileAsTextInputFormat().testFormat(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java index e50c396a434..93f21ce9e49 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java @@ -18,17 +18,21 @@ package org.apache.hadoop.mapred; -import java.io.*; -import java.util.*; -import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.junit.Test; -import org.apache.commons.logging.*; +import java.io.IOException; +import java.util.Random; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.conf.*; +import static org.junit.Assert.assertEquals; -public class TestSequenceFileInputFilter extends TestCase { +public class TestSequenceFileInputFilter { private static final Log LOG = FileInputFormat.LOG; private static final int MAX_LENGTH = 15000; @@ -97,7 +101,8 @@ public class TestSequenceFileInputFilter extends TestCase { } return count; } - + + @Test public void testRegexFilter() throws Exception { // set the filter class LOG.info("Testing Regex Filter with patter: \\A10*"); @@ -121,6 +126,7 @@ public class TestSequenceFileInputFilter extends TestCase { fs.delete(inDir, true); } + @Test public void testPercentFilter() throws Exception { LOG.info("Testing Percent Filter with frequency: 1000"); // set the filter class @@ -147,7 +153,8 @@ public class TestSequenceFileInputFilter extends TestCase { // clean up fs.delete(inDir, true); } - + + @Test public void testMD5Filter() throws Exception { // set the filter class LOG.info("Testing MD5 Filter with frequency: 1000"); @@ -168,9 +175,4 @@ public class TestSequenceFileInputFilter extends TestCase { // clean up fs.delete(inDir, true); } - - public static void main(String[] args) throws Exception { - TestSequenceFileInputFilter filter = new TestSequenceFileInputFilter(); - filter.testRegexFilter(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java index 575ed532545..ba4dce30974 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFormat.java @@ -18,22 +18,28 @@ package org.apache.hadoop.mapred; -import java.io.*; -import java.util.*; -import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.junit.Test; -import org.apache.commons.logging.*; +import java.util.BitSet; +import java.util.Random; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.conf.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -public class TestSequenceFileInputFormat extends TestCase { +public class TestSequenceFileInputFormat { private static final Log LOG = FileInputFormat.LOG; private static int MAX_LENGTH = 10000; private static Configuration conf = new Configuration(); + @Test public void testFormat() throws Exception { JobConf job = new JobConf(conf); FileSystem fs = FileSystem.getLocal(conf); @@ -110,7 +116,6 @@ public class TestSequenceFileInputFormat extends TestCase { } } - public static void main(String[] args) throws Exception { new TestSequenceFileInputFormat().testFormat(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java index ad4d4ce17a9..82d1d2d09a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java @@ -17,18 +17,20 @@ */ package org.apache.hadoop.mapred; -import java.util.Iterator; - -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.SortedRanges.Range; +import org.junit.Test; -public class TestSortedRanges extends TestCase { - private static final Log LOG = +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; + +public class TestSortedRanges { + private static final Log LOG = LogFactory.getLog(TestSortedRanges.class); - + + @Test public void testAdd() { SortedRanges sr = new SortedRanges(); sr.add(new Range(2,9)); @@ -66,7 +68,8 @@ public class TestSortedRanges extends TestCase { assertEquals(77, it.next().longValue()); } - + + @Test public void testRemove() { SortedRanges sr = new SortedRanges(); sr.add(new Range(2,19)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java index 426686f9bb5..b9e32759fa4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java @@ -18,12 +18,6 @@ package org.apache.hadoop.mapred; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.URI; - -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -34,14 +28,20 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.Progressable; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.URI; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * A JUnit test to test that jobs' output filenames are not HTML-encoded (cf HADOOP-1795). */ -public class TestSpecialCharactersInOutputPath extends TestCase { +public class TestSpecialCharactersInOutputPath { private static final Log LOG = LogFactory.getLog(TestSpecialCharactersInOutputPath.class.getName()); @@ -96,7 +96,8 @@ public class TestSpecialCharactersInOutputPath extends TestCase { LOG.info("job is complete: " + runningJob.isSuccessful()); return (runningJob.isSuccessful()); } - + + @Test public void testJobWithDFS() throws IOException { String namenode = null; MiniDFSCluster dfs = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java index 12568d09175..8a83e8153e3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java @@ -19,14 +19,18 @@ package org.apache.hadoop.mapred; import java.util.Map; -import junit.framework.TestCase; - import org.apache.hadoop.mapred.StatisticsCollector.TimeWindow; import org.apache.hadoop.mapred.StatisticsCollector.Stat; +import org.junit.Test; -public class TestStatisticsCollector extends TestCase{ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class TestStatisticsCollector { @SuppressWarnings("rawtypes") + @Test public void testMovingWindow() throws Exception { StatisticsCollector collector = new StatisticsCollector(1); TimeWindow window = new TimeWindow("test", 6, 2); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUserDefinedCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUserDefinedCounters.java index 3c2cf215fb3..2d67edc581a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUserDefinedCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUserDefinedCounters.java @@ -17,6 +17,15 @@ */ package org.apache.hadoop.mapred; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.lib.IdentityMapper; +import org.apache.hadoop.mapred.lib.IdentityReducer; +import org.junit.Test; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -26,18 +35,10 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; - -public class TestUserDefinedCounters extends TestCase { - +public class TestUserDefinedCounters { private static String TEST_ROOT_DIR = new File(System.getProperty("test.build.data", "/tmp")).toURI() .toString().replace(' ', '+') @@ -75,6 +76,7 @@ public class TestUserDefinedCounters extends TestCase { wr.close(); } + @Test public void testMapReduceJob() throws Exception { JobConf conf = new JobConf(TestUserDefinedCounters.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestWritableJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestWritableJobConf.java index 2c0cedcbb30..82c68db30c5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestWritableJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestWritableJobConf.java @@ -18,12 +18,6 @@ package org.apache.hadoop.mapred; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; @@ -31,8 +25,15 @@ import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.Serializer; import org.apache.hadoop.util.GenericsUtil; +import org.junit.Test; -public class TestWritableJobConf extends TestCase { +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +public class TestWritableJobConf { private static final Configuration CONF = new Configuration(); @@ -78,15 +79,17 @@ public class TestWritableJobConf extends TestCase { } } - assertEquals(map1, map2); + assertTrue(map1.equals(map2)); } + @Test public void testEmptyConfiguration() throws Exception { JobConf conf = new JobConf(); Configuration deser = serDeser(conf); assertEquals(conf, deser); } + @Test public void testNonEmptyConfiguration() throws Exception { JobConf conf = new JobConf(); conf.set("a", "A"); @@ -95,6 +98,7 @@ public class TestWritableJobConf extends TestCase { assertEquals(conf, deser); } + @Test public void testConfigurationWithDefaults() throws Exception { JobConf conf = new JobConf(false); conf.set("a", "A"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 0e340428214..abf2e72e0d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -18,6 +18,10 @@ package org.apache.hadoop.mapred; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -38,8 +42,6 @@ import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Map; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -113,7 +115,7 @@ import org.mockito.stubbing.Answer; * Test YarnRunner and make sure the client side plugin works * fine */ -public class TestYARNRunner extends TestCase { +public class TestYARNRunner { private static final Log LOG = LogFactory.getLog(TestYARNRunner.class); private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java index 15cea69dab2..a3066765ec0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java @@ -22,11 +22,6 @@ import java.io.DataOutput; import java.io.IOException; import java.util.Iterator; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.extensions.TestSetup; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -54,23 +49,27 @@ import org.apache.hadoop.mapred.Utils; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.util.ReflectionUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; -public class TestDatamerge extends TestCase { +public class TestDatamerge { private static MiniDFSCluster cluster = null; - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestDatamerge.class)) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - } - protected void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - }; - return setup; + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + } + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } } private static SequenceFile.Writer[] createWriters(Path testdir, @@ -246,18 +245,22 @@ public class TestDatamerge extends TestCase { base.getFileSystem(job).delete(base, true); } + @Test public void testSimpleInnerJoin() throws Exception { joinAs("inner", InnerJoinChecker.class); } + @Test public void testSimpleOuterJoin() throws Exception { joinAs("outer", OuterJoinChecker.class); } + @Test public void testSimpleOverride() throws Exception { joinAs("override", OverrideChecker.class); } + @Test public void testNestedJoin() throws Exception { // outer(inner(S1,...,Sn),outer(S1,...Sn)) final int SOURCES = 3; @@ -350,6 +353,7 @@ public class TestDatamerge extends TestCase { } + @Test public void testEmptyJoin() throws Exception { JobConf job = new JobConf(); Path base = cluster.getFileSystem().makeQualified(new Path("/empty")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestTupleWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestTupleWritable.java index e421ede9827..56871550dc9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestTupleWritable.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestTupleWritable.java @@ -26,8 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Random; -import junit.framework.TestCase; - import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -36,8 +34,12 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; -public class TestTupleWritable extends TestCase { +public class TestTupleWritable { private TupleWritable makeTuple(Writable[] writs) { Writable[] sub1 = { writs[1], writs[2] }; @@ -100,6 +102,7 @@ public class TestTupleWritable extends TestCase { return i; } + @Test public void testIterable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -121,6 +124,7 @@ public class TestTupleWritable extends TestCase { verifIter(writs, t, 0); } + @Test public void testNestedIterable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -139,6 +143,7 @@ public class TestTupleWritable extends TestCase { assertTrue("Bad count", writs.length == verifIter(writs, sTuple, 0)); } + @Test public void testWritable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -162,6 +167,7 @@ public class TestTupleWritable extends TestCase { assertTrue("Failed to write/read tuple", sTuple.equals(dTuple)); } + @Test public void testWideWritable() throws Exception { Writable[] manyWrits = makeRandomWritables(131); @@ -180,7 +186,8 @@ public class TestTupleWritable extends TestCase { assertTrue("Failed to write/read tuple", sTuple.equals(dTuple)); assertEquals("All tuple data has not been read from the stream",-1,in.read()); } - + + @Test public void testWideWritable2() throws Exception { Writable[] manyWrits = makeRandomWritables(71); @@ -202,6 +209,7 @@ public class TestTupleWritable extends TestCase { * Tests a tuple writable with more than 64 values and the values set written * spread far apart. */ + @Test public void testSparseWideWritable() throws Exception { Writable[] manyWrits = makeRandomWritables(131); @@ -220,7 +228,7 @@ public class TestTupleWritable extends TestCase { assertTrue("Failed to write/read tuple", sTuple.equals(dTuple)); assertEquals("All tuple data has not been read from the stream",-1,in.read()); } - + @Test public void testWideTuple() throws Exception { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; @@ -240,7 +248,7 @@ public class TestTupleWritable extends TestCase { } } } - + @Test public void testWideTuple2() throws Exception { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; @@ -264,6 +272,7 @@ public class TestTupleWritable extends TestCase { /** * Tests that we can write more than 64 values. */ + @Test public void testWideTupleBoundary() throws Exception { Text emptyText = new Text("Should not be set written"); Writable[] values = new Writable[65]; @@ -287,6 +296,7 @@ public class TestTupleWritable extends TestCase { /** * Tests compatibility with pre-0.21 versions of TupleWritable */ + @Test public void testPreVersion21Compatibility() throws Exception { Writable[] manyWrits = makeRandomWritables(64); PreVersion21TupleWritable oldTuple = new PreVersion21TupleWritable(manyWrits); @@ -304,7 +314,7 @@ public class TestTupleWritable extends TestCase { assertTrue("Tuple writable is unable to read pre-0.21 versions of TupleWritable", oldTuple.isCompatible(dTuple)); assertEquals("All tuple data has not been read from the stream",-1,in.read()); } - + @Test public void testPreVersion21CompatibilityEmptyTuple() throws Exception { Writable[] manyWrits = new Writable[0]; PreVersion21TupleWritable oldTuple = new PreVersion21TupleWritable(manyWrits); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java index 3ca175a5049..ae5572f5dcd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java @@ -21,8 +21,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; @@ -35,13 +33,16 @@ import org.apache.hadoop.mapred.JobConfigurable; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.ReflectionUtils; +import org.junit.Test; +import static org.junit.Assert.assertTrue; -public class TestWrappedRecordReaderClassloader extends TestCase { +public class TestWrappedRecordReaderClassloader { /** * Tests the class loader set by {@link JobConf#setClassLoader(ClassLoader)} * is inherited by any {@link WrappedRecordReader}s created by * {@link CompositeRecordReader} */ + @Test public void testClassLoader() throws Exception { JobConf job = new JobConf(); Fake_ClassLoader classLoader = new Fake_ClassLoader(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java index 8bd855433ea..b916026272e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java @@ -20,8 +20,6 @@ package org.apache.hadoop.mapred.lib; import java.io.DataOutputStream; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -32,9 +30,12 @@ import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TextInputFormat; +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; -public class TestDelegatingInputFormat extends TestCase { - +public class TestDelegatingInputFormat { + @Test public void testSplitting() throws Exception { JobConf conf = new JobConf(); MiniDFSCluster dfs = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestLineInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestLineInputFormat.java index db9c219e9c1..388de0fb88d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestLineInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestLineInputFormat.java @@ -20,13 +20,14 @@ package org.apache.hadoop.mapred.lib; import java.io.*; import java.util.*; -import junit.framework.TestCase; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.*; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -public class TestLineInputFormat extends TestCase { +public class TestLineInputFormat { private static int MAX_LENGTH = 200; private static JobConf defaultConf = new JobConf(); @@ -43,7 +44,7 @@ public class TestLineInputFormat extends TestCase { private static Path workDir = new Path(new Path(System.getProperty("test.build.data", "."), "data"), "TestLineInputFormat"); - + @Test public void testFormat() throws Exception { JobConf job = new JobConf(); Path file = new Path(workDir, "test.txt"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleInputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleInputs.java index 3a9cb9ec337..115a6f70d08 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleInputs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleInputs.java @@ -36,7 +36,6 @@ import static org.junit.Assert.assertEquals; * @see TestDelegatingInputFormat */ public class TestMultipleInputs { - @Test public void testAddInputPathWithFormat() { final JobConf conf = new JobConf(); @@ -49,7 +48,6 @@ public class TestMultipleInputs { assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar")) .getClass()); } - @Test public void testAddInputPathWithMapper() { final JobConf conf = new JobConf(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java index e2fdd429c56..b839a2c3afe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java @@ -22,13 +22,14 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.mapred.lib.*; import org.apache.hadoop.mapreduce.MapReduceTestUtil; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -import junit.framework.TestCase; import java.io.*; import java.util.*; import java.text.NumberFormat; -public class TestAggregates extends TestCase { +public class TestAggregates { private static NumberFormat idFormat = NumberFormat.getInstance(); static { @@ -36,7 +37,7 @@ public class TestAggregates extends TestCase { idFormat.setGroupingUsed(false); } - + @Test public void testAggregates() throws Exception { launch(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java index 968bb066565..203da4e0b7c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java @@ -19,13 +19,13 @@ package org.apache.hadoop.mapred.lib.db; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; -public class TestConstructQuery extends TestCase { - +public class TestConstructQuery { private String[] fieldNames = new String[] { "id", "name", "value" }; private String[] nullFieldNames = new String[] { null, null, null }; private String expected = "INSERT INTO hadoop_output (id,name,value) VALUES (?,?,?);"; @@ -33,15 +33,15 @@ public class TestConstructQuery extends TestCase { private DBOutputFormat format = new DBOutputFormat(); - - public void testConstructQuery() { + @Test + public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); - + actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); } - + @Test public void testSetOutput() throws IOException { JobConf job = new JobConf(); DBOutputFormat.setOutput(job, "hadoop_output", fieldNames); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java index dd7817d65b5..34b1d75dfed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java @@ -44,10 +44,13 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; import org.junit.Ignore; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; -import junit.framework.TestCase; @Ignore -public class TestPipes extends TestCase { +public class TestPipes { private static final Log LOG = LogFactory.getLog(TestPipes.class.getName()); @@ -66,7 +69,7 @@ public class TestPipes extends TestCase { fs.delete(p, true); assertFalse("output not cleaned up", fs.exists(p)); } - + @Test public void testPipes() throws IOException { if (System.getProperty("compile.c++") == null) { LOG.info("compile.c++ is not defined, so skipping TestPipes"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java index 29640c8854b..8177ecd405b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java @@ -17,36 +17,42 @@ */ package org.apache.hadoop.mapreduce; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.fs.*; import org.apache.hadoop.mapred.LocalJobRunner; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ReflectionUtils; - import org.junit.Test; -import junit.framework.TestCase; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * Stress tests for the LocalJobRunner */ -public class TestLocalRunner extends TestCase { +public class TestLocalRunner { private static final Log LOG = LogFactory.getLog(TestLocalRunner.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java index 6f45b5f5dc2..8fe9078e9e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java @@ -17,6 +17,23 @@ */ package org.apache.hadoop.mapreduce; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.mapred.ClusterMapReduceTestCase; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.mapreduce.tools.CLI; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; + import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -31,23 +48,11 @@ import java.io.PipedOutputStream; import java.io.PrintStream; import java.util.Arrays; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.RemoteIterator; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.junit.Assert; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.ClusterMapReduceTestCase; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hadoop.mapreduce.tools.CLI; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** test CLI class. CLI class implemented the Tool interface. @@ -103,7 +108,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { throw new IOException(); } } - + @Test public void testJobSubmissionSpecsAndFiles() throws Exception { Configuration conf = createJobConf(); Job job = MapReduceTestUtil.createJob(conf, getInputDir(), getOutputDir(), @@ -127,7 +132,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { /** * main test method */ - + @Test public void testJobClient() throws Exception { Configuration conf = createJobConf(); Job job = runJob(conf); @@ -180,8 +185,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out); String answer = new String(out.toByteArray(), "UTF-8"); - Assert - .assertTrue(answer.contains("Killed task " + taid + " by failing it")); + assertTrue(answer.contains("Killed task " + taid + " by failing it")); } /** @@ -199,7 +203,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out); String answer = new String(out.toByteArray(), "UTF-8"); - Assert.assertTrue(answer.contains("Killed task " + taid)); + assertTrue(answer.contains("Killed task " + taid)); } /** @@ -686,6 +690,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { * Test -list option displays job name. * The name is capped to 20 characters for display. */ + @Test public void testJobName() throws Exception { Configuration conf = createJobConf(); CLI jc = createJobClient(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java index 1e4f4de9f93..a69e06eacd9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java @@ -25,8 +25,6 @@ import java.io.Writer; import java.util.Arrays; import java.util.List; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -42,13 +40,16 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; /** * A JUnit test to test the Map-Reduce framework's feature to create part * files only if there is an explicit output.collect. This helps in preventing * 0 byte files */ -public class TestMapReduceLazyOutput extends TestCase { +public class TestMapReduceLazyOutput { private static final int NUM_HADOOP_SLAVES = 3; private static final int NUM_MAPS_PER_NODE = 2; private static final Path INPUT = new Path("/testlazy/input"); @@ -122,7 +123,7 @@ public class TestMapReduceLazyOutput extends TestCase { } } - + @Test public void testLazyOutput() throws Exception { MiniDFSCluster dfs = null; MiniMRCluster mr = null; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java index 5cf08991869..b757fb2c34f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java @@ -27,8 +27,6 @@ import java.io.Writer; import java.util.ArrayList; import java.util.StringTokenizer; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -43,12 +41,15 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; /** * A JUnit test to test the Map-Reduce framework's support for the * "mark-reset" functionality in Reduce Values Iterator */ -public class TestValueIterReset extends TestCase { +public class TestValueIterReset { private static final int NUM_MAPS = 1; private static final int NUM_TESTS = 4; private static final int NUM_VALUES = 40; @@ -518,6 +519,7 @@ public class TestValueIterReset extends TestCase { } } + @Test public void testValueIterReset() { try { Configuration conf = new Configuration(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java index 4d84fa9e108..308b7775a67 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,7 +27,6 @@ import static org.mockito.Mockito.doNothing; import java.io.IOException; import java.nio.ByteBuffer; -import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -44,8 +44,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Test; -public class TestYarnClientProtocolProvider extends TestCase { - +public class TestYarnClientProtocolProvider { private static final RecordFactory recordFactory = RecordFactoryProvider. getRecordFactory(null); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java index 788ad41ff96..3aac54e7159 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java @@ -18,22 +18,24 @@ package org.apache.hadoop.mapreduce.lib.aggregate; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.*; -import org.apache.hadoop.mapred.Utils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.junit.Test; -import junit.framework.TestCase; -import java.io.*; import java.text.NumberFormat; -public class TestMapReduceAggregates extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMapReduceAggregates { private static NumberFormat idFormat = NumberFormat.getInstance(); static { @@ -41,7 +43,7 @@ public class TestMapReduceAggregates extends TestCase { idFormat.setGroupingUsed(false); } - + @Test public void testAggregates() throws Exception { launch(); } @@ -123,11 +125,4 @@ public class TestMapReduceAggregates extends TestCase { fs.delete(OUTPUT_DIR, true); fs.delete(INPUT_DIR, true); } - - /** - * Launches all the tasks in order. - */ - public static void main(String[] argv) throws Exception { - launch(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java index bff25d20038..014855f7d6a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java @@ -19,14 +19,15 @@ package org.apache.hadoop.mapreduce.lib.db; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Job; +import org.junit.Test; -public class TestDBOutputFormat extends TestCase { - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestDBOutputFormat { private String[] fieldNames = new String[] { "id", "name", "value" }; private String[] nullFieldNames = new String[] { null, null, null }; private String expected = "INSERT INTO hadoop_output " + @@ -35,15 +36,17 @@ public class TestDBOutputFormat extends TestCase { private DBOutputFormat format = new DBOutputFormat(); - - public void testConstructQuery() { + + @Test + public void testConstructQuery() { String actual = format.constructQuery("hadoop_output", fieldNames); assertEquals(expected, actual); actual = format.constructQuery("hadoop_output", nullFieldNames); assertEquals(nullExpected, actual); } - + + @Test public void testSetOutput() throws IOException { Job job = Job.getInstance(new Configuration()); DBOutputFormat.setOutput(job, "hadoop_output", fieldNames); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestIntegerSplitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestIntegerSplitter.java index e50aba4f462..8b5d907dcdc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestIntegerSplitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestIntegerSplitter.java @@ -17,15 +17,15 @@ */ package org.apache.hadoop.mapreduce.lib.db; -import java.io.IOException; -import java.math.BigDecimal; +import org.junit.Test; + import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; -public class TestIntegerSplitter extends TestCase { +public class TestIntegerSplitter { private long [] toLongArray(List in) { long [] out = new long[in.size()]; for (int i = 0; i < in.size(); i++) { @@ -70,12 +70,14 @@ public class TestIntegerSplitter extends TestCase { } } + @Test public void testEvenSplits() throws SQLException { List splits = new IntegerSplitter().split(10, 0, 100); long [] expected = { 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; assertLongArrayEquals(expected, toLongArray(splits)); } + @Test public void testOddSplits() throws SQLException { List splits = new IntegerSplitter().split(10, 0, 95); long [] expected = { 0, 9, 18, 27, 36, 45, 54, 63, 72, 81, 90, 95 }; @@ -83,12 +85,14 @@ public class TestIntegerSplitter extends TestCase { } + @Test public void testSingletonSplit() throws SQLException { List splits = new IntegerSplitter().split(1, 5, 5); long [] expected = { 5, 5 }; assertLongArrayEquals(expected, toLongArray(splits)); } + @Test public void testSingletonSplit2() throws SQLException { // Same test, but overly-high numSplits List splits = new IntegerSplitter().split(5, 5, 5); @@ -96,6 +100,7 @@ public class TestIntegerSplitter extends TestCase { assertLongArrayEquals(expected, toLongArray(splits)); } + @Test public void testTooManySplits() throws SQLException { List splits = new IntegerSplitter().split(5, 3, 5); long [] expected = { 3, 4, 5 }; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestTextSplitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestTextSplitter.java index 045e3a1b1f6..e16f4234877 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestTextSplitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestTextSplitter.java @@ -17,15 +17,16 @@ */ package org.apache.hadoop.mapreduce.lib.db; -import java.io.IOException; +import org.junit.Test; + import java.math.BigDecimal; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; -public class TestTextSplitter extends TestCase { +public class TestTextSplitter { public String formatArray(Object [] ar) { StringBuilder sb = new StringBuilder(); @@ -62,48 +63,56 @@ public class TestTextSplitter extends TestCase { } } + @Test public void testStringConvertEmpty() { TextSplitter splitter = new TextSplitter(); BigDecimal emptyBigDec = splitter.stringToBigDecimal(""); assertEquals(BigDecimal.ZERO, emptyBigDec); } + @Test public void testBigDecConvertEmpty() { TextSplitter splitter = new TextSplitter(); String emptyStr = splitter.bigDecimalToString(BigDecimal.ZERO); assertEquals("", emptyStr); } + @Test public void testConvertA() { TextSplitter splitter = new TextSplitter(); String out = splitter.bigDecimalToString(splitter.stringToBigDecimal("A")); assertEquals("A", out); } + @Test public void testConvertZ() { TextSplitter splitter = new TextSplitter(); String out = splitter.bigDecimalToString(splitter.stringToBigDecimal("Z")); assertEquals("Z", out); } + @Test public void testConvertThreeChars() { TextSplitter splitter = new TextSplitter(); String out = splitter.bigDecimalToString(splitter.stringToBigDecimal("abc")); assertEquals("abc", out); } + @Test public void testConvertStr() { TextSplitter splitter = new TextSplitter(); String out = splitter.bigDecimalToString(splitter.stringToBigDecimal("big str")); assertEquals("big str", out); } + @Test public void testConvertChomped() { TextSplitter splitter = new TextSplitter(); String out = splitter.bigDecimalToString(splitter.stringToBigDecimal("AVeryLongStringIndeed")); assertEquals("AVeryLon", out); } + @Test public void testAlphabetSplit() throws SQLException { // This should give us 25 splits, one per letter. TextSplitter splitter = new TextSplitter(); @@ -113,6 +122,7 @@ public class TestTextSplitter extends TestCase { assertArrayEquals(expected, splits.toArray(new String [0])); } + @Test public void testCommonPrefix() throws SQLException { // Splits between 'Hand' and 'Hardy' TextSplitter splitter = new TextSplitter(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java index 91070f89c42..6f9183ab21b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java @@ -18,15 +18,19 @@ package org.apache.hadoop.mapreduce.lib.fieldsel; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MapReduceTestUtil; +import org.junit.Test; -import junit.framework.TestCase; import java.text.NumberFormat; -public class TestMRFieldSelection extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMRFieldSelection { private static NumberFormat idFormat = NumberFormat.getInstance(); static { @@ -34,6 +38,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); idFormat.setGroupingUsed(false); } + @Test public void testFieldSelection() throws Exception { launch(); } @@ -114,11 +119,4 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); System.out.println("ExpectedData:"); System.out.println(expectedOutput.toString()); } - - /** - * Launches all the tasks in order. - */ - public static void main(String[] argv) throws Exception { - launch(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsBinaryInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsBinaryInputFormat.java index f0b3d57486c..cbf9d183ef2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsBinaryInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsBinaryInputFormat.java @@ -18,11 +18,12 @@ package org.apache.hadoop.mapreduce.lib.input; -import java.io.IOException; -import java.util.Random; - -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; @@ -31,12 +32,18 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.task.MapContextImpl; +import org.junit.Test; -import junit.framework.TestCase; +import java.io.IOException; +import java.util.Random; -public class TestMRSequenceFileAsBinaryInputFormat extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMRSequenceFileAsBinaryInputFormat { private static final int RECORDS = 10000; + @Test public void testBinary() throws IOException, InterruptedException { Job job = Job.getInstance(); FileSystem fs = FileSystem.getLocal(job.getConfiguration()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsTextInputFormat.java index 2d03c2dd96a..335ce050d82 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileAsTextInputFormat.java @@ -18,11 +18,13 @@ package org.apache.hadoop.mapreduce.lib.input; -import java.util.*; -import junit.framework.TestCase; - -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; @@ -31,12 +33,19 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.task.MapContextImpl; -import org.apache.hadoop.conf.*; +import org.junit.Test; -public class TestMRSequenceFileAsTextInputFormat extends TestCase { +import java.util.BitSet; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class TestMRSequenceFileAsTextInputFormat { private static int MAX_LENGTH = 10000; private static Configuration conf = new Configuration(); + @Test public void testFormat() throws Exception { Job job = Job.getInstance(conf); FileSystem fs = FileSystem.getLocal(conf); @@ -112,8 +121,4 @@ public class TestMRSequenceFileAsTextInputFormat extends TestCase { } } - - public static void main(String[] args) throws Exception { - new TestMRSequenceFileAsTextInputFormat().testFormat(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java index edf7e1ad10d..89aa7b23057 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java @@ -18,14 +18,14 @@ package org.apache.hadoop.mapreduce.lib.input; -import java.io.*; -import java.util.*; -import junit.framework.TestCase; - -import org.apache.commons.logging.*; - -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; @@ -34,10 +34,15 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.task.MapContextImpl; -import org.apache.hadoop.conf.*; +import org.junit.Test; -public class TestMRSequenceFileInputFilter extends TestCase { - private static final Log LOG = +import java.io.IOException; +import java.util.Random; + +import static org.junit.Assert.assertEquals; + +public class TestMRSequenceFileInputFilter { + private static final Log LOG = LogFactory.getLog(TestMRSequenceFileInputFilter.class.getName()); private static final int MAX_LENGTH = 15000; @@ -113,7 +118,8 @@ public class TestMRSequenceFileInputFilter extends TestCase { } return count; } - + + @Test public void testRegexFilter() throws Exception { // set the filter class LOG.info("Testing Regex Filter with patter: \\A10*"); @@ -138,6 +144,7 @@ public class TestMRSequenceFileInputFilter extends TestCase { fs.delete(inDir, true); } + @Test public void testPercentFilter() throws Exception { LOG.info("Testing Percent Filter with frequency: 1000"); // set the filter class @@ -165,7 +172,8 @@ public class TestMRSequenceFileInputFilter extends TestCase { // clean up fs.delete(inDir, true); } - + + @Test public void testMD5Filter() throws Exception { // set the filter class LOG.info("Testing MD5 Filter with frequency: 1000"); @@ -187,9 +195,4 @@ public class TestMRSequenceFileInputFilter extends TestCase { // clean up fs.delete(inDir, true); } - - public static void main(String[] args) throws Exception { - TestMRSequenceFileInputFilter filter = new TestMRSequenceFileInputFilter(); - filter.testRegexFilter(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java index 7b3878d9475..477866f4e35 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java @@ -18,17 +18,28 @@ package org.apache.hadoop.mapreduce.lib.input; -import java.io.*; -import java.util.*; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MapContext; +import org.apache.hadoop.mapreduce.MapReduceTestUtil; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.task.MapContextImpl; +import org.junit.Test; -public class TestNLineInputFormat extends TestCase { +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestNLineInputFormat { private static int MAX_LENGTH = 200; private static Configuration conf = new Configuration(); @@ -45,7 +56,8 @@ public class TestNLineInputFormat extends TestCase { private static Path workDir = new Path(new Path(System.getProperty("test.build.data", "."), "data"), "TestNLineInputFormat"); - + + @Test public void testFormat() throws Exception { Job job = Job.getInstance(conf); Path file = new Path(workDir, "test.txt"); @@ -116,8 +128,4 @@ public class TestNLineInputFormat extends TestCase { } } } - - public static void main(String[] args) throws Exception { - new TestNLineInputFormat().testFormat(); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java index d245bfd6cde..1173ea4fa47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java @@ -19,11 +19,6 @@ package org.apache.hadoop.mapreduce.lib.join; import java.io.IOException; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.extensions.TestSetup; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,23 +32,31 @@ import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; -public class TestJoinDatamerge extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestJoinDatamerge { private static MiniDFSCluster cluster = null; - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestJoinDatamerge.class)) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - } - protected void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - }; - return setup; + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + } + + @AfterClass + public static void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } } private static SequenceFile.Writer[] createWriters(Path testdir, @@ -111,7 +114,7 @@ public class TestJoinDatamerge extends TestCase { extends Mapper{ protected final static IntWritable one = new IntWritable(1); int srcs; - + public void setup(Context context) { srcs = context.getConfiguration().getInt("testdatamerge.sources", 0); assertTrue("Invalid src count: " + srcs, srcs > 0); @@ -123,7 +126,7 @@ public class TestJoinDatamerge extends TestCase { protected final static IntWritable one = new IntWritable(1); int srcs; - + public void setup(Context context) { srcs = context.getConfiguration().getInt("testdatamerge.sources", 0); assertTrue("Invalid src count: " + srcs, srcs > 0); @@ -272,10 +275,12 @@ public class TestJoinDatamerge extends TestCase { base.getFileSystem(conf).delete(base, true); } + @Test public void testSimpleInnerJoin() throws Exception { joinAs("inner", InnerJoinMapChecker.class, InnerJoinReduceChecker.class); } + @Test public void testSimpleOuterJoin() throws Exception { joinAs("outer", OuterJoinMapChecker.class, OuterJoinReduceChecker.class); } @@ -322,11 +327,13 @@ public class TestJoinDatamerge extends TestCase { } return product; } - + + @Test public void testSimpleOverride() throws Exception { joinAs("override", OverrideMapChecker.class, OverrideReduceChecker.class); } + @Test public void testNestedJoin() throws Exception { // outer(inner(S1,...,Sn),outer(S1,...Sn)) final int SOURCES = 3; @@ -422,6 +429,7 @@ public class TestJoinDatamerge extends TestCase { } + @Test public void testEmptyJoin() throws Exception { Configuration conf = new Configuration(); Path base = cluster.getFileSystem().makeQualified(new Path("/empty")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java index 151bc875ad3..b6e76069d95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java @@ -20,11 +20,6 @@ package org.apache.hadoop.mapreduce.lib.join; import java.io.IOException; import java.util.List; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.extensions.TestSetup; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -36,8 +31,14 @@ import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.task.MapContextImpl; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; -public class TestJoinProperties extends TestCase { +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestJoinProperties { private static MiniDFSCluster cluster = null; final static int SOURCES = 3; @@ -46,21 +47,19 @@ public class TestJoinProperties extends TestCase { static Path[] src; static Path base; - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestJoinProperties.class)) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - base = cluster.getFileSystem().makeQualified(new Path("/nested")); - src = generateSources(conf); - } - protected void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - }; - return setup; + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + base = cluster.getFileSystem().makeQualified(new Path("/nested")); + src = generateSources(conf); + } + + @AfterClass + public static void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } } // Sources from 0 to srcs-2 have IntWritable key and IntWritable value @@ -233,6 +232,7 @@ public class TestJoinProperties extends TestCase { } // outer(outer(A, B), C) == outer(A,outer(B, C)) == outer(A, B, C) + @Test public void testOuterAssociativity() throws Exception { Configuration conf = new Configuration(); testExpr1(conf, "outer", TestType.OUTER_ASSOCIATIVITY, 33); @@ -241,6 +241,7 @@ public class TestJoinProperties extends TestCase { } // inner(inner(A, B), C) == inner(A,inner(B, C)) == inner(A, B, C) + @Test public void testInnerAssociativity() throws Exception { Configuration conf = new Configuration(); testExpr1(conf, "inner", TestType.INNER_ASSOCIATIVITY, 2); @@ -249,6 +250,7 @@ public class TestJoinProperties extends TestCase { } // override(inner(A, B), A) == A + @Test public void testIdentity() throws Exception { Configuration conf = new Configuration(); testExpr4(conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinTupleWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinTupleWritable.java index d35941fc884..093da266b95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinTupleWritable.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinTupleWritable.java @@ -24,8 +24,6 @@ import java.io.DataOutputStream; import java.util.Arrays; import java.util.Random; -import junit.framework.TestCase; - import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -33,8 +31,13 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.junit.Test; -public class TestJoinTupleWritable extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestJoinTupleWritable { private TupleWritable makeTuple(Writable[] writs) { Writable[] sub1 = { writs[1], writs[2] }; @@ -97,6 +100,7 @@ public class TestJoinTupleWritable extends TestCase { return i; } + @Test public void testIterable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -118,6 +122,7 @@ public class TestJoinTupleWritable extends TestCase { verifIter(writs, t, 0); } + @Test public void testNestedIterable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -136,6 +141,7 @@ public class TestJoinTupleWritable extends TestCase { assertTrue("Bad count", writs.length == verifIter(writs, sTuple, 0)); } + @Test public void testWritable() throws Exception { Random r = new Random(); Writable[] writs = { @@ -159,6 +165,7 @@ public class TestJoinTupleWritable extends TestCase { assertTrue("Failed to write/read tuple", sTuple.equals(dTuple)); } + @Test public void testWideWritable() throws Exception { Writable[] manyWrits = makeRandomWritables(131); @@ -178,7 +185,8 @@ public class TestJoinTupleWritable extends TestCase { assertEquals("All tuple data has not been read from the stream", -1, in.read()); } - + + @Test public void testWideWritable2() throws Exception { Writable[] manyWrits = makeRandomWritables(71); @@ -201,6 +209,7 @@ public class TestJoinTupleWritable extends TestCase { * Tests a tuple writable with more than 64 values and the values set written * spread far apart. */ + @Test public void testSparseWideWritable() throws Exception { Writable[] manyWrits = makeRandomWritables(131); @@ -220,7 +229,8 @@ public class TestJoinTupleWritable extends TestCase { assertEquals("All tuple data has not been read from the stream", -1, in.read()); } - + + @Test public void testWideTuple() throws Exception { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; @@ -241,7 +251,8 @@ public class TestJoinTupleWritable extends TestCase { } } } - + + @Test public void testWideTuple2() throws Exception { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; @@ -266,6 +277,7 @@ public class TestJoinTupleWritable extends TestCase { /** * Tests that we can write more than 64 values. */ + @Test public void testWideTupleBoundary() throws Exception { Text emptyText = new Text("Should not be set written"); Writable[] values = new Writable[65]; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java index 36cf1872ad4..680e246b4e3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java @@ -17,23 +17,32 @@ */ package org.apache.hadoop.mapreduce.lib.join; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.apache.hadoop.mapreduce.MapReduceTestUtil.Fake_RR; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.junit.Test; -public class TestWrappedRRClassloader extends TestCase { +import static org.junit.Assert.assertTrue; + +public class TestWrappedRRClassloader { /** * Tests the class loader set by * {@link Configuration#setClassLoader(ClassLoader)} * is inherited by any {@link WrappedRecordReader}s created by * {@link CompositeRecordReader} */ + @Test public void testClassLoader() throws Exception { Configuration conf = new Configuration(); Fake_ClassLoader classLoader = new Fake_ClassLoader(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java index 2e40f72fdd2..5a8aeda83be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java @@ -18,12 +18,17 @@ package org.apache.hadoop.mapreduce.lib.output; -import java.io.IOException; -import java.util.Random; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.InputFormat; @@ -38,16 +43,22 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.task.MapContextImpl; +import org.junit.Test; -import junit.framework.TestCase; -import org.apache.commons.logging.*; +import java.io.IOException; +import java.util.Random; -public class TestMRSequenceFileAsBinaryOutputFormat extends TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestMRSequenceFileAsBinaryOutputFormat { private static final Log LOG = LogFactory.getLog(TestMRSequenceFileAsBinaryOutputFormat.class.getName()); private static final int RECORDS = 10000; - + + @Test public void testBinary() throws IOException, InterruptedException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf); @@ -144,7 +155,8 @@ public class TestMRSequenceFileAsBinaryOutputFormat extends TestCase { assertEquals("Some records not found", RECORDS, count); } - public void testSequenceOutputClassDefaultsToMapRedOutputClass() + @Test + public void testSequenceOutputClassDefaultsToMapRedOutputClass() throws IOException { Job job = Job.getInstance(); // Setting Random class to test getSequenceFileOutput{Key,Value}Class @@ -172,7 +184,8 @@ public class TestMRSequenceFileAsBinaryOutputFormat extends TestCase { SequenceFileAsBinaryOutputFormat.getSequenceFileOutputValueClass(job)); } - public void testcheckOutputSpecsForbidRecordCompression() + @Test + public void testcheckOutputSpecsForbidRecordCompression() throws IOException { Job job = Job.getInstance(); FileSystem fs = FileSystem.getLocal(job.getConfiguration()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestBinaryPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestBinaryPartitioner.java index 7be538ecf41..f83bc11a216 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestBinaryPartitioner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestBinaryPartitioner.java @@ -22,11 +22,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.util.ReflectionUtils; +import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; -public class TestBinaryPartitioner extends TestCase { +public class TestBinaryPartitioner { + @Test public void testDefaultOffsets() { Configuration conf = new Configuration(); BinaryPartitioner partitioner = @@ -50,7 +53,8 @@ public class TestBinaryPartitioner extends TestCase { partition2 = partitioner.getPartition(key2, null, 10); assertTrue(partition1 != partition2); } - + + @Test public void testCustomOffsets() { Configuration conf = new Configuration(); BinaryComparable key1 = new BytesWritable(new byte[] { 1, 2, 3, 4, 5 }); @@ -75,7 +79,8 @@ public class TestBinaryPartitioner extends TestCase { partition2 = partitioner.getPartition(key2, null, 10); assertEquals(partition1, partition2); } - + + @Test public void testLowerBound() { Configuration conf = new Configuration(); BinaryPartitioner.setLeftOffset(conf, 0); @@ -87,7 +92,8 @@ public class TestBinaryPartitioner extends TestCase { int partition2 = partitioner.getPartition(key2, null, 10); assertTrue(partition1 != partition2); } - + + @Test public void testUpperBound() { Configuration conf = new Configuration(); BinaryPartitioner.setRightOffset(conf, 4); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java index 6bad846f6d3..4d05d13d445 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java @@ -19,14 +19,17 @@ package org.apache.hadoop.mapreduce.lib.partition; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; -public class TestKeyFieldHelper extends TestCase { +public class TestKeyFieldHelper { private static final Log LOG = LogFactory.getLog(TestKeyFieldHelper.class); /** * Test is key-field-helper's parse option. */ + @Test public void testparseOption() throws Exception { KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); @@ -212,6 +215,7 @@ public class TestKeyFieldHelper extends TestCase { /** * Test is key-field-helper's getWordLengths. */ + @Test public void testGetWordLengths() throws Exception { KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); @@ -270,6 +274,7 @@ public class TestKeyFieldHelper extends TestCase { /** * Test is key-field-helper's getStartOffset/getEndOffset. */ + @Test public void testgetStartEndOffset() throws Exception { KeyFieldHelper helper = new KeyFieldHelper(); helper.setKeyFieldSeparator("\t"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedPartitioner.java index 9c2fb48d9bf..00b415f32cb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedPartitioner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedPartitioner.java @@ -19,14 +19,16 @@ package org.apache.hadoop.mapreduce.lib.partition; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; +import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; -public class TestMRKeyFieldBasedPartitioner extends TestCase { +public class TestMRKeyFieldBasedPartitioner { /** * Test is key-field-based partitioned works with empty key. */ + @Test public void testEmptyKey() throws Exception { int numReducers = 10; KeyFieldBasedPartitioner kfbp = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java index a844737e09d..bdb4ff4794e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; @@ -41,8 +39,11 @@ import org.apache.hadoop.io.serializer.JavaSerializationComparator; import org.apache.hadoop.io.serializer.Serialization; import org.apache.hadoop.io.serializer.WritableSerialization; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; -public class TestTotalOrderPartitioner extends TestCase { +import static org.junit.Assert.assertEquals; + +public class TestTotalOrderPartitioner { private static final Text[] splitStrings = new Text[] { // -inf // 0 @@ -140,6 +141,7 @@ public class TestTotalOrderPartitioner extends TestCase { return p; } + @Test public void testTotalOrderWithCustomSerialization() throws Exception { TotalOrderPartitioner partitioner = new TotalOrderPartitioner(); @@ -165,6 +167,7 @@ public class TestTotalOrderPartitioner extends TestCase { } } + @Test public void testTotalOrderMemCmp() throws Exception { TotalOrderPartitioner partitioner = new TotalOrderPartitioner(); @@ -184,6 +187,7 @@ public class TestTotalOrderPartitioner extends TestCase { } } + @Test public void testTotalOrderBinarySearch() throws Exception { TotalOrderPartitioner partitioner = new TotalOrderPartitioner(); @@ -216,6 +220,7 @@ public class TestTotalOrderPartitioner extends TestCase { } } + @Test public void testTotalOrderCustomComparator() throws Exception { TotalOrderPartitioner partitioner = new TotalOrderPartitioner(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java index e1849a3ce9c..07b5d8b9f50 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java @@ -20,8 +20,6 @@ package org.apache.hadoop.mapreduce.util; import java.io.File; import java.io.IOException; -import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,20 +28,27 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * A test for MRAsyncDiskService. */ -public class TestMRAsyncDiskService extends TestCase { +public class TestMRAsyncDiskService { public static final Log LOG = LogFactory.getLog(TestMRAsyncDiskService.class); private static String TEST_ROOT_DIR = new Path(System.getProperty( "test.build.data", "/tmp")).toString(); - @Override - protected void setUp() { + @Before + public void setUp() { FileUtil.fullyDelete(new File(TEST_ROOT_DIR)); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java index aa769f85974..f68cc8310a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.mapreduce.v2; -import junit.framework.TestCase; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -29,22 +28,25 @@ import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; -import java.net.InetAddress; -import java.io.File; -import java.io.FileOutputStream; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.net.InetAddress; import java.security.PrivilegedExceptionAction; -public class TestMiniMRProxyUser extends TestCase { +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestMiniMRProxyUser { private MiniDFSCluster dfsCluster = null; private MiniMRCluster mrCluster = null; - - protected void setUp() throws Exception { - super.setUp(); + + @Before + public void setUp() throws Exception { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "/tmp"); } @@ -91,15 +93,14 @@ public class TestMiniMRProxyUser extends TestCase { return mrCluster.createJobConf(); } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { if (mrCluster != null) { mrCluster.shutdown(); } if (dfsCluster != null) { dfsCluster.shutdown(); } - super.tearDown(); } private void mrRun() throws Exception { @@ -125,11 +126,13 @@ public class TestMiniMRProxyUser extends TestCase { assertTrue(runJob.isComplete()); assertTrue(runJob.isSuccessful()); } - + + @Test public void __testCurrentUser() throws Exception { mrRun(); } + @Test public void testValidProxyUser() throws Exception { UserGroupInformation ugi = UserGroupInformation.createProxyUser("u1", UserGroupInformation.getLoginUser()); ugi.doAs(new PrivilegedExceptionAction() { @@ -142,6 +145,7 @@ public class TestMiniMRProxyUser extends TestCase { }); } + @Test public void ___testInvalidProxyUser() throws Exception { UserGroupInformation ugi = UserGroupInformation.createProxyUser("u2", UserGroupInformation.getLoginUser()); ugi.doAs(new PrivilegedExceptionAction() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java index b6947f3fc48..e90c509d7a8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.mapreduce.v2; -import junit.framework.TestCase; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -28,17 +27,22 @@ import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.net.InetAddress; -public class TestNonExistentJob extends TestCase { +import static org.junit.Assert.assertNull; + +public class TestNonExistentJob { private MiniDFSCluster dfsCluster = null; private MiniMRCluster mrCluster = null; - protected void setUp() throws Exception { - super.setUp(); + @Before + public void setUp() throws Exception { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "/tmp"); } @@ -78,17 +82,17 @@ public class TestNonExistentJob extends TestCase { return mrCluster.createJobConf(); } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { if (mrCluster != null) { mrCluster.shutdown(); } if (dfsCluster != null) { dfsCluster.shutdown(); } - super.tearDown(); } + @Test public void testGetInvalidJob() throws Exception { RunningJob runJob = new JobClient(getJobConf()).getJob(JobID.forName("job_0_0")); assertNull(runJob); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java index 7b7901faad1..860fb89cfcf 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingBadRecords.java @@ -42,6 +42,11 @@ import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.SkipBadRecords; import org.apache.hadoop.mapred.Utils; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestStreamingBadRecords extends ClusterMapReduceTestCase { @@ -68,7 +73,8 @@ public class TestStreamingBadRecords extends ClusterMapReduceTestCase utilTest.redirectIfAntJunit(); } - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { Properties props = new Properties(); props.setProperty(JTConfig.JT_RETIREJOBS, "false"); props.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "false"); @@ -242,6 +248,7 @@ public class TestStreamingBadRecords extends ClusterMapReduceTestCase } */ + @Test public void testNoOp() { // Added to avoid warnings when running this disabled test } From f2aec4eb824647a01e14b4eede03af0babe65fb6 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 29 Mar 2016 11:54:42 -0700 Subject: [PATCH 09/75] HDFS-10197. TestFsDatasetCache failing intermittently due to timeout. Contributed by Lin Yiqun. --- .../src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +- .../hadoop/hdfs/server/datanode/TestFsDatasetCache.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 0b08996a777..8a52bbb1e08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1464,7 +1464,7 @@ public class DFSTestUtil { NativeIO.POSIX.getCacheManipulator().getMemlockLimit()); return true; } - }, 100, 60000); + }, 100, 120000); return expectedCacheUsed; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java index 77b0c447897..d82e383d8c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java @@ -88,6 +88,8 @@ import org.apache.log4j.LogManager; import com.google.common.base.Supplier; import com.google.common.primitives.Ints; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY; + public class TestFsDatasetCache { private static final Log LOG = LogFactory.getLog(TestFsDatasetCache.class); @@ -124,6 +126,7 @@ public class TestFsDatasetCache { conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, CACHE_CAPACITY); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.setInt(DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY, 10); prevCacheManipulator = NativeIO.POSIX.getCacheManipulator(); NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); @@ -454,7 +457,7 @@ public class TestFsDatasetCache { }, 100, 10000); } - @Test(timeout=60000) + @Test(timeout=600000) public void testPageRounder() throws Exception { // Write a small file Path fileName = new Path("/testPageRounder"); From 46a5245db95f2aad199100d2886381398070124f Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 29 Mar 2016 12:08:46 -0700 Subject: [PATCH 10/75] HDFS-9478. Reason for failing ipc.FairCallQueue contruction should be thrown. (Contributed by Ajith S) --- .../apache/hadoop/ipc/CallQueueManager.java | 10 ++++++++ .../hadoop/ipc/TestCallQueueManager.java | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java index c10f839db4f..2ee15d3809e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ipc; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -70,6 +71,9 @@ public class CallQueueManager { return ctor.newInstance(maxLen, ns, conf); } catch (RuntimeException e) { throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); } catch (Exception e) { } @@ -79,6 +83,9 @@ public class CallQueueManager { return ctor.newInstance(maxLen); } catch (RuntimeException e) { throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); } catch (Exception e) { } @@ -88,6 +95,9 @@ public class CallQueueManager { return ctor.newInstance(); } catch (RuntimeException e) { throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); } catch (Exception e) { } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java index 51a97506fac..4d659acd46d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java @@ -19,6 +19,8 @@ package org.apache.hadoop.ipc; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.HashMap; @@ -219,4 +221,27 @@ public class TestCallQueueManager { assertEquals(totalCallsConsumed, totalCallsCreated); } + + public static class ExceptionFakeCall { + + public ExceptionFakeCall() { + throw new IllegalArgumentException("Exception caused by constructor.!!"); + } + } + + private static final Class> exceptionQueueClass = CallQueueManager + .convertQueueClass(ExceptionFakeCall.class, ExceptionFakeCall.class); + + @Test + public void testInvocationException() throws InterruptedException { + try { + new CallQueueManager(exceptionQueueClass, false, 10, + "", null); + fail(); + } catch (RuntimeException re) { + assertTrue(re.getCause() instanceof IllegalArgumentException); + assertEquals("Exception caused by constructor.!!", re.getCause() + .getMessage()); + } + } } \ No newline at end of file From c9307e48b71d2e293be1b5517d12be959222c5fe Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 29 Mar 2016 13:50:08 -0700 Subject: [PATCH 11/75] HDFS-10228. TestHDFSCLI fails. (Contributed by Akira AJISAKA) --- .../hadoop-hdfs/src/test/resources/testHDFSConf.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index ef9e29ae849..086bbe9fa33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -1861,7 +1861,7 @@ RegexpComparator - ^mv: `file2': No such file or directory + ^mv: `file2': No such file or directory: `NAMENODE/user/USERNAME/file2' @@ -2618,7 +2618,7 @@ RegexpComparator - ^cp: `file2': No such file or directory + ^cp: `file2': No such file or directory: `NAMENODE/user/USERNAME/file2' @@ -6555,7 +6555,7 @@ RegexpComparator - touchz: `file0': No such file or directory + touchz: `file0': No such file or directory: `NAMENODE/user/USERNAME/file0' From ddfe6774c21c8ccf5582a05bb0b58e961bbec309 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 29 Mar 2016 13:55:00 -0700 Subject: [PATCH 12/75] HDFS-9439. Support reconfiguring fs.protected.directories without NN restart. (Contributed by Xiaobing Zhou) --- .../hdfs/server/namenode/FSDirectory.java | 48 +++++++++++++++++-- .../hadoop/hdfs/server/namenode/NameNode.java | 9 +++- .../namenode/TestProtectedDirectories.java | 44 +++++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 51b7747c1f3..4930c1d1646 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import org.apache.hadoop.util.StringUtils; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; @@ -146,7 +149,7 @@ public class FSDirectory implements Closeable { // be deleted unless they are empty. // // Each entry in this set must be a normalized path. - private final SortedSet protectedDirectories; + private volatile SortedSet protectedDirectories; // lock to protect the directory and BlockMap private final ReentrantReadWriteLock dirLock; @@ -370,16 +373,53 @@ public class FSDirectory implements Closeable { */ @VisibleForTesting static SortedSet parseProtectedDirectories(Configuration conf) { + return parseProtectedDirectories(conf + .getTrimmedStringCollection(FS_PROTECTED_DIRECTORIES)); + } + + /** + * Parse configuration setting dfs.namenode.protected.directories to retrieve + * the set of protected directories. + * + * @param protectedDirsString + * a comma separated String representing a bunch of paths. + * @return a TreeSet + */ + @VisibleForTesting + static SortedSet parseProtectedDirectories( + final String protectedDirsString) { + return parseProtectedDirectories(StringUtils + .getTrimmedStringCollection(protectedDirsString)); + } + + private static SortedSet parseProtectedDirectories( + final Collection protectedDirs) { // Normalize each input path to guard against administrator error. - return new TreeSet<>(normalizePaths( - conf.getTrimmedStringCollection(FS_PROTECTED_DIRECTORIES), - FS_PROTECTED_DIRECTORIES)); + return new TreeSet<>( + normalizePaths(protectedDirs, FS_PROTECTED_DIRECTORIES)); } SortedSet getProtectedDirectories() { return protectedDirectories; } + /** + * Set directories that cannot be removed unless empty, even by an + * administrator. + * + * @param protectedDirsString + * comma separated list of protected directories + */ + String setProtectedDirectories(String protectedDirsString) { + if (protectedDirsString == null) { + protectedDirectories = new TreeSet<>(); + } else { + protectedDirectories = parseProtectedDirectories(protectedDirsString); + } + + return Joiner.on(",").skipNulls().join(protectedDirectories); + } + BlockManager getBlockManager() { return getFSNamesystem().getBlockManager(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 148626b2514..1301940734d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -150,6 +150,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.FS_PROTECTED_DIRECTORIES; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; @@ -273,8 +274,10 @@ public class NameNode extends ReconfigurableBase implements /** A list of property that are reconfigurable at runtime. */ static final List RECONFIGURABLE_PROPERTIES = Collections - .unmodifiableList(Arrays.asList(DFS_HEARTBEAT_INTERVAL_KEY, - DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY)); + .unmodifiableList(Arrays + .asList(DFS_HEARTBEAT_INTERVAL_KEY, + DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + FS_PROTECTED_DIRECTORIES)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -2004,6 +2007,8 @@ public class NameNode extends ReconfigurableBase implements LOG.info("RECONFIGURE* changed heartbeatRecheckInterval to " + datanodeManager.getHeartbeatRecheckInterval()); } + case FS_PROTECTED_DIRECTORIES: + return getNamesystem().getFSDirectory().setProtectedDirectories(newVal); default: break; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java index be7b6866757..e7d2d6e180a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Joiner; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; @@ -28,6 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -38,8 +40,10 @@ import java.io.IOException; import java.util.*; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_PROTECTED_DIRECTORIES; /** * Verify that the dfs.namenode.protected.directories setting is respected. @@ -189,6 +193,46 @@ public class TestProtectedDirectories { return matrix; } + @Test + public void testReconfigureProtectedPaths() throws Throwable { + Configuration conf = new HdfsConfiguration(); + Collection protectedPaths = Arrays.asList(new Path("/a"), new Path( + "/b"), new Path("/c")); + Collection unprotectedPaths = Arrays.asList(); + + MiniDFSCluster cluster = setupTestCase(conf, protectedPaths, + unprotectedPaths); + + SortedSet protectedPathsNew = new TreeSet<>( + FSDirectory.normalizePaths(Arrays.asList("/aa", "/bb", "/cc"), + FS_PROTECTED_DIRECTORIES)); + + String protectedPathsStrNew = "/aa,/bb,/cc"; + + NameNode nn = cluster.getNameNode(); + + // change properties + nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, protectedPathsStrNew); + + FSDirectory fsDirectory = nn.getNamesystem().getFSDirectory(); + // verify change + assertEquals(String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES), + protectedPathsNew, fsDirectory.getProtectedDirectories()); + + assertEquals(String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES), + protectedPathsStrNew, nn.getConf().get(FS_PROTECTED_DIRECTORIES)); + + // revert to default + nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, null); + + // verify default + assertEquals(String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES), + new TreeSet(), fsDirectory.getProtectedDirectories()); + + assertEquals(String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES), + null, nn.getConf().get(FS_PROTECTED_DIRECTORIES)); + } + @Test public void testAll() throws Throwable { for (TestMatrixEntry testMatrixEntry : createTestMatrix()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 81f93aa99b9..3ca7fec54dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -234,7 +234,7 @@ public class TestDFSAdmin { final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(3, outs.size()); + assertEquals(4, outs.size()); assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(1)); assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(2)); assertEquals(errs.size(), 0); From fc055a3cbe9545cf1c59421641c7b296aa33f953 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 29 Mar 2016 17:07:55 -0700 Subject: [PATCH 13/75] YARN-4865. Track Reserved resources in ResourceUsage and QueueCapacities. (Sunil G via wangda) --- .../resourcemanager/scheduler/Queue.java | 20 +++++ .../scheduler/capacity/AbstractCSQueue.java | 26 +++++- .../scheduler/capacity/CSQueueUtils.java | 13 +++ .../scheduler/capacity/LeafQueue.java | 16 +++- .../scheduler/capacity/QueueCapacities.java | 38 ++++++++- .../scheduler/fair/FSQueue.java | 8 ++ .../scheduler/fifo/FifoScheduler.java | 12 +++ .../capacity/TestContainerAllocation.java | 13 ++- .../TestNodeLabelContainerAllocation.java | 79 ++++++++++++++++++- .../capacity/TestQueueCapacities.java | 4 +- 10 files changed, 222 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java index 8646381febc..ada2a0b1eea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java @@ -118,4 +118,24 @@ public interface Queue { * @return default application priority */ public Priority getDefaultApplicationPriority(); + + /** + * Increment Reserved Capacity + * + * @param partition + * asked by application + * @param reservedRes + * reserved resource asked + */ + public void incReservedResource(String partition, Resource reservedRes); + + /** + * Decrement Reserved Capacity + * + * @param partition + * asked by application + * @param reservedRes + * reserved resource asked + */ + public void decReservedResource(String partition, Resource reservedRes); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 955f8faca68..6e715fb225d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -534,7 +534,31 @@ public abstract class AbstractCSQueue implements CSQueue { } return true; } - + + @Override + public void incReservedResource(String partition, Resource reservedRes) { + if (partition == null) { + partition = RMNodeLabelsManager.NO_LABEL; + } + + queueUsage.incReserved(partition, reservedRes); + if(null != parent){ + parent.incReservedResource(partition, reservedRes); + } + } + + @Override + public void decReservedResource(String partition, Resource reservedRes) { + if (partition == null) { + partition = RMNodeLabelsManager.NO_LABEL; + } + + queueUsage.decReserved(partition, reservedRes); + if(null != parent){ + parent.decReservedResource(partition, reservedRes); + } + } + @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { if (nodeLabel == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 2f981a74823..9cdcb727899 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -186,6 +186,8 @@ class CSQueueUtils { String nodePartition) { float absoluteUsedCapacity = 0.0f; float usedCapacity = 0.0f; + float reservedCapacity = 0.0f; + float absoluteReservedCapacity = 0.0f; if (Resources.greaterThan(rc, totalPartitionResource, totalPartitionResource, Resources.none())) { @@ -207,11 +209,22 @@ class CSQueueUtils { usedCapacity = Resources.divide(rc, totalPartitionResource, usedResource, queueGuranteedResource); + + Resource resResource = queueResourceUsage.getReserved(nodePartition); + reservedCapacity = + Resources.divide(rc, totalPartitionResource, resResource, + queueGuranteedResource); + absoluteReservedCapacity = + Resources.divide(rc, totalPartitionResource, resResource, + totalPartitionResource); } queueCapacities .setAbsoluteUsedCapacity(nodePartition, absoluteUsedCapacity); queueCapacities.setUsedCapacity(nodePartition, usedCapacity); + queueCapacities.setReservedCapacity(nodePartition, reservedCapacity); + queueCapacities + .setAbsoluteReservedCapacity(nodePartition, absoluteReservedCapacity); } private static Resource getNonPartitionedMaxAvailableResourceToQueue( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 3dc209082db..9a74c2288c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -963,6 +963,13 @@ public class LeafQueue extends AbstractCSQueue { node.getPartition(), reservedOrAllocatedRMContainer, assignment.isIncreasedAllocation()); + // Update reserved metrics + Resource reservedRes = assignment.getAssignmentInformation() + .getReserved(); + if (reservedRes != null && !reservedRes.equals(Resources.none())) { + incReservedResource(node.getPartition(), reservedRes); + } + // Done return assignment; } else if (assignment.getSkipped()) { @@ -1315,7 +1322,14 @@ public class LeafQueue extends AbstractCSQueue { // Book-keeping if (removed) { - + + // track reserved resource for metrics, for normal container + // getReservedResource will be null. + Resource reservedRes = rmContainer.getReservedResource(); + if (reservedRes != null && !reservedRes.equals(Resources.none())) { + decReservedResource(node.getPartition(), reservedRes); + } + // Inform the ordering policy orderingPolicy.containerReleased(application, rmContainer); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index f2c26327ae0..cc4af3dfb47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -50,7 +50,7 @@ public class QueueCapacities { // Usage enum here to make implement cleaner private enum CapacityType { USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5), - MAX_AM_PERC(6); + MAX_AM_PERC(6), RESERVED_CAP(7), ABS_RESERVED_CAP(8); private int idx; @@ -76,6 +76,8 @@ public class QueueCapacities { sb.append("cap=" + capacitiesArr[4] + "%, "); sb.append("abs_cap=" + capacitiesArr[5] + "%}"); sb.append("max_am_perc=" + capacitiesArr[6] + "%}"); + sb.append("reserved_cap=" + capacitiesArr[7] + "%}"); + sb.append("abs_reserved_cap=" + capacitiesArr[8] + "%}"); return sb.toString(); } } @@ -234,6 +236,40 @@ public class QueueCapacities { _set(NL, CapacityType.MAX_AM_PERC, value); } + /* Reserved Capacity Getter and Setter */ + public float getReservedCapacity() { + return _get(NL, CapacityType.RESERVED_CAP); + } + + public float getReservedCapacity(String label) { + return _get(label, CapacityType.RESERVED_CAP); + } + + public void setReservedCapacity(float value) { + _set(NL, CapacityType.RESERVED_CAP, value); + } + + public void setReservedCapacity(String label, float value) { + _set(label, CapacityType.RESERVED_CAP, value); + } + + /* Absolute Reserved Capacity Getter and Setter */ + public float getAbsoluteReservedCapacity() { + return _get(NL, CapacityType.ABS_RESERVED_CAP); + } + + public float getAbsoluteReservedCapacity(String label) { + return _get(label, CapacityType.ABS_RESERVED_CAP); + } + + public void setAbsoluteReservedCapacity(float value) { + _set(NL, CapacityType.ABS_RESERVED_CAP, value); + } + + public void setAbsoluteReservedCapacity(String label, float value) { + _set(label, CapacityType.ABS_RESERVED_CAP, value); + } + /** * Clear configurable fields, like * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index f82411daed5..a33084f80c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -340,6 +340,14 @@ public abstract class FSQueue implements Queue, Schedulable { public void decPendingResource(String nodeLabel, Resource resourceToDec) { } + @Override + public void incReservedResource(String nodeLabel, Resource resourceToInc) { + } + + @Override + public void decReservedResource(String nodeLabel, Resource resourceToDec) { + } + @Override public Priority getDefaultApplicationPriority() { // TODO add implementation for FSParentQueue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index cf125011f95..fba4c1318d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -220,6 +220,18 @@ public class FifoScheduler extends // TODO add implementation for FIFO scheduler return null; } + + @Override + public void incReservedResource(String partition, Resource reservedRes) { + // TODO add implementation for FIFO scheduler + + } + + @Override + public void decReservedResource(String partition, Resource reservedRes) { + // TODO add implementation for FIFO scheduler + + } }; public FifoScheduler() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 1f22a062d64..84eba109611 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -367,7 +367,8 @@ public class TestContainerAllocation { CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); - + LeafQueue leafQueue = (LeafQueue) cs.getQueue("default"); + // Do node heartbeats 2 times // First time will allocate container for app1, second time will reserve // container for app2 @@ -393,7 +394,11 @@ public class TestContainerAllocation { // Usage of queue = 4G + 2 * 1G + 4G (reserved) Assert.assertEquals(10 * GB, cs.getRootQueue().getQueueResourceUsage() .getUsed().getMemory()); - + Assert.assertEquals(4 * GB, cs.getRootQueue().getQueueResourceUsage() + .getReserved().getMemory()); + Assert.assertEquals(4 * GB, leafQueue.getQueueResourceUsage().getReserved() + .getMemory()); + // Cancel asks of app2 and re-kick RM am2.allocate("*", 4 * GB, 0, new ArrayList()); cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); @@ -405,6 +410,10 @@ public class TestContainerAllocation { Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer()); Assert.assertEquals(6 * GB, cs.getRootQueue().getQueueResourceUsage() .getUsed().getMemory()); + Assert.assertEquals(0, cs.getRootQueue().getQueueResourceUsage() + .getReserved().getMemory()); + Assert.assertEquals(0, leafQueue.getQueueResourceUsage().getReserved() + .getMemory()); rm1.close(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 1ee201dfc9f..dc74593eef0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -462,7 +462,84 @@ public class TestNodeLabelContainerAllocation { rm1.close(); } - + + @Test (timeout = 120000) + public void testContainerReservationWithLabels() throws Exception { + // This test is pretty much similar to testContainerAllocateWithLabel. + // Difference is, this test doesn't specify label expression in + // ResourceRequest, + // instead, it uses default queue label expression + + // set node -> label + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", + "z")); + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), + toSet("x"), NodeId.newInstance("h2", 0), toSet("y"), + NodeId.newInstance("h3", 0), toSet("x"))); + + // inject node label manager + MockRM rm1 = new MockRM( + TestUtils.getConfigurationWithDefaultQueueLabels(conf)) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // label = x + rm1.registerNode("h2:1234", 8 * GB); // label = y + rm1.registerNode("h3:1234", 8 * GB); // label = x + + ContainerId containerId; + + // launch an app to queue a1 (label = x), and check all container will + // be allocated in h1 + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + + // request a container. + am1.allocate("*", 4 * GB, 2, new ArrayList()); + containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); + + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); + LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1"); + + // Do node heartbeats 2 times + // First time will allocate container for app1, second time will reserve + // container for app1 + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + checkTaskContainersHost(am1.getApplicationAttemptId(), containerId, rm1, + "h1"); + + // Check if a 4G container allocated for app1, and 4G is reserved + FiCaSchedulerApp schedulerApp1 = cs.getApplicationAttempt(am1 + .getApplicationAttemptId()); + Assert.assertEquals(2, schedulerApp1.getLiveContainers().size()); + Assert.assertTrue(schedulerApp1.getReservedContainers().size() > 0); + Assert.assertEquals(9 * GB, cs.getRootQueue().getQueueResourceUsage() + .getUsed("x").getMemory()); + Assert.assertEquals(4 * GB, cs.getRootQueue().getQueueResourceUsage() + .getReserved("x").getMemory()); + Assert.assertEquals(4 * GB, + leafQueue.getQueueResourceUsage().getReserved("x").getMemory()); + + // Cancel asks of app2 and re-kick RM + am1.allocate("*", 4 * GB, 0, new ArrayList()); + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + + Assert.assertEquals(5 * GB, cs.getRootQueue().getQueueResourceUsage() + .getUsed("x").getMemory()); + Assert.assertEquals(0, cs.getRootQueue().getQueueResourceUsage() + .getReserved("x").getMemory()); + Assert.assertEquals(0, leafQueue.getQueueResourceUsage().getReserved("x") + .getMemory()); + rm1.close(); + } + private void checkPendingResource(MockRM rm, int priority, ApplicationAttemptId attemptId, int memory) { CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java index 9d2fa15d4e7..356ed464678 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java @@ -44,7 +44,9 @@ public class TestQueueCapacities { { "AbsoluteUsedCapacity" }, { "MaximumCapacity" }, { "AbsoluteMaximumCapacity" }, - { "MaxAMResourcePercentage" } }); + { "MaxAMResourcePercentage" }, + { "ReservedCapacity" }, + { "AbsoluteReservedCapacity" }}); } public TestQueueCapacities(String suffix) { From e3d15a2e236f93325c67f9e52fb16f645c44238a Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Tue, 29 Mar 2016 17:13:52 -0700 Subject: [PATCH 14/75] DOOP-12972. Lz4Compressor#getLibraryName returns the wrong version number (cmccabe) --- .../src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index f742384efc7..2c8af1b9115 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -87,7 +87,9 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_getLibraryName( JNIEnv *env, jclass class ) { - return (*env)->NewStringUTF(env, "revision:99"); + char version_buf[128]; + snprintf(version_buf, sizeof(version_buf), "revision:%d", LZ4_versionNumber()); + return (*env)->NewStringUTF(env, version_buf); } JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirectHC From 82862704665aa0bad51118b1b48c7230b194c01c Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 29 Mar 2016 17:21:29 -0700 Subject: [PATCH 15/75] YARN-4436. DistShell ApplicationMaster.ExecBatScripStringtPath is misspelled (mattlamantia via rkanter) --- .../distributedshell/ApplicationMaster.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 0f82903dfe0..cbe03480550 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -259,8 +259,9 @@ public class ApplicationMaster { private String domainId = null; // Hardcoded path to shell script in launch container's local env - private static final String ExecShellStringPath = Client.SCRIPT_PATH + ".sh"; - private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH + private static final String EXEC_SHELL_STRING_PATH = Client.SCRIPT_PATH + + ".sh"; + private static final String EXEC_BAT_SCRIPT_STRING_PATH = Client.SCRIPT_PATH + ".bat"; // Hardcoded path to custom log_properties @@ -1025,8 +1026,8 @@ public class ApplicationMaster { LocalResource shellRsrc = LocalResource.newInstance(yarnUrl, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, shellScriptPathLen, shellScriptPathTimestamp); - localResources.put(Shell.WINDOWS ? ExecBatScripStringtPath : - ExecShellStringPath, shellRsrc); + localResources.put(Shell.WINDOWS ? EXEC_BAT_SCRIPT_STRING_PATH : + EXEC_SHELL_STRING_PATH, shellRsrc); shellCommand = Shell.WINDOWS ? windows_command : linux_bash_command; } @@ -1037,8 +1038,8 @@ public class ApplicationMaster { vargs.add(shellCommand); // Set shell script path if (!scriptPath.isEmpty()) { - vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath - : ExecShellStringPath); + vargs.add(Shell.WINDOWS ? EXEC_BAT_SCRIPT_STRING_PATH + : EXEC_SHELL_STRING_PATH); } // Set args for the shell command if any From 690d8a368d3e967495eafea27659b6124989c89e Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 30 Mar 2016 11:42:54 +0900 Subject: [PATCH 16/75] MAPREDUCE-6663. [NNBench] Refactor nnbench as a Tool implementation. Contributed by Brahma Reddy Battula. --- .../java/org/apache/hadoop/hdfs/NNBench.java | 237 ++++++++++-------- .../org/apache/hadoop/hdfs/TestNNBench.java | 84 +++++++ 2 files changed, 210 insertions(+), 111 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java index 96c4710b48c..ee3cc00fd1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java @@ -25,7 +25,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; -import java.net.InetAddress; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; @@ -33,6 +32,7 @@ import java.util.StringTokenizer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataInputStream; @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; @@ -54,6 +55,8 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * This program executes a specified operation that applies load to @@ -74,49 +77,48 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat; * must be run before running the other operations. */ -public class NNBench { +public class NNBench extends Configured implements Tool { private static final Log LOG = LogFactory.getLog( "org.apache.hadoop.hdfs.NNBench"); - protected static String CONTROL_DIR_NAME = "control"; - protected static String OUTPUT_DIR_NAME = "output"; - protected static String DATA_DIR_NAME = "data"; - protected static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log"; - protected static final String NNBENCH_VERSION = "NameNode Benchmark 0.4"; - - public static String operation = "none"; - public static long numberOfMaps = 1l; // default is 1 - public static long numberOfReduces = 1l; // default is 1 - public static long startTime = + private static String CONTROL_DIR_NAME = "control"; + private static String OUTPUT_DIR_NAME = "output"; + private static String DATA_DIR_NAME = "data"; + static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log"; + private static final String NNBENCH_VERSION = "NameNode Benchmark 0.4"; + + private String operation = "none"; + private long numberOfMaps = 1l; // default is 1 + private long numberOfReduces = 1l; // default is 1 + private long startTime = System.currentTimeMillis() + (120 * 1000); // default is 'now' + 2min - public static long blockSize = 1l; // default is 1 - public static int bytesToWrite = 0; // default is 0 - public static long bytesPerChecksum = 1l; // default is 1 - public static long numberOfFiles = 1l; // default is 1 - public static short replicationFactorPerFile = 1; // default is 1 - public static String baseDir = "/benchmarks/NNBench"; // default - public static boolean readFileAfterOpen = false; // default is to not read - + private long blockSize = 1l; // default is 1 + private int bytesToWrite = 0; // default is 0 + private long bytesPerChecksum = 1l; // default is 1 + private long numberOfFiles = 1l; // default is 1 + private short replicationFactorPerFile = 1; // default is 1 + private String baseDir = "/benchmarks/NNBench"; // default + private boolean readFileAfterOpen = false; // default is to not read + private boolean isHelpMessage = false; // Supported operations private static final String OP_CREATE_WRITE = "create_write"; private static final String OP_OPEN_READ = "open_read"; private static final String OP_RENAME = "rename"; private static final String OP_DELETE = "delete"; + private static final int MAX_OPERATION_EXCEPTIONS = 1000; // To display in the format that matches the NN and DN log format // Example: 2007-10-26 00:01:19,853 static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd' 'HH:mm:ss','S"); - - private static Configuration config = new Configuration(); /** * Clean up the files before a test run * * @throws IOException on error */ - private static void cleanupBeforeTestrun() throws IOException { - FileSystem tempFS = FileSystem.get(config); + private void cleanupBeforeTestrun() throws IOException { + FileSystem tempFS = FileSystem.get(getConf()); // Delete the data directory only if it is the create/write operation if (operation.equals(OP_CREATE_WRITE)) { @@ -133,8 +135,7 @@ public class NNBench { * * @throws IOException on error */ - private static void createControlFiles() throws IOException { - FileSystem tempFS = FileSystem.get(config); + private void createControlFiles() throws IOException { LOG.info("Creating " + numberOfMaps + " control files"); for (int i = 0; i < numberOfMaps; i++) { @@ -144,8 +145,9 @@ public class NNBench { SequenceFile.Writer writer = null; try { - writer = SequenceFile.createWriter(tempFS, config, filePath, Text.class, - LongWritable.class, CompressionType.NONE); + writer = SequenceFile.createWriter(getConf(), Writer.file(filePath), + Writer.keyClass(Text.class), Writer.valueClass(LongWritable.class), + Writer.compression(CompressionType.NONE)); writer.append(new Text(strFileName), new LongWritable(i)); } finally { if (writer != null) { @@ -208,23 +210,23 @@ public class NNBench { * line's arguments * @param length total number of arguments */ - public static void checkArgs(final int index, final int length) { + private static void checkArgs(final int index, final int length) { if (index == length) { displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException("Not enough arguments"); } } /** * Parse input arguments + * @param args array of command line's parameters to be parsed * - * @param args array of command line's parameters to be parsed */ - public static void parseInputs(final String[] args) { + private void parseInputs(final String[] args) { // If there are no command line arguments, exit if (args.length == 0) { displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException("Give valid inputs"); } // Parse command line args @@ -263,7 +265,7 @@ public class NNBench { readFileAfterOpen = Boolean.parseBoolean(args[++i]); } else if (args[i].equals("-help")) { displayUsage(); - System.exit(-1); + isHelpMessage = true; } } @@ -281,31 +283,30 @@ public class NNBench { LOG.info(" Read file after open: " + readFileAfterOpen); // Set user-defined parameters, so the map method can access the values - config.set("test.nnbench.operation", operation); - config.setLong("test.nnbench.maps", numberOfMaps); - config.setLong("test.nnbench.reduces", numberOfReduces); - config.setLong("test.nnbench.starttime", startTime); - config.setLong("test.nnbench.blocksize", blockSize); - config.setInt("test.nnbench.bytestowrite", bytesToWrite); - config.setLong("test.nnbench.bytesperchecksum", bytesPerChecksum); - config.setLong("test.nnbench.numberoffiles", numberOfFiles); - config.setInt("test.nnbench.replicationfactor", + getConf().set("test.nnbench.operation", operation); + getConf().setLong("test.nnbench.maps", numberOfMaps); + getConf().setLong("test.nnbench.reduces", numberOfReduces); + getConf().setLong("test.nnbench.starttime", startTime); + getConf().setLong("test.nnbench.blocksize", blockSize); + getConf().setInt("test.nnbench.bytestowrite", bytesToWrite); + getConf().setLong("test.nnbench.bytesperchecksum", bytesPerChecksum); + getConf().setLong("test.nnbench.numberoffiles", numberOfFiles); + getConf().setInt("test.nnbench.replicationfactor", (int) replicationFactorPerFile); - config.set("test.nnbench.basedir", baseDir); - config.setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen); + getConf().set("test.nnbench.basedir", baseDir); + getConf().setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen); - config.set("test.nnbench.datadir.name", DATA_DIR_NAME); - config.set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME); - config.set("test.nnbench.controldir.name", CONTROL_DIR_NAME); + getConf().set("test.nnbench.datadir.name", DATA_DIR_NAME); + getConf().set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME); + getConf().set("test.nnbench.controldir.name", CONTROL_DIR_NAME); } /** * Analyze the results - * * @throws IOException on error */ - private static void analyzeResults() throws IOException { - final FileSystem fs = FileSystem.get(config); + private int analyzeResults() throws IOException { + final FileSystem fs = FileSystem.get(getConf()); Path reduceDir = new Path(baseDir, OUTPUT_DIR_NAME); long totalTimeAL1 = 0l; @@ -322,32 +323,31 @@ public class NNBench { for (FileStatus status : fss) { Path reduceFile = status.getPath(); - DataInputStream in; - in = new DataInputStream(fs.open(reduceFile)); + try (DataInputStream in = new DataInputStream(fs.open(reduceFile)); + BufferedReader lines = + new BufferedReader(new InputStreamReader(in))) { - BufferedReader lines; - lines = new BufferedReader(new InputStreamReader(in)); - - String line; - while ((line = lines.readLine()) != null) { - StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;"); - String attr = tokens.nextToken(); - if (attr.endsWith(":totalTimeAL1")) { - totalTimeAL1 = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":totalTimeAL2")) { - totalTimeAL2 = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":totalTimeTPmS")) { - totalTimeTPmS = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":latemaps")) { - lateMaps = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":numOfExceptions")) { - numOfExceptions = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":successfulFileOps")) { - successfulFileOps = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":mapStartTimeTPmS")) { - mapStartTimeTPmS = Long.parseLong(tokens.nextToken()); - } else if (attr.endsWith(":mapEndTimeTPmS")) { - mapEndTimeTPmS = Long.parseLong(tokens.nextToken()); + String line; + while ((line = lines.readLine()) != null) { + StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;"); + String attr = tokens.nextToken(); + if (attr.endsWith(":totalTimeAL1")) { + totalTimeAL1 = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":totalTimeAL2")) { + totalTimeAL2 = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":totalTimeTPmS")) { + totalTimeTPmS = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":latemaps")) { + lateMaps = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":numOfExceptions")) { + numOfExceptions = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":successfulFileOps")) { + successfulFileOps = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":mapStartTimeTPmS")) { + mapStartTimeTPmS = Long.parseLong(tokens.nextToken()); + } else if (attr.endsWith(":mapEndTimeTPmS")) { + mapEndTimeTPmS = Long.parseLong(tokens.nextToken()); + } } } } @@ -444,25 +444,29 @@ public class NNBench { " RAW DATA: # of exceptions: " + numOfExceptions, "" }; - PrintStream res = new PrintStream(new FileOutputStream( - new File(DEFAULT_RES_FILE_NAME), true)); - - // Write to a file and also dump to log - for(int i = 0; i < resultLines.length; i++) { - LOG.info(resultLines[i]); - res.println(resultLines[i]); + try (PrintStream res = new PrintStream( + new FileOutputStream(new File(DEFAULT_RES_FILE_NAME), true))) { + // Write to a file and also dump to log + for (String resultLine : resultLines) { + LOG.info(resultLine); + res.println(resultLine); + } } + if(numOfExceptions >= MAX_OPERATION_EXCEPTIONS){ + return -1; + } + return 0; } - + /** * Run the test * * @throws IOException on error */ - public static void runTests() throws IOException { - config.setLong("io.bytes.per.checksum", bytesPerChecksum); + private void runTests() throws IOException { + getConf().setLong("io.bytes.per.checksum", bytesPerChecksum); - JobConf job = new JobConf(config, NNBench.class); + JobConf job = new JobConf(getConf(), NNBench.class); job.setJobName("NNBench-" + operation); FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME)); @@ -487,7 +491,7 @@ public class NNBench { /** * Validate the inputs */ - public static void validateInputs() { + private void validateInputs() { // If it is not one of the four operations, then fail if (!operation.equals(OP_CREATE_WRITE) && !operation.equals(OP_OPEN_READ) && @@ -495,7 +499,8 @@ public class NNBench { !operation.equals(OP_DELETE)) { System.err.println("Error: Unknown operation: " + operation); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Unknown operation: " + operation); } // If number of maps is a negative number, then fail @@ -503,57 +508,66 @@ public class NNBench { if (numberOfMaps < 0) { System.err.println("Error: Number of maps must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Number of maps must be a positive number"); } // If number of reduces is a negative number or 0, then fail if (numberOfReduces <= 0) { System.err.println("Error: Number of reduces must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Number of reduces must be a positive number"); } // If blocksize is a negative number or 0, then fail if (blockSize <= 0) { System.err.println("Error: Block size must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Block size must be a positive number"); } // If bytes to write is a negative number, then fail if (bytesToWrite < 0) { System.err.println("Error: Bytes to write must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Bytes to write must be a positive number"); } // If bytes per checksum is a negative number, then fail if (bytesPerChecksum < 0) { System.err.println("Error: Bytes per checksum must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Bytes per checksum must be a positive number"); } // If number of files is a negative number, then fail if (numberOfFiles < 0) { System.err.println("Error: Number of files must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Number of files must be a positive number"); } // If replication factor is a negative number, then fail if (replicationFactorPerFile < 0) { System.err.println("Error: Replication factor must be a positive number"); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Replication factor must be a positive number"); } // If block size is not a multiple of bytesperchecksum, fail if (blockSize % bytesPerChecksum != 0) { - System.err.println("Error: Block Size in bytes must be a multiple of " + - "bytes per checksum: "); + System.err.println("Error: Block Size in bytes must be a multiple of " + + "bytes per checksum: "); displayUsage(); - System.exit(-1); + throw new HadoopIllegalArgumentException( + "Error: Block Size in bytes must be a multiple of " + + "bytes per checksum:"); } } /** @@ -562,13 +576,22 @@ public class NNBench { * @param args array of command line arguments * @throws IOException indicates a problem with test startup */ - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(new NNBench(), args); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { // Display the application version string displayVersion(); // Parse the inputs parseInputs(args); - + if (isHelpMessage) { + return 0; + } + // Validate inputs validateInputs(); @@ -582,7 +605,7 @@ public class NNBench { runTests(); // Analyze results - analyzeResults(); + return analyzeResults(); } @@ -592,7 +615,6 @@ public class NNBench { static class NNBenchMapper extends Configured implements Mapper { FileSystem filesystem = null; - private String hostName = null; long numberOfFiles = 1l; long blkSize = 1l; @@ -602,7 +624,6 @@ public class NNBench { String dataDirName = null; String op = null; boolean readFile = false; - final int MAX_OPERATION_EXCEPTIONS = 1000; // Data to collect from the operation int numOfExceptions = 0; @@ -628,12 +649,6 @@ public class NNBench { } catch(Exception e) { throw new RuntimeException("Cannot get file system.", e); } - - try { - hostName = InetAddress.getLocalHost().getHostName(); - } catch(Exception e) { - throw new RuntimeException("Error getting hostname", e); - } } /** @@ -678,7 +693,7 @@ public class NNBench { LongWritable value, OutputCollector output, Reporter reporter) throws IOException { - Configuration conf = filesystem.getConf(); + Configuration conf = getConf(); numberOfFiles = conf.getLong("test.nnbench.numberoffiles", 1l); blkSize = conf.getLong("test.nnbench.blocksize", 1l); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java new file mode 100644 index 00000000000..9f9814d7226 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.HadoopTestCase; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Test; + +public class TestNNBench extends HadoopTestCase { + private static final String BASE_DIR = + new File(System.getProperty("test.build.data", "build/test/data"), + "NNBench").getAbsolutePath(); + + public TestNNBench() throws IOException { + super(LOCAL_MR, LOCAL_FS, 1, 1); + } + + @After + public void tearDown() throws Exception { + getFileSystem().delete(new Path(BASE_DIR), true); + getFileSystem().delete(new Path(NNBench.DEFAULT_RES_FILE_NAME), true); + super.tearDown(); + } + + @Test(timeout = 30000) + public void testNNBenchCreateReadAndDelete() throws Exception { + runNNBench(createJobConf(), "create_write"); + Path path = new Path(BASE_DIR + "/data/file_0_0"); + assertTrue("create_write should create the file", + getFileSystem().exists(path)); + runNNBench(createJobConf(), "open_read"); + runNNBench(createJobConf(), "delete"); + assertFalse("Delete operation should delete the file", + getFileSystem().exists(path)); + } + + @Test(timeout = 30000) + public void testNNBenchCreateAndRename() throws Exception { + runNNBench(createJobConf(), "create_write"); + Path path = new Path(BASE_DIR + "/data/file_0_0"); + assertTrue("create_write should create the file", + getFileSystem().exists(path)); + runNNBench(createJobConf(), "rename"); + Path renamedPath = new Path(BASE_DIR + "/data/file_0_r_0"); + assertFalse("Rename should rename the file", getFileSystem().exists(path)); + assertTrue("Rename should rename the file", + getFileSystem().exists(renamedPath)); + } + + private void runNNBench(Configuration conf, String operation) + throws Exception { + String[] genArgs = { "-operation", operation, "-baseDir", BASE_DIR, + "-startTime", "" + (Time.now() / 1000 + 3) }; + + assertEquals(0, ToolRunner.run(conf, new NNBench(), genArgs)); + } + +} From 09d63d5a192b5d6b172f94ff6c94da348fd49ea6 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Wed, 30 Mar 2016 14:22:00 +0800 Subject: [PATCH 17/75] HDFS-5177. blocksScheduled count should be decremented for abandoned blocks (Contributed by Vinayakumar B) --- .../blockmanagement/DatanodeStorageInfo.java | 10 ++++ .../server/namenode/FSDirWriteFileOp.java | 4 ++ .../hdfs/TestBlocksScheduledCounter.java | 51 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index c4729eafecd..843a8d514ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -298,6 +298,16 @@ public class DatanodeStorageInfo { } } + /** + * Decrement the number of blocks scheduled for each given storage. This will + * be called during abandon block or delete of UC block. + */ + public static void decrementBlocksScheduled(DatanodeStorageInfo... storages) { + for (DatanodeStorageInfo s : storages) { + s.getDatanodeDescriptor().decrementBlocksScheduled(s.getStorageType()); + } + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 41fd8690a18..ab2f0fa32f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -82,6 +82,10 @@ class FSDirWriteFileOp { if (uc == null) { return false; } + if (uc.getUnderConstructionFeature() != null) { + DatanodeStorageInfo.decrementBlocksScheduled(uc + .getUnderConstructionFeature().getExpectedStorageLocations()); + } fsd.getBlockManager().removeBlockFromMap(uc); if(NameNode.stateChangeLog.isDebugEnabled()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java index b9432197489..18942780e31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java @@ -78,4 +78,55 @@ public class TestBlocksScheduledCounter { out.close(); assertEquals(0, dn.getBlocksScheduled()); } + + /** + * Abandon block should decrement the scheduledBlocks count for the dataNode. + */ + @Test + public void testScheduledBlocksCounterShouldDecrementOnAbandonBlock() + throws Exception { + cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).numDataNodes( + 2).build(); + + cluster.waitActive(); + fs = cluster.getFileSystem(); + + DatanodeManager datanodeManager = cluster.getNamesystem().getBlockManager() + .getDatanodeManager(); + ArrayList dnList = new ArrayList(); + datanodeManager.fetchDatanodes(dnList, dnList, false); + for (DatanodeDescriptor descriptor : dnList) { + assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(), + 0, descriptor.getBlocksScheduled()); + } + + cluster.getDataNodes().get(0).shutdown(); + // open a file an write a few bytes: + FSDataOutputStream out = fs.create(new Path("/testBlockScheduledCounter"), + (short) 2); + for (int i = 0; i < 1024; i++) { + out.write(i); + } + // flush to make sure a block is allocated. + out.hflush(); + + DatanodeDescriptor abandonedDn = datanodeManager.getDatanode(cluster + .getDataNodes().get(0).getDatanodeId()); + assertEquals("for the abandoned dn scheduled counts should be 0", 0, + abandonedDn.getBlocksScheduled()); + + for (DatanodeDescriptor descriptor : dnList) { + if (descriptor.equals(abandonedDn)) { + continue; + } + assertEquals("Blocks scheduled should be 1 for " + descriptor.getName(), + 1, descriptor.getBlocksScheduled()); + } + // close the file and the counter should go to zero. + out.close(); + for (DatanodeDescriptor descriptor : dnList) { + assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(), + 0, descriptor.getBlocksScheduled()); + } + } } From 60e4116bf1d00afed91010e57357fe54057e4e39 Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 30 Mar 2016 12:43:52 -0700 Subject: [PATCH 18/75] YARN-4822. Refactor existing Preemption Policy of CS for easier adding new approach to select preemption candidates. Contributed by Wangda Tan --- .../monitor/SchedulingEditPolicy.java | 8 +- .../monitor/SchedulingMonitor.java | 4 - .../CapacitySchedulerPreemptionContext.java | 52 + .../CapacitySchedulerPreemptionUtils.java | 65 + .../capacity/FifoCandidatesSelector.java | 364 ++++++ .../PreemptableResourceCalculator.java | 370 ++++++ .../PreemptionCandidatesSelector.java | 52 + .../ProportionalCapacityPreemptionPolicy.java | 1096 ++++------------- .../capacity/TempQueuePerPartition.java | 159 +++ .../CapacitySchedulerConfiguration.java | 45 + .../capacity/preemption/PreemptableQueue.java | 6 - .../preemption/PreemptionManager.java | 2 +- ...tProportionalCapacityPreemptionPolicy.java | 133 +- ...cityPreemptionPolicyForNodePartitions.java | 78 +- .../TestCapacitySchedulerPreemption.java | 14 +- 15 files changed, 1437 insertions(+), 1011 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java index 0d587d8d25e..47458a3b347 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java @@ -23,7 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResour public interface SchedulingEditPolicy { - public void init(Configuration config, RMContext context, + void init(Configuration config, RMContext context, PreemptableResourceScheduler scheduler); /** @@ -31,10 +31,10 @@ public interface SchedulingEditPolicy { * allowed to track containers and affect the scheduler. The "actions" * performed are passed back through an EventHandler. */ - public void editSchedule(); + void editSchedule(); - public long getMonitoringInterval(); + long getMonitoringInterval(); - public String getPolicyName(); + String getPolicyName(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java index d4c129b3bfd..55ec858fa6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java @@ -45,10 +45,6 @@ public class SchedulingMonitor extends AbstractService { this.rmContext = rmContext; } - public long getMonitorInterval() { - return monitorInterval; - } - @VisibleForTesting public synchronized SchedulingEditPolicy getSchedulingEditPolicy() { return scheduleEditPolicy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java new file mode 100644 index 00000000000..c52127d25ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; + +import java.util.Collection; +import java.util.Set; + +interface CapacitySchedulerPreemptionContext { + CapacityScheduler getScheduler(); + + TempQueuePerPartition getQueueByPartition(String queueName, + String partition); + + Collection getQueuePartitions(String queueName); + + ResourceCalculator getResourceCalculator(); + + RMContext getRMContext(); + + boolean isObserveOnly(); + + Set getKillableContainers(); + + double getMaxIgnoreOverCapacity(); + + double getNaturalTerminationFactor(); + + Set getLeafQueueNames(); + + Set getAllPartitions(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java new file mode 100644 index 00000000000..a71f108449d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class CapacitySchedulerPreemptionUtils { + public static Map getResToObtainByPartitionForLeafQueue( + CapacitySchedulerPreemptionContext context, String queueName, + Resource clusterResource) { + Map resToObtainByPartition = new HashMap<>(); + // compute resToObtainByPartition considered inter-queue preemption + for (TempQueuePerPartition qT : context.getQueuePartitions(queueName)) { + if (qT.preemptionDisabled) { + continue; + } + + // Only add resToObtainByPartition when actuallyToBePreempted resource >= 0 + if (Resources.greaterThan(context.getResourceCalculator(), + clusterResource, qT.actuallyToBePreempted, Resources.none())) { + resToObtainByPartition.put(qT.partition, + Resources.clone(qT.actuallyToBePreempted)); + } + } + + return resToObtainByPartition; + } + + public static boolean isContainerAlreadySelected(RMContainer container, + Map> selectedCandidates) { + if (null == selectedCandidates) { + return false; + } + + Set containers = selectedCandidates.get( + container.getApplicationAttemptId()); + if (containers == null) { + return false; + } + return containers.contains(container); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java new file mode 100644 index 00000000000..499d0ff88dc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java @@ -0,0 +1,364 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +public class FifoCandidatesSelector + extends PreemptionCandidatesSelector { + private static final Log LOG = + LogFactory.getLog(FifoCandidatesSelector.class); + private PreemptableResourceCalculator preemptableAmountCalculator; + + FifoCandidatesSelector( + CapacitySchedulerPreemptionContext preemptionContext) { + super(preemptionContext); + + preemptableAmountCalculator = new PreemptableResourceCalculator( + preemptionContext); + } + + @Override + public Map> selectCandidates( + Map> selectedCandidates, + Resource clusterResource, Resource totalPreemptionAllowed) { + // Calculate how much resources we need to preempt + preemptableAmountCalculator.computeIdealAllocation(clusterResource, + totalPreemptionAllowed); + + Map> preemptMap = + new HashMap<>(); + List skippedAMContainerlist = new ArrayList<>(); + + // Loop all leaf queues + for (String queueName : preemptionContext.getLeafQueueNames()) { + // check if preemption disabled for the queue + if (preemptionContext.getQueueByPartition(queueName, + RMNodeLabelsManager.NO_LABEL).preemptionDisabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("skipping from queue=" + queueName + + " because it's a non-preemptable queue"); + } + continue; + } + + // compute resToObtainByPartition considered inter-queue preemption + LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName, + RMNodeLabelsManager.NO_LABEL).leafQueue; + + Map resToObtainByPartition = + CapacitySchedulerPreemptionUtils + .getResToObtainByPartitionForLeafQueue(preemptionContext, + queueName, clusterResource); + + synchronized (leafQueue) { + // go through all ignore-partition-exclusivity containers first to make + // sure such containers will be preemptionCandidates first + Map> ignorePartitionExclusivityContainers = + leafQueue.getIgnoreExclusivityRMContainers(); + for (String partition : resToObtainByPartition.keySet()) { + if (ignorePartitionExclusivityContainers.containsKey(partition)) { + TreeSet rmContainers = + ignorePartitionExclusivityContainers.get(partition); + // We will check container from reverse order, so latter submitted + // application's containers will be preemptionCandidates first. + for (RMContainer c : rmContainers.descendingSet()) { + if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, + selectedCandidates)) { + // Skip already selected containers + continue; + } + boolean preempted = tryPreemptContainerAndDeductResToObtain( + resToObtainByPartition, c, clusterResource, preemptMap, + totalPreemptionAllowed); + if (!preempted) { + continue; + } + } + } + } + + // preempt other containers + Resource skippedAMSize = Resource.newInstance(0, 0); + Iterator desc = + leafQueue.getOrderingPolicy().getPreemptionIterator(); + while (desc.hasNext()) { + FiCaSchedulerApp fc = desc.next(); + // When we complete preempt from one partition, we will remove from + // resToObtainByPartition, so when it becomes empty, we can get no + // more preemption is needed + if (resToObtainByPartition.isEmpty()) { + break; + } + + preemptFrom(fc, clusterResource, resToObtainByPartition, + skippedAMContainerlist, skippedAMSize, preemptMap, + totalPreemptionAllowed); + } + + // Can try preempting AMContainers (still saving atmost + // maxAMCapacityForThisQueue AMResource's) if more resources are + // required to be preemptionCandidates from this Queue. + Resource maxAMCapacityForThisQueue = Resources.multiply( + Resources.multiply(clusterResource, + leafQueue.getAbsoluteCapacity()), + leafQueue.getMaxAMResourcePerQueuePercent()); + + preemptAMContainers(clusterResource, preemptMap, skippedAMContainerlist, + resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue, + totalPreemptionAllowed); + } + } + + return preemptMap; + } + + /** + * As more resources are needed for preemption, saved AMContainers has to be + * rescanned. Such AMContainers can be preemptionCandidates based on resToObtain, but + * maxAMCapacityForThisQueue resources will be still retained. + * + * @param clusterResource + * @param preemptMap + * @param skippedAMContainerlist + * @param skippedAMSize + * @param maxAMCapacityForThisQueue + */ + private void preemptAMContainers(Resource clusterResource, + Map> preemptMap, + List skippedAMContainerlist, + Map resToObtainByPartition, Resource skippedAMSize, + Resource maxAMCapacityForThisQueue, Resource totalPreemptionAllowed) { + for (RMContainer c : skippedAMContainerlist) { + // Got required amount of resources for preemption, can stop now + if (resToObtainByPartition.isEmpty()) { + break; + } + // Once skippedAMSize reaches down to maxAMCapacityForThisQueue, + // container selection iteration for preemption will be stopped. + if (Resources.lessThanOrEqual(rc, clusterResource, skippedAMSize, + maxAMCapacityForThisQueue)) { + break; + } + + boolean preempted = + tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, + clusterResource, preemptMap, totalPreemptionAllowed); + if (preempted) { + Resources.subtractFrom(skippedAMSize, c.getAllocatedResource()); + } + } + skippedAMContainerlist.clear(); + } + + private boolean preemptMapContains( + Map> preemptMap, + ApplicationAttemptId attemptId, RMContainer rmContainer) { + Set rmContainers; + if (null == (rmContainers = preemptMap.get(attemptId))) { + return false; + } + return rmContainers.contains(rmContainer); + } + + /** + * Return should we preempt rmContainer. If we should, deduct from + * resourceToObtainByPartition + */ + private boolean tryPreemptContainerAndDeductResToObtain( + Map resourceToObtainByPartitions, + RMContainer rmContainer, Resource clusterResource, + Map> preemptMap, + Resource totalPreemptionAllowed) { + ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId(); + + // We will not account resource of a container twice or more + if (preemptMapContains(preemptMap, attemptId, rmContainer)) { + return false; + } + + String nodePartition = getPartitionByNodeId(rmContainer.getAllocatedNode()); + Resource toObtainByPartition = + resourceToObtainByPartitions.get(nodePartition); + + if (null != toObtainByPartition && Resources.greaterThan(rc, + clusterResource, toObtainByPartition, Resources.none()) && Resources + .fitsIn(rc, clusterResource, rmContainer.getAllocatedResource(), + totalPreemptionAllowed)) { + Resources.subtractFrom(toObtainByPartition, + rmContainer.getAllocatedResource()); + Resources.subtractFrom(totalPreemptionAllowed, + rmContainer.getAllocatedResource()); + + // When we have no more resource need to obtain, remove from map. + if (Resources.lessThanOrEqual(rc, clusterResource, toObtainByPartition, + Resources.none())) { + resourceToObtainByPartitions.remove(nodePartition); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Marked container=" + rmContainer.getContainerId() + + " in partition=" + nodePartition + + " to be preemption candidates"); + } + // Add to preemptMap + addToPreemptMap(preemptMap, attemptId, rmContainer); + return true; + } + + return false; + } + + private String getPartitionByNodeId(NodeId nodeId) { + return preemptionContext.getScheduler().getSchedulerNode(nodeId) + .getPartition(); + } + + /** + * Given a target preemption for a specific application, select containers + * to preempt (after unreserving all reservation for that app). + */ + @SuppressWarnings("unchecked") + private void preemptFrom(FiCaSchedulerApp app, + Resource clusterResource, Map resToObtainByPartition, + List skippedAMContainerlist, Resource skippedAMSize, + Map> selectedContainers, + Resource totalPreemptionAllowed) { + ApplicationAttemptId appId = app.getApplicationAttemptId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Looking at application=" + app.getApplicationAttemptId() + + " resourceToObtain=" + resToObtainByPartition); + } + + // first drop reserved containers towards rsrcPreempt + List reservedContainers = + new ArrayList<>(app.getReservedContainers()); + for (RMContainer c : reservedContainers) { + if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, + selectedContainers)) { + continue; + } + if (resToObtainByPartition.isEmpty()) { + return; + } + + // Try to preempt this container + tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, + clusterResource, selectedContainers, totalPreemptionAllowed); + + if (!preemptionContext.isObserveOnly()) { + preemptionContext.getRMContext().getDispatcher().getEventHandler() + .handle(new ContainerPreemptEvent(appId, c, + SchedulerEventType.KILL_RESERVED_CONTAINER)); + } + } + + // if more resources are to be freed go through all live containers in + // reverse priority and reverse allocation order and mark them for + // preemption + List liveContainers = + new ArrayList<>(app.getLiveContainers()); + + sortContainers(liveContainers); + + for (RMContainer c : liveContainers) { + if (resToObtainByPartition.isEmpty()) { + return; + } + + if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, + selectedContainers)) { + continue; + } + + // Skip already marked to killable containers + if (null != preemptionContext.getKillableContainers() && preemptionContext + .getKillableContainers().contains(c.getContainerId())) { + continue; + } + + // Skip AM Container from preemption for now. + if (c.isAMContainer()) { + skippedAMContainerlist.add(c); + Resources.addTo(skippedAMSize, c.getAllocatedResource()); + continue; + } + + // Try to preempt this container + tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, + clusterResource, selectedContainers, totalPreemptionAllowed); + } + } + + /** + * Compare by reversed priority order first, and then reversed containerId + * order + * @param containers + */ + @VisibleForTesting + static void sortContainers(List containers){ + Collections.sort(containers, new Comparator() { + @Override + public int compare(RMContainer a, RMContainer b) { + Comparator c = new org.apache.hadoop.yarn.server + .resourcemanager.resource.Priority.Comparator(); + int priorityComp = c.compare(b.getContainer().getPriority(), + a.getContainer().getPriority()); + if (priorityComp != 0) { + return priorityComp; + } + return b.getContainerId().compareTo(a.getContainerId()); + } + }); + } + + private void addToPreemptMap( + Map> preemptMap, + ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) { + Set set; + if (null == (set = preemptMap.get(appAttemptId))) { + set = new HashSet<>(); + preemptMap.put(appAttemptId, set); + } + set.add(containerToPreempt); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java new file mode 100644 index 00000000000..2217210e75c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java @@ -0,0 +1,370 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.PriorityQueue; +import java.util.Set; + +/** + * Calculate how much resources need to be preempted for each queue, + * will be used by {@link FifoCandidatesSelector} + */ +public class PreemptableResourceCalculator { + private static final Log LOG = + LogFactory.getLog(PreemptableResourceCalculator.class); + + private final CapacitySchedulerPreemptionContext context; + private final ResourceCalculator rc; + + static class TQComparator implements Comparator { + private ResourceCalculator rc; + private Resource clusterRes; + + TQComparator(ResourceCalculator rc, Resource clusterRes) { + this.rc = rc; + this.clusterRes = clusterRes; + } + + @Override + public int compare(TempQueuePerPartition tq1, TempQueuePerPartition tq2) { + if (getIdealPctOfGuaranteed(tq1) < getIdealPctOfGuaranteed(tq2)) { + return -1; + } + if (getIdealPctOfGuaranteed(tq1) > getIdealPctOfGuaranteed(tq2)) { + return 1; + } + return 0; + } + + // Calculates idealAssigned / guaranteed + // TempQueues with 0 guarantees are always considered the most over + // capacity and therefore considered last for resources. + private double getIdealPctOfGuaranteed(TempQueuePerPartition q) { + double pctOver = Integer.MAX_VALUE; + if (q != null && Resources.greaterThan( + rc, clusterRes, q.guaranteed, Resources.none())) { + pctOver = + Resources.divide(rc, clusterRes, q.idealAssigned, q.guaranteed); + } + return (pctOver); + } + } + + public PreemptableResourceCalculator(CapacitySchedulerPreemptionContext preemptionContext) { + context = preemptionContext; + rc = preemptionContext.getResourceCalculator(); + } + + /** + * Computes a normalizedGuaranteed capacity based on active queues + * @param rc resource calculator + * @param clusterResource the total amount of resources in the cluster + * @param queues the list of queues to consider + */ + private void resetCapacity(ResourceCalculator rc, Resource clusterResource, + Collection queues, boolean ignoreGuar) { + Resource activeCap = Resource.newInstance(0, 0); + + if (ignoreGuar) { + for (TempQueuePerPartition q : queues) { + q.normalizedGuarantee = 1.0f / queues.size(); + } + } else { + for (TempQueuePerPartition q : queues) { + Resources.addTo(activeCap, q.guaranteed); + } + for (TempQueuePerPartition q : queues) { + q.normalizedGuarantee = Resources.divide(rc, clusterResource, + q.guaranteed, activeCap); + } + } + } + + // Take the most underserved TempQueue (the one on the head). Collect and + // return the list of all queues that have the same idealAssigned + // percentage of guaranteed. + protected Collection getMostUnderservedQueues( + PriorityQueue orderedByNeed, TQComparator tqComparator) { + ArrayList underserved = new ArrayList<>(); + while (!orderedByNeed.isEmpty()) { + TempQueuePerPartition q1 = orderedByNeed.remove(); + underserved.add(q1); + TempQueuePerPartition q2 = orderedByNeed.peek(); + // q1's pct of guaranteed won't be larger than q2's. If it's less, then + // return what has already been collected. Otherwise, q1's pct of + // guaranteed == that of q2, so add q2 to underserved list during the + // next pass. + if (q2 == null || tqComparator.compare(q1,q2) < 0) { + return underserved; + } + } + return underserved; + } + + + /** + * Given a set of queues compute the fix-point distribution of unassigned + * resources among them. As pending request of a queue are exhausted, the + * queue is removed from the set and remaining capacity redistributed among + * remaining queues. The distribution is weighted based on guaranteed + * capacity, unless asked to ignoreGuarantee, in which case resources are + * distributed uniformly. + */ + private void computeFixpointAllocation(ResourceCalculator rc, + Resource tot_guarant, Collection qAlloc, + Resource unassigned, boolean ignoreGuarantee) { + // Prior to assigning the unused resources, process each queue as follows: + // If current > guaranteed, idealAssigned = guaranteed + untouchable extra + // Else idealAssigned = current; + // Subtract idealAssigned resources from unassigned. + // If the queue has all of its needs met (that is, if + // idealAssigned >= current + pending), remove it from consideration. + // Sort queues from most under-guaranteed to most over-guaranteed. + TQComparator tqComparator = new TQComparator(rc, tot_guarant); + PriorityQueue orderedByNeed = new PriorityQueue<>(10, + tqComparator); + for (Iterator i = qAlloc.iterator(); i.hasNext();) { + TempQueuePerPartition q = i.next(); + if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) { + q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra); + } else { + q.idealAssigned = Resources.clone(q.current); + } + Resources.subtractFrom(unassigned, q.idealAssigned); + // If idealAssigned < (current + pending), q needs more resources, so + // add it to the list of underserved queues, ordered by need. + Resource curPlusPend = Resources.add(q.current, q.pending); + if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) { + orderedByNeed.add(q); + } + } + + //assign all cluster resources until no more demand, or no resources are left + while (!orderedByNeed.isEmpty() + && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) { + Resource wQassigned = Resource.newInstance(0, 0); + // we compute normalizedGuarantees capacity based on currently active + // queues + resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee); + + // For each underserved queue (or set of queues if multiple are equally + // underserved), offer its share of the unassigned resources based on its + // normalized guarantee. After the offer, if the queue is not satisfied, + // place it back in the ordered list of queues, recalculating its place + // in the order of most under-guaranteed to most over-guaranteed. In this + // way, the most underserved queue(s) are always given resources first. + Collection underserved = + getMostUnderservedQueues(orderedByNeed, tqComparator); + for (Iterator i = underserved.iterator(); i + .hasNext();) { + TempQueuePerPartition sub = i.next(); + Resource wQavail = Resources.multiplyAndNormalizeUp(rc, + unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1)); + Resource wQidle = sub.offer(wQavail, rc, tot_guarant); + Resource wQdone = Resources.subtract(wQavail, wQidle); + + if (Resources.greaterThan(rc, tot_guarant, + wQdone, Resources.none())) { + // The queue is still asking for more. Put it back in the priority + // queue, recalculating its order based on need. + orderedByNeed.add(sub); + } + Resources.addTo(wQassigned, wQdone); + } + Resources.subtractFrom(unassigned, wQassigned); + } + } + + /** + * This method computes (for a single level in the tree, passed as a {@code + * List}) the ideal assignment of resources. This is done + * recursively to allocate capacity fairly across all queues with pending + * demands. It terminates when no resources are left to assign, or when all + * demand is satisfied. + * + * @param rc resource calculator + * @param queues a list of cloned queues to be assigned capacity to (this is + * an out param) + * @param totalPreemptionAllowed total amount of preemption we allow + * @param tot_guarant the amount of capacity assigned to this pool of queues + */ + private void computeIdealResourceDistribution(ResourceCalculator rc, + List queues, Resource totalPreemptionAllowed, + Resource tot_guarant) { + + // qAlloc tracks currently active queues (will decrease progressively as + // demand is met) + List qAlloc = new ArrayList<>(queues); + // unassigned tracks how much resources are still to assign, initialized + // with the total capacity for this set of queues + Resource unassigned = Resources.clone(tot_guarant); + + // group queues based on whether they have non-zero guaranteed capacity + Set nonZeroGuarQueues = new HashSet<>(); + Set zeroGuarQueues = new HashSet<>(); + + for (TempQueuePerPartition q : qAlloc) { + if (Resources + .greaterThan(rc, tot_guarant, q.guaranteed, Resources.none())) { + nonZeroGuarQueues.add(q); + } else { + zeroGuarQueues.add(q); + } + } + + // first compute the allocation as a fixpoint based on guaranteed capacity + computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned, + false); + + // if any capacity is left unassigned, distributed among zero-guarantee + // queues uniformly (i.e., not based on guaranteed capacity, as this is zero) + if (!zeroGuarQueues.isEmpty() + && Resources.greaterThan(rc, tot_guarant, unassigned, Resources.none())) { + computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned, + true); + } + + // based on ideal assignment computed above and current assignment we derive + // how much preemption is required overall + Resource totPreemptionNeeded = Resource.newInstance(0, 0); + for (TempQueuePerPartition t:queues) { + if (Resources.greaterThan(rc, tot_guarant, t.current, t.idealAssigned)) { + Resources.addTo(totPreemptionNeeded, + Resources.subtract(t.current, t.idealAssigned)); + } + } + + // if we need to preempt more than is allowed, compute a factor (0 0) { + // compute ideal distribution at this level + computeIdealResourceDistribution(rc, root.getChildren(), + totalPreemptionAllowed, root.idealAssigned); + // compute recursively for lower levels and build list of leafs + for(TempQueuePerPartition t : root.getChildren()) { + recursivelyComputeIdealAssignment(t, totalPreemptionAllowed); + } + } + } + + + private void calculateResToObtainByPartitionForLeafQueues( + Set leafQueueNames, Resource clusterResource) { + // Loop all leaf queues + for (String queueName : leafQueueNames) { + // check if preemption disabled for the queue + if (context.getQueueByPartition(queueName, + RMNodeLabelsManager.NO_LABEL).preemptionDisabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("skipping from queue=" + queueName + + " because it's a non-preemptable queue"); + } + continue; + } + + // compute resToObtainByPartition considered inter-queue preemption + for (TempQueuePerPartition qT : context.getQueuePartitions(queueName)) { + // we act only if we are violating balance by more than + // maxIgnoredOverCapacity + if (Resources.greaterThan(rc, clusterResource, qT.current, + Resources.multiply(qT.guaranteed, 1.0 + context.getMaxIgnoreOverCapacity()))) { + // we introduce a dampening factor naturalTerminationFactor that + // accounts for natural termination of containers + Resource resToObtain = Resources.multiply(qT.toBePreempted, + context.getNaturalTerminationFactor()); + // Only add resToObtain when it >= 0 + if (Resources.greaterThan(rc, clusterResource, resToObtain, + Resources.none())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Queue=" + queueName + " partition=" + qT.partition + + " resource-to-obtain=" + resToObtain); + } + } + qT.actuallyToBePreempted = Resources.clone(resToObtain); + } else { + qT.actuallyToBePreempted = Resources.none(); + } + } + } + } + + public void computeIdealAllocation(Resource clusterResource, + Resource totalPreemptionAllowed) { + for (String partition : context.getAllPartitions()) { + TempQueuePerPartition tRoot = + context.getQueueByPartition(CapacitySchedulerConfiguration.ROOT, partition); + // compute the ideal distribution of resources among queues + // updates cloned queues state accordingly + tRoot.idealAssigned = tRoot.guaranteed; + recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed); + } + + // based on ideal allocation select containers to be preempted from each + // calculate resource-to-obtain by partition for each leaf queues + calculateResToObtainByPartitionForLeafQueues(context.getLeafQueueNames(), + clusterResource); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java new file mode 100644 index 00000000000..dd33d8f2f13 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; + +import java.util.Map; +import java.util.Set; + +public abstract class PreemptionCandidatesSelector { + protected CapacitySchedulerPreemptionContext preemptionContext; + protected ResourceCalculator rc; + + PreemptionCandidatesSelector( + CapacitySchedulerPreemptionContext preemptionContext) { + this.preemptionContext = preemptionContext; + this.rc = preemptionContext.getResourceCalculator(); + } + + /** + * Get preemption candidates from computed resource sharing and already + * selected candidates. + * + * @param selectedCandidates already selected candidates from previous policies + * @param clusterResource + * @param totalPreemptedResourceAllowed how many resources allowed to be + * preempted in this round + * @return merged selected candidates. + */ + public abstract Map> selectCandidates( + Map> selectedCandidates, + Resource clusterResource, Resource totalPreemptedResourceAllowed); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 9b499c8ede3..7e668b4a683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -17,26 +17,13 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.PriorityQueue; -import java.util.Set; -import java.util.TreeSet; - +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -50,7 +37,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.util.Clock; @@ -58,8 +44,16 @@ import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * This class implement a {@link SchedulingEditPolicy} that is designed to be @@ -80,79 +74,59 @@ import com.google.common.collect.ImmutableSet; * this policy will trigger forced termination of containers (again by generating * {@link ContainerPreemptEvent}). */ -public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolicy { - +public class ProportionalCapacityPreemptionPolicy + implements SchedulingEditPolicy, CapacitySchedulerPreemptionContext { private static final Log LOG = LogFactory.getLog(ProportionalCapacityPreemptionPolicy.class); - /** If true, run the policy but do not affect the cluster with preemption and - * kill events. */ - public static final String OBSERVE_ONLY = - "yarn.resourcemanager.monitor.capacity.preemption.observe_only"; - /** Time in milliseconds between invocations of this policy */ - public static final String MONITORING_INTERVAL = - "yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval"; - /** Time in milliseconds between requesting a preemption from an application - * and killing the container. */ - public static final String WAIT_TIME_BEFORE_KILL = - "yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill"; - /** Maximum percentage of resources preempted in a single round. By - * controlling this value one can throttle the pace at which containers are - * reclaimed from the cluster. After computing the total desired preemption, - * the policy scales it back within this limit. */ - public static final String TOTAL_PREEMPTION_PER_ROUND = - "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round"; - /** Maximum amount of resources above the target capacity ignored for - * preemption. This defines a deadzone around the target capacity that helps - * prevent thrashing and oscillations around the computed target balance. - * High values would slow the time to capacity and (absent natural - * completions) it might prevent convergence to guaranteed capacity. */ - public static final String MAX_IGNORED_OVER_CAPACITY = - "yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity"; - /** - * Given a computed preemption target, account for containers naturally - * expiring and preempt only this percentage of the delta. This determines - * the rate of geometric convergence into the deadzone ({@link - * #MAX_IGNORED_OVER_CAPACITY}). For example, a termination factor of 0.5 - * will reclaim almost 95% of resources within 5 * {@link - * #WAIT_TIME_BEFORE_KILL}, even absent natural termination. */ - public static final String NATURAL_TERMINATION_FACTOR = - "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor"; - - private RMContext rmContext; - private final Clock clock; + + // Configurable fields private double maxIgnoredOverCapacity; private long maxWaitTime; - private CapacityScheduler scheduler; private long monitoringInterval; - private final Map preempted = new HashMap<>(); - - private ResourceCalculator rc; private float percentageClusterPreemptionAllowed; private double naturalTerminationFactor; private boolean observeOnly; - private Map> queueToPartitions = - new HashMap<>(); + private boolean lazyPreempionEnabled; + + // Pointer to other RM components + private RMContext rmContext; + private ResourceCalculator rc; + private CapacityScheduler scheduler; private RMNodeLabelsManager nlm; + // Internal properties to make decisions of what to preempt + private final Map preemptionCandidates = + new HashMap<>(); + private Map> queueToPartitions = + new HashMap<>(); + private List + candidatesSelectionPolicies = new ArrayList<>(); + private Set allPartitions; + private Set leafQueueNames; + // Preemptable Entities, synced from scheduler at every run - private Map preemptableEntities = null; + private Map preemptableQueues; private Set killableContainers; + @SuppressWarnings("unchecked") public ProportionalCapacityPreemptionPolicy() { clock = SystemClock.getInstance(); + allPartitions = Collections.EMPTY_SET; + leafQueueNames = Collections.EMPTY_SET; + preemptableQueues = Collections.EMPTY_MAP; } - public ProportionalCapacityPreemptionPolicy(Configuration config, - RMContext context, CapacityScheduler scheduler) { - this(config, context, scheduler, SystemClock.getInstance()); - } - - public ProportionalCapacityPreemptionPolicy(Configuration config, - RMContext context, CapacityScheduler scheduler, Clock clock) { - init(config, context, scheduler); + @SuppressWarnings("unchecked") + @VisibleForTesting + public ProportionalCapacityPreemptionPolicy(RMContext context, + CapacityScheduler scheduler, Clock clock) { + init(context.getYarnConfiguration(), context, scheduler); this.clock = clock; + allPartitions = Collections.EMPTY_SET; + leafQueueNames = Collections.EMPTY_SET; + preemptableQueues = Collections.EMPTY_MAP; } public void init(Configuration config, RMContext context, @@ -166,19 +140,45 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic } rmContext = context; scheduler = (CapacityScheduler) sched; - maxIgnoredOverCapacity = config.getDouble(MAX_IGNORED_OVER_CAPACITY, 0.1); - naturalTerminationFactor = - config.getDouble(NATURAL_TERMINATION_FACTOR, 0.2); - maxWaitTime = config.getLong(WAIT_TIME_BEFORE_KILL, 15000); - monitoringInterval = config.getLong(MONITORING_INTERVAL, 3000); - percentageClusterPreemptionAllowed = - config.getFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 0.1); - observeOnly = config.getBoolean(OBSERVE_ONLY, false); + CapacitySchedulerConfiguration csConfig = scheduler.getConfiguration(); + + maxIgnoredOverCapacity = csConfig.getDouble( + CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY, + CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY); + + naturalTerminationFactor = csConfig.getDouble( + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR); + + maxWaitTime = csConfig.getLong( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL); + + monitoringInterval = csConfig.getLong( + CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MONITORING_INTERVAL); + + percentageClusterPreemptionAllowed = csConfig.getFloat( + CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, + CapacitySchedulerConfiguration.DEFAULT_TOTAL_PREEMPTION_PER_ROUND); + + observeOnly = csConfig.getBoolean( + CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY, + CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_OBSERVE_ONLY); + + lazyPreempionEnabled = csConfig.getBoolean( + CapacitySchedulerConfiguration.LAZY_PREEMPTION_ENALBED, + CapacitySchedulerConfiguration.DEFAULT_LAZY_PREEMPTION_ENABLED); + rc = scheduler.getResourceCalculator(); nlm = scheduler.getRMContext().getNodeLabelManager(); + + // initialize candidates preemption selection policies + candidatesSelectionPolicies.add( + new FifoCandidatesSelector(this)); } - @VisibleForTesting + @Override public ResourceCalculator getResourceCalculator() { return rc; } @@ -191,42 +191,37 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic } @SuppressWarnings("unchecked") - private void cleanupStaledKillableContainers(Resource cluster, - Set leafQueueNames) { - for (String q : leafQueueNames) { - for (TempQueuePerPartition tq : getQueuePartitions(q)) { - // When queue's used - killable <= guaranteed and, killable > 0, we need - // to check if any of killable containers needs to be reverted - if (Resources.lessThanOrEqual(rc, cluster, - Resources.subtract(tq.current, tq.killable), tq.idealAssigned) - && Resources.greaterThan(rc, cluster, tq.killable, Resources.none())) { - // How many killable resources need to be reverted - // need-to-revert = already-marked-killable - (current - ideal) - Resource toBeRevertedFromKillable = Resources.subtract(tq.killable, - Resources.subtract(tq.current, tq.idealAssigned)); - - Resource alreadyReverted = Resources.createResource(0); - - for (RMContainer c : preemptableEntities.get(q).getKillableContainers( - tq.partition).values()) { - if (Resources.greaterThanOrEqual(rc, cluster, alreadyReverted, - toBeRevertedFromKillable)) { - break; - } - - if (Resources.greaterThan(rc, cluster, - Resources.add(alreadyReverted, c.getAllocatedResource()), - toBeRevertedFromKillable)) { - continue; - } else { - // This container need to be marked to unkillable - Resources.addTo(alreadyReverted, c.getAllocatedResource()); - rmContext.getDispatcher().getEventHandler().handle( - new ContainerPreemptEvent(c.getApplicationAttemptId(), c, - SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE)); - } + private void preemptOrkillSelectedContainerAfterWait( + Map> selectedCandidates) { + // preempt (or kill) the selected containers + for (Map.Entry> e : selectedCandidates + .entrySet()) { + ApplicationAttemptId appAttemptId = e.getKey(); + if (LOG.isDebugEnabled()) { + LOG.debug("Send to scheduler: in app=" + appAttemptId + + " #containers-to-be-preemptionCandidates=" + e.getValue().size()); + } + for (RMContainer container : e.getValue()) { + // if we tried to preempt this for more than maxWaitTime + if (preemptionCandidates.get(container) != null + && preemptionCandidates.get(container) + maxWaitTime < clock + .getTime()) { + // kill it + rmContext.getDispatcher().getEventHandler().handle( + new ContainerPreemptEvent(appAttemptId, container, + SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE)); + preemptionCandidates.remove(container); + } else { + if (preemptionCandidates.get(container) != null) { + // We already updated the information to scheduler earlier, we need + // not have to raise another event. + continue; } - + //otherwise just send preemption events + rmContext.getDispatcher().getEventHandler().handle( + new ContainerPreemptEvent(appAttemptId, container, + SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION)); + preemptionCandidates.put(container, clock.getTime()); } } } @@ -234,11 +229,11 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic private void syncKillableContainersFromScheduler() { // sync preemptable entities from scheduler - preemptableEntities = - scheduler.getPreemptionManager().getShallowCopyOfPreemptableEntities(); + preemptableQueues = + scheduler.getPreemptionManager().getShallowCopyOfPreemptableQueues(); killableContainers = new HashSet<>(); - for (Map.Entry entry : preemptableEntities + for (Map.Entry entry : preemptableQueues .entrySet()) { PreemptableQueue entity = entry.getValue(); for (Map map : entity.getKillableContainers() @@ -247,9 +242,34 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic } } } + + private void cleanupStaledPreemptionCandidates() { + // Keep the preemptionCandidates list clean + for (Iterator i = preemptionCandidates.keySet().iterator(); + i.hasNext(); ) { + RMContainer id = i.next(); + // garbage collect containers that are irrelevant for preemption + if (preemptionCandidates.get(id) + 2 * maxWaitTime < clock.getTime()) { + i.remove(); + } + } + } + + private Set getLeafQueueNames(TempQueuePerPartition q) { + if (q.children == null || q.children.isEmpty()) { + return ImmutableSet.of(q.queueName); + } + + Set leafQueueNames = new HashSet<>(); + for (TempQueuePerPartition child : q.children) { + leafQueueNames.addAll(getLeafQueueNames(child)); + } + + return leafQueueNames; + } /** - * This method selects and tracks containers to be preempted. If a container + * This method selects and tracks containers to be preemptionCandidates. If a container * is in the target list for more than maxWaitTime it is killed. * * @param root the root of the CapacityScheduler queue hierarchy @@ -258,13 +278,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic @SuppressWarnings("unchecked") private void containerBasedPreemptOrKill(CSQueue root, Resource clusterResources) { - // All partitions to look at - Set allPartitions = new HashSet<>(); - allPartitions.addAll(scheduler.getRMContext() - .getNodeLabelManager().getClusterNodeLabelNames()); - allPartitions.add(RMNodeLabelsManager.NO_LABEL); + // Sync killable containers from scheduler when lazy preemption enabled + if (lazyPreempionEnabled) { + syncKillableContainersFromScheduler(); + } - syncKillableContainersFromScheduler(); + // All partitions to look at + Set partitions = new HashSet<>(); + partitions.addAll(scheduler.getRMContext() + .getNodeLabelManager().getClusterNodeLabelNames()); + partitions.add(RMNodeLabelsManager.NO_LABEL); + this.allPartitions = ImmutableSet.copyOf(partitions); // extract a summary of the queues from scheduler synchronized (scheduler) { @@ -277,30 +301,22 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic } } + this.leafQueueNames = ImmutableSet.copyOf(getLeafQueueNames( + getQueueByPartition(CapacitySchedulerConfiguration.ROOT, + RMNodeLabelsManager.NO_LABEL))); + // compute total preemption allowed Resource totalPreemptionAllowed = Resources.multiply(clusterResources, percentageClusterPreemptionAllowed); - Set leafQueueNames = null; - for (String partition : allPartitions) { - TempQueuePerPartition tRoot = - getQueueByPartition(CapacitySchedulerConfiguration.ROOT, partition); - // compute the ideal distribution of resources among queues - // updates cloned queues state accordingly - tRoot.idealAssigned = tRoot.guaranteed; - - leafQueueNames = - recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed); - } - - // remove containers from killable list when we want to preempt less resources - // from queue. - cleanupStaledKillableContainers(clusterResources, leafQueueNames); - - // based on ideal allocation select containers to be preempted from each + // based on ideal allocation select containers to be preemptionCandidates from each // queue and each application - Map> toPreempt = - getContainersToPreempt(leafQueueNames, clusterResources); + Map> toPreempt = null; + for (PreemptionCandidatesSelector selector : + candidatesSelectionPolicies) { + toPreempt = selector.selectCandidates(toPreempt, + clusterResources, totalPreemptionAllowed); + } if (LOG.isDebugEnabled()) { logToCSV(new ArrayList<>(leafQueueNames)); @@ -311,582 +327,19 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic return; } + // TODO: need consider revert killable containers when no more demandings. + // Since we could have several selectors to make decisions concurrently. + // So computed ideal-allocation varies between different selectors. + // + // We may need to "score" killable containers and revert the most preferred + // containers. The bottom line is, we shouldn't preempt a queue which is already + // below its guaranteed resource. + // preempt (or kill) the selected containers - for (Map.Entry> e - : toPreempt.entrySet()) { - ApplicationAttemptId appAttemptId = e.getKey(); - if (LOG.isDebugEnabled()) { - LOG.debug("Send to scheduler: in app=" + appAttemptId - + " #containers-to-be-preempted=" + e.getValue().size()); - } - for (RMContainer container : e.getValue()) { - // if we tried to preempt this for more than maxWaitTime - if (preempted.get(container) != null && - preempted.get(container) + maxWaitTime < clock.getTime()) { - // mark container killable - rmContext.getDispatcher().getEventHandler().handle( - new ContainerPreemptEvent(appAttemptId, container, - SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE)); - preempted.remove(container); - } else { - if (preempted.get(container) != null) { - // We already updated the information to scheduler earlier, we need - // not have to raise another event. - continue; - } - //otherwise just send preemption events - rmContext.getDispatcher().getEventHandler().handle( - new ContainerPreemptEvent(appAttemptId, container, - SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION)); - preempted.put(container, clock.getTime()); - } - } - } + preemptOrkillSelectedContainerAfterWait(toPreempt); - // Keep the preempted list clean - for (Iterator i = preempted.keySet().iterator(); i.hasNext();){ - RMContainer id = i.next(); - // garbage collect containers that are irrelevant for preemption - if (preempted.get(id) + 2 * maxWaitTime < clock.getTime()) { - i.remove(); - } - } - } - - /** - * This method recursively computes the ideal assignment of resources to each - * level of the hierarchy. This ensures that leafs that are over-capacity but - * with parents within capacity will not be preempted. Preemptions are allowed - * within each subtree according to local over/under capacity. - * - * @param root the root of the cloned queue hierachy - * @param totalPreemptionAllowed maximum amount of preemption allowed - * @return a list of leaf queues updated with preemption targets - */ - private Set recursivelyComputeIdealAssignment( - TempQueuePerPartition root, Resource totalPreemptionAllowed) { - Set leafQueueNames = new HashSet<>(); - if (root.getChildren() != null && - root.getChildren().size() > 0) { - // compute ideal distribution at this level - computeIdealResourceDistribution(rc, root.getChildren(), - totalPreemptionAllowed, root.idealAssigned); - // compute recursively for lower levels and build list of leafs - for(TempQueuePerPartition t : root.getChildren()) { - leafQueueNames.addAll(recursivelyComputeIdealAssignment(t, - totalPreemptionAllowed)); - } - } else { - // we are in a leaf nothing to do, just return yourself - return ImmutableSet.of(root.queueName); - } - return leafQueueNames; - } - - /** - * This method computes (for a single level in the tree, passed as a {@code - * List}) the ideal assignment of resources. This is done - * recursively to allocate capacity fairly across all queues with pending - * demands. It terminates when no resources are left to assign, or when all - * demand is satisfied. - * - * @param rc resource calculator - * @param queues a list of cloned queues to be assigned capacity to (this is - * an out param) - * @param totalPreemptionAllowed total amount of preemption we allow - * @param tot_guarant the amount of capacity assigned to this pool of queues - */ - private void computeIdealResourceDistribution(ResourceCalculator rc, - List queues, Resource totalPreemptionAllowed, - Resource tot_guarant) { - - // qAlloc tracks currently active queues (will decrease progressively as - // demand is met) - List qAlloc = new ArrayList<>(queues); - // unassigned tracks how much resources are still to assign, initialized - // with the total capacity for this set of queues - Resource unassigned = Resources.clone(tot_guarant); - - // group queues based on whether they have non-zero guaranteed capacity - Set nonZeroGuarQueues = new HashSet<>(); - Set zeroGuarQueues = new HashSet<>(); - - for (TempQueuePerPartition q : qAlloc) { - if (Resources - .greaterThan(rc, tot_guarant, q.guaranteed, Resources.none())) { - nonZeroGuarQueues.add(q); - } else { - zeroGuarQueues.add(q); - } - } - - // first compute the allocation as a fixpoint based on guaranteed capacity - computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned, - false); - - // if any capacity is left unassigned, distributed among zero-guarantee - // queues uniformly (i.e., not based on guaranteed capacity, as this is zero) - if (!zeroGuarQueues.isEmpty() - && Resources.greaterThan(rc, tot_guarant, unassigned, Resources.none())) { - computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned, - true); - } - - // based on ideal assignment computed above and current assignment we derive - // how much preemption is required overall - Resource totPreemptionNeeded = Resource.newInstance(0, 0); - for (TempQueuePerPartition t:queues) { - if (Resources.greaterThan(rc, tot_guarant, t.current, t.idealAssigned)) { - Resources.addTo(totPreemptionNeeded, - Resources.subtract(t.current, t.idealAssigned)); - } - } - - // if we need to preempt more than is allowed, compute a factor (0 qAlloc, - Resource unassigned, boolean ignoreGuarantee) { - // Prior to assigning the unused resources, process each queue as follows: - // If current > guaranteed, idealAssigned = guaranteed + untouchable extra - // Else idealAssigned = current; - // Subtract idealAssigned resources from unassigned. - // If the queue has all of its needs met (that is, if - // idealAssigned >= current + pending), remove it from consideration. - // Sort queues from most under-guaranteed to most over-guaranteed. - TQComparator tqComparator = new TQComparator(rc, tot_guarant); - PriorityQueue orderedByNeed = new PriorityQueue<>(10, - tqComparator); - for (Iterator i = qAlloc.iterator(); i.hasNext();) { - TempQueuePerPartition q = i.next(); - if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) { - q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra); - } else { - q.idealAssigned = Resources.clone(q.current); - } - Resources.subtractFrom(unassigned, q.idealAssigned); - // If idealAssigned < (current + pending), q needs more resources, so - // add it to the list of underserved queues, ordered by need. - Resource curPlusPend = Resources.add(q.current, q.pending); - if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) { - orderedByNeed.add(q); - } - } - - //assign all cluster resources until no more demand, or no resources are left - while (!orderedByNeed.isEmpty() - && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) { - Resource wQassigned = Resource.newInstance(0, 0); - // we compute normalizedGuarantees capacity based on currently active - // queues - resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee); - - // For each underserved queue (or set of queues if multiple are equally - // underserved), offer its share of the unassigned resources based on its - // normalized guarantee. After the offer, if the queue is not satisfied, - // place it back in the ordered list of queues, recalculating its place - // in the order of most under-guaranteed to most over-guaranteed. In this - // way, the most underserved queue(s) are always given resources first. - Collection underserved = - getMostUnderservedQueues(orderedByNeed, tqComparator); - for (Iterator i = underserved.iterator(); i - .hasNext();) { - TempQueuePerPartition sub = i.next(); - Resource wQavail = Resources.multiplyAndNormalizeUp(rc, - unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1)); - Resource wQidle = sub.offer(wQavail, rc, tot_guarant); - Resource wQdone = Resources.subtract(wQavail, wQidle); - - if (Resources.greaterThan(rc, tot_guarant, - wQdone, Resources.none())) { - // The queue is still asking for more. Put it back in the priority - // queue, recalculating its order based on need. - orderedByNeed.add(sub); - } - Resources.addTo(wQassigned, wQdone); - } - Resources.subtractFrom(unassigned, wQassigned); - } - } - - // Take the most underserved TempQueue (the one on the head). Collect and - // return the list of all queues that have the same idealAssigned - // percentage of guaranteed. - protected Collection getMostUnderservedQueues( - PriorityQueue orderedByNeed, TQComparator tqComparator) { - ArrayList underserved = new ArrayList<>(); - while (!orderedByNeed.isEmpty()) { - TempQueuePerPartition q1 = orderedByNeed.remove(); - underserved.add(q1); - TempQueuePerPartition q2 = orderedByNeed.peek(); - // q1's pct of guaranteed won't be larger than q2's. If it's less, then - // return what has already been collected. Otherwise, q1's pct of - // guaranteed == that of q2, so add q2 to underserved list during the - // next pass. - if (q2 == null || tqComparator.compare(q1,q2) < 0) { - return underserved; - } - } - return underserved; - } - - /** - * Computes a normalizedGuaranteed capacity based on active queues - * @param rc resource calculator - * @param clusterResource the total amount of resources in the cluster - * @param queues the list of queues to consider - */ - private void resetCapacity(ResourceCalculator rc, Resource clusterResource, - Collection queues, boolean ignoreGuar) { - Resource activeCap = Resource.newInstance(0, 0); - - if (ignoreGuar) { - for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = 1.0f / queues.size(); - } - } else { - for (TempQueuePerPartition q : queues) { - Resources.addTo(activeCap, q.guaranteed); - } - for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = Resources.divide(rc, clusterResource, - q.guaranteed, activeCap); - } - } - } - - private String getPartitionByRMContainer(RMContainer rmContainer) { - return scheduler.getSchedulerNode(rmContainer.getAllocatedNode()) - .getPartition(); - } - - /** - * Return should we preempt rmContainer. If we should, deduct from - * resourceToObtainByPartition - */ - private boolean tryPreemptContainerAndDeductResToObtain( - Map resourceToObtainByPartitions, - RMContainer rmContainer, Resource clusterResource, - Map> preemptMap) { - ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId(); - - // We will not account resource of a container twice or more - if (preemptMapContains(preemptMap, attemptId, rmContainer)) { - return false; - } - - String nodePartition = getPartitionByRMContainer(rmContainer); - Resource toObtainByPartition = - resourceToObtainByPartitions.get(nodePartition); - - if (null != toObtainByPartition - && Resources.greaterThan(rc, clusterResource, toObtainByPartition, - Resources.none())) { - Resources.subtractFrom(toObtainByPartition, - rmContainer.getAllocatedResource()); - // When we have no more resource need to obtain, remove from map. - if (Resources.lessThanOrEqual(rc, clusterResource, toObtainByPartition, - Resources.none())) { - resourceToObtainByPartitions.remove(nodePartition); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Marked container=" + rmContainer.getContainerId() - + " in partition=" + nodePartition + " will be preempted"); - } - // Add to preemptMap - addToPreemptMap(preemptMap, attemptId, rmContainer); - return true; - } - - return false; - } - - private boolean preemptMapContains( - Map> preemptMap, - ApplicationAttemptId attemptId, RMContainer rmContainer) { - Set rmContainers; - if (null == (rmContainers = preemptMap.get(attemptId))) { - return false; - } - return rmContainers.contains(rmContainer); - } - - private void addToPreemptMap( - Map> preemptMap, - ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) { - Set set; - if (null == (set = preemptMap.get(appAttemptId))) { - set = new HashSet<>(); - preemptMap.put(appAttemptId, set); - } - set.add(containerToPreempt); - } - - /** - * Based a resource preemption target drop reservations of containers and - * if necessary select containers for preemption from applications in each - * over-capacity queue. It uses {@link #NATURAL_TERMINATION_FACTOR} to - * account for containers that will naturally complete. - * - * @param leafQueueNames set of leaf queues to preempt from - * @param clusterResource total amount of cluster resources - * @return a map of applciationID to set of containers to preempt - */ - private Map> getContainersToPreempt( - Set leafQueueNames, Resource clusterResource) { - - Map> preemptMap = - new HashMap<>(); - List skippedAMContainerlist = new ArrayList<>(); - - // Loop all leaf queues - for (String queueName : leafQueueNames) { - // check if preemption disabled for the queue - if (getQueueByPartition(queueName, - RMNodeLabelsManager.NO_LABEL).preemptionDisabled) { - if (LOG.isDebugEnabled()) { - LOG.debug("skipping from queue=" + queueName - + " because it's a non-preemptable queue"); - } - continue; - } - - // compute resToObtainByPartition considered inter-queue preemption - LeafQueue leafQueue = null; - - Map resToObtainByPartition = - new HashMap<>(); - for (TempQueuePerPartition qT : getQueuePartitions(queueName)) { - leafQueue = qT.leafQueue; - // we act only if we are violating balance by more than - // maxIgnoredOverCapacity - if (Resources.greaterThan(rc, clusterResource, qT.current, - Resources.multiply(qT.guaranteed, 1.0 + maxIgnoredOverCapacity))) { - // we introduce a dampening factor naturalTerminationFactor that - // accounts for natural termination of containers - Resource resToObtain = - Resources.multiply(qT.toBePreempted, naturalTerminationFactor); - // Only add resToObtain when it >= 0 - if (Resources.greaterThan(rc, clusterResource, resToObtain, - Resources.none())) { - resToObtainByPartition.put(qT.partition, resToObtain); - if (LOG.isDebugEnabled()) { - LOG.debug("Queue=" + queueName + " partition=" + qT.partition - + " resource-to-obtain=" + resToObtain); - } - } - qT.actuallyPreempted = Resources.clone(resToObtain); - } else { - qT.actuallyPreempted = Resources.none(); - } - } - - synchronized (leafQueue) { - // go through all ignore-partition-exclusivity containers first to make - // sure such containers will be preempted first - Map> ignorePartitionExclusivityContainers = - leafQueue.getIgnoreExclusivityRMContainers(); - for (String partition : resToObtainByPartition.keySet()) { - if (ignorePartitionExclusivityContainers.containsKey(partition)) { - TreeSet rmContainers = - ignorePartitionExclusivityContainers.get(partition); - // We will check container from reverse order, so latter submitted - // application's containers will be preempted first. - for (RMContainer c : rmContainers.descendingSet()) { - boolean preempted = - tryPreemptContainerAndDeductResToObtain( - resToObtainByPartition, c, clusterResource, preemptMap); - if (!preempted) { - break; - } - } - } - } - - // preempt other containers - Resource skippedAMSize = Resource.newInstance(0, 0); - Iterator desc = - leafQueue.getOrderingPolicy().getPreemptionIterator(); - while (desc.hasNext()) { - FiCaSchedulerApp fc = desc.next(); - // When we complete preempt from one partition, we will remove from - // resToObtainByPartition, so when it becomes empty, we can get no - // more preemption is needed - if (resToObtainByPartition.isEmpty()) { - break; - } - - preemptFrom(fc, clusterResource, resToObtainByPartition, - skippedAMContainerlist, skippedAMSize, preemptMap); - } - - // Can try preempting AMContainers (still saving atmost - // maxAMCapacityForThisQueue AMResource's) if more resources are - // required to be preempted from this Queue. - Resource maxAMCapacityForThisQueue = Resources.multiply( - Resources.multiply(clusterResource, - leafQueue.getAbsoluteCapacity()), - leafQueue.getMaxAMResourcePerQueuePercent()); - - preemptAMContainers(clusterResource, preemptMap, skippedAMContainerlist, - resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue); - } - } - - return preemptMap; - } - - /** - * As more resources are needed for preemption, saved AMContainers has to be - * rescanned. Such AMContainers can be preempted based on resToObtain, but - * maxAMCapacityForThisQueue resources will be still retained. - * - * @param clusterResource - * @param preemptMap - * @param skippedAMContainerlist - * @param skippedAMSize - * @param maxAMCapacityForThisQueue - */ - private void preemptAMContainers(Resource clusterResource, - Map> preemptMap, - List skippedAMContainerlist, - Map resToObtainByPartition, Resource skippedAMSize, - Resource maxAMCapacityForThisQueue) { - for (RMContainer c : skippedAMContainerlist) { - // Got required amount of resources for preemption, can stop now - if (resToObtainByPartition.isEmpty()) { - break; - } - // Once skippedAMSize reaches down to maxAMCapacityForThisQueue, - // container selection iteration for preemption will be stopped. - if (Resources.lessThanOrEqual(rc, clusterResource, skippedAMSize, - maxAMCapacityForThisQueue)) { - break; - } - - boolean preempted = - tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, - clusterResource, preemptMap); - if (preempted) { - Resources.subtractFrom(skippedAMSize, c.getAllocatedResource()); - } - } - skippedAMContainerlist.clear(); - } - - /** - * Given a target preemption for a specific application, select containers - * to preempt (after unreserving all reservation for that app). - */ - @SuppressWarnings("unchecked") - private void preemptFrom(FiCaSchedulerApp app, - Resource clusterResource, Map resToObtainByPartition, - List skippedAMContainerlist, Resource skippedAMSize, - Map> preemptMap) { - ApplicationAttemptId appId = app.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Looking at application=" + app.getApplicationAttemptId() - + " resourceToObtain=" + resToObtainByPartition); - } - - // first drop reserved containers towards rsrcPreempt - List reservedContainers = - new ArrayList<>(app.getReservedContainers()); - for (RMContainer c : reservedContainers) { - if (resToObtainByPartition.isEmpty()) { - return; - } - - // Try to preempt this container - tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, - clusterResource, preemptMap); - - if (!observeOnly) { - rmContext.getDispatcher().getEventHandler().handle( - new ContainerPreemptEvent( - appId, c, SchedulerEventType.KILL_RESERVED_CONTAINER)); - } - } - - // if more resources are to be freed go through all live containers in - // reverse priority and reverse allocation order and mark them for - // preemption - List liveContainers = new ArrayList<>(app.getLiveContainers()); - - sortContainers(liveContainers); - - for (RMContainer c : liveContainers) { - if (resToObtainByPartition.isEmpty()) { - return; - } - - // Skip AM Container from preemption for now. - if (c.isAMContainer()) { - skippedAMContainerlist.add(c); - Resources.addTo(skippedAMSize, c.getAllocatedResource()); - continue; - } - - // Skip already marked to killable containers - if (killableContainers.contains(c.getContainerId())) { - continue; - } - - // Try to preempt this container - tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c, - clusterResource, preemptMap); - } - } - - /** - * Compare by reversed priority order first, and then reversed containerId - * order - * @param containers - */ - @VisibleForTesting - static void sortContainers(List containers){ - Collections.sort(containers, new Comparator() { - @Override - public int compare(RMContainer a, RMContainer b) { - Comparator c = new org.apache.hadoop.yarn.server - .resourcemanager.resource.Priority.Comparator(); - int priorityComp = c.compare(b.getContainer().getPriority(), - a.getContainer().getPriority()); - if (priorityComp != 0) { - return priorityComp; - } - return b.getContainerId().compareTo(a.getContainerId()); - } - }); + // cleanup staled preemption candidates + cleanupStaledPreemptionCandidates(); } @Override @@ -901,7 +354,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic @VisibleForTesting public Map getToPreemptContainers() { - return preempted; + return preemptionCandidates; } /** @@ -929,8 +382,8 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic Resource guaranteed = Resources.multiply(partitionResource, absCap); Resource maxCapacity = Resources.multiply(partitionResource, absMaxCap); Resource killable = Resources.none(); - if (null != preemptableEntities.get(queueName)) { - killable = preemptableEntities.get(queueName) + if (null != preemptableQueues.get(queueName)) { + killable = preemptableQueues.get(queueName) .getKillableResource(partitionToLookAt); } @@ -1023,9 +476,10 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic /** * Get queue partition by given queueName and partitionName */ - private TempQueuePerPartition getQueueByPartition(String queueName, + @Override + public TempQueuePerPartition getQueueByPartition(String queueName, String partition) { - Map partitionToQueues = null; + Map partitionToQueues; if (null == (partitionToQueues = queueToPartitions.get(queueName))) { return null; } @@ -1035,180 +489,56 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic /** * Get all queue partitions by given queueName */ - private Collection getQueuePartitions(String queueName) { + @Override + public Collection getQueuePartitions(String queueName) { if (!queueToPartitions.containsKey(queueName)) { return null; } return queueToPartitions.get(queueName).values(); } - /** - * Temporary data-structure tracking resource availability, pending resource - * need, current utilization. This is per-queue-per-partition data structure - */ - static class TempQueuePerPartition { - final String queueName; - final Resource current; - final Resource pending; - final Resource guaranteed; - final Resource maxCapacity; - final String partition; - final Resource killable; - Resource idealAssigned; - Resource toBePreempted; - - // For logging purpose - Resource actuallyPreempted; - Resource untouchableExtra; - Resource preemptableExtra; - - double normalizedGuarantee; - - final ArrayList children; - LeafQueue leafQueue; - boolean preemptionDisabled; - - TempQueuePerPartition(String queueName, Resource current, Resource pending, - Resource guaranteed, Resource maxCapacity, boolean preemptionDisabled, - String partition, Resource killableResource) { - this.queueName = queueName; - this.current = current; - this.pending = pending; - this.guaranteed = guaranteed; - this.maxCapacity = maxCapacity; - this.idealAssigned = Resource.newInstance(0, 0); - this.actuallyPreempted = Resource.newInstance(0, 0); - this.toBePreempted = Resource.newInstance(0, 0); - this.normalizedGuarantee = Float.NaN; - this.children = new ArrayList<>(); - this.untouchableExtra = Resource.newInstance(0, 0); - this.preemptableExtra = Resource.newInstance(0, 0); - this.preemptionDisabled = preemptionDisabled; - this.partition = partition; - this.killable = killableResource; - } - - public void setLeafQueue(LeafQueue l){ - assert children.size() == 0; - this.leafQueue = l; - } - - /** - * When adding a child we also aggregate its pending resource needs. - * @param q the child queue to add to this queue - */ - public void addChild(TempQueuePerPartition q) { - assert leafQueue == null; - children.add(q); - Resources.addTo(pending, q.pending); - } - - public ArrayList getChildren(){ - return children; - } - - // This function "accepts" all the resources it can (pending) and return - // the unused ones - Resource offer(Resource avail, ResourceCalculator rc, - Resource clusterResource) { - Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( - Resources.subtract(maxCapacity, idealAssigned), - Resource.newInstance(0, 0)); - // remain = avail - min(avail, (max - assigned), (current + pending - assigned)) - Resource accepted = - Resources.min(rc, clusterResource, - absMaxCapIdealAssignedDelta, - Resources.min(rc, clusterResource, avail, Resources.subtract( - Resources.add(current, pending), idealAssigned))); - Resource remain = Resources.subtract(avail, accepted); - Resources.addTo(idealAssigned, accepted); - return remain; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(" NAME: " + queueName) - .append(" CUR: ").append(current) - .append(" PEN: ").append(pending) - .append(" GAR: ").append(guaranteed) - .append(" NORM: ").append(normalizedGuarantee) - .append(" IDEAL_ASSIGNED: ").append(idealAssigned) - .append(" IDEAL_PREEMPT: ").append(toBePreempted) - .append(" ACTUAL_PREEMPT: ").append(actuallyPreempted) - .append(" UNTOUCHABLE: ").append(untouchableExtra) - .append(" PREEMPTABLE: ").append(preemptableExtra) - .append("\n"); - - return sb.toString(); - } - - public void assignPreemption(float scalingFactor, - ResourceCalculator rc, Resource clusterResource) { - if (Resources.greaterThan(rc, clusterResource, - Resources.subtract(current, killable), idealAssigned)) { - toBePreempted = Resources.multiply(Resources.subtract( - Resources.subtract(current, killable), idealAssigned), - scalingFactor); - } else { - toBePreempted = Resource.newInstance(0, 0); - } - } - - void appendLogString(StringBuilder sb) { - sb.append(queueName).append(", ") - .append(current.getMemory()).append(", ") - .append(current.getVirtualCores()).append(", ") - .append(pending.getMemory()).append(", ") - .append(pending.getVirtualCores()).append(", ") - .append(guaranteed.getMemory()).append(", ") - .append(guaranteed.getVirtualCores()).append(", ") - .append(idealAssigned.getMemory()).append(", ") - .append(idealAssigned.getVirtualCores()).append(", ") - .append(toBePreempted.getMemory()).append(", ") - .append(toBePreempted.getVirtualCores() ).append(", ") - .append(actuallyPreempted.getMemory()).append(", ") - .append(actuallyPreempted.getVirtualCores()); - } - + @Override + public CapacityScheduler getScheduler() { + return scheduler; } - static class TQComparator implements Comparator { - private ResourceCalculator rc; - private Resource clusterRes; + @Override + public RMContext getRMContext() { + return rmContext; + } - TQComparator(ResourceCalculator rc, Resource clusterRes) { - this.rc = rc; - this.clusterRes = clusterRes; - } + @Override + public boolean isObserveOnly() { + return observeOnly; + } - @Override - public int compare(TempQueuePerPartition tq1, TempQueuePerPartition tq2) { - if (getIdealPctOfGuaranteed(tq1) < getIdealPctOfGuaranteed(tq2)) { - return -1; - } - if (getIdealPctOfGuaranteed(tq1) > getIdealPctOfGuaranteed(tq2)) { - return 1; - } - return 0; - } + @Override + public Set getKillableContainers() { + return killableContainers; + } - // Calculates idealAssigned / guaranteed - // TempQueues with 0 guarantees are always considered the most over - // capacity and therefore considered last for resources. - private double getIdealPctOfGuaranteed(TempQueuePerPartition q) { - double pctOver = Integer.MAX_VALUE; - if (q != null && Resources.greaterThan( - rc, clusterRes, q.guaranteed, Resources.none())) { - pctOver = - Resources.divide(rc, clusterRes, q.idealAssigned, q.guaranteed); - } - return (pctOver); - } + @Override + public double getMaxIgnoreOverCapacity() { + return maxIgnoredOverCapacity; + } + + @Override + public double getNaturalTerminationFactor() { + return naturalTerminationFactor; + } + + @Override + public Set getLeafQueueNames() { + return leafQueueNames; + } + + @Override + public Set getAllPartitions() { + return allPartitions; } @VisibleForTesting - public Map> getQueuePartitions() { + Map> getQueuePartitions() { return queueToPartitions; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java new file mode 100644 index 00000000000..8b01a73cd9a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.ArrayList; + +/** + * Temporary data-structure tracking resource availability, pending resource + * need, current utilization. This is per-queue-per-partition data structure + */ +public class TempQueuePerPartition { + // Following fields are copied from scheduler + final String queueName; + final Resource current; + final Resource pending; + final Resource guaranteed; + final Resource maxCapacity; + final Resource killable; + final String partition; + + // Following fields are setted and used by candidate selection policies + Resource idealAssigned; + Resource toBePreempted; + Resource untouchableExtra; + Resource preemptableExtra; + // For logging purpose + Resource actuallyToBePreempted; + + double normalizedGuarantee; + + final ArrayList children; + LeafQueue leafQueue; + boolean preemptionDisabled; + + TempQueuePerPartition(String queueName, Resource current, Resource pending, + Resource guaranteed, Resource maxCapacity, boolean preemptionDisabled, + String partition, Resource killable) { + this.queueName = queueName; + this.current = current; + this.pending = pending; + this.guaranteed = guaranteed; + this.maxCapacity = maxCapacity; + this.idealAssigned = Resource.newInstance(0, 0); + this.actuallyToBePreempted = Resource.newInstance(0, 0); + this.toBePreempted = Resource.newInstance(0, 0); + this.normalizedGuarantee = Float.NaN; + this.children = new ArrayList<>(); + this.untouchableExtra = Resource.newInstance(0, 0); + this.preemptableExtra = Resource.newInstance(0, 0); + this.preemptionDisabled = preemptionDisabled; + this.partition = partition; + this.killable = killable; + } + + public void setLeafQueue(LeafQueue l) { + assert children.size() == 0; + this.leafQueue = l; + } + + /** + * When adding a child we also aggregate its pending resource needs. + * @param q the child queue to add to this queue + */ + public void addChild(TempQueuePerPartition q) { + assert leafQueue == null; + children.add(q); + Resources.addTo(pending, q.pending); + } + + public ArrayList getChildren(){ + return children; + } + + // This function "accepts" all the resources it can (pending) and return + // the unused ones + Resource offer(Resource avail, ResourceCalculator rc, + Resource clusterResource) { + Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( + Resources.subtract(maxCapacity, idealAssigned), + Resource.newInstance(0, 0)); + // remain = avail - min(avail, (max - assigned), (current + pending - assigned)) + Resource accepted = + Resources.min(rc, clusterResource, + absMaxCapIdealAssignedDelta, + Resources.min(rc, clusterResource, avail, Resources.subtract( + Resources.add(current, pending), idealAssigned))); + Resource remain = Resources.subtract(avail, accepted); + Resources.addTo(idealAssigned, accepted); + return remain; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(" NAME: " + queueName) + .append(" CUR: ").append(current) + .append(" PEN: ").append(pending) + .append(" GAR: ").append(guaranteed) + .append(" NORM: ").append(normalizedGuarantee) + .append(" IDEAL_ASSIGNED: ").append(idealAssigned) + .append(" IDEAL_PREEMPT: ").append(toBePreempted) + .append(" ACTUAL_PREEMPT: ").append(actuallyToBePreempted) + .append(" UNTOUCHABLE: ").append(untouchableExtra) + .append(" PREEMPTABLE: ").append(preemptableExtra) + .append("\n"); + + return sb.toString(); + } + + public void assignPreemption(float scalingFactor, ResourceCalculator rc, + Resource clusterResource) { + if (Resources.greaterThan(rc, clusterResource, + Resources.subtract(current, killable), idealAssigned)) { + toBePreempted = Resources.multiply(Resources + .subtract(Resources.subtract(current, killable), idealAssigned), + scalingFactor); + } else { + toBePreempted = Resource.newInstance(0, 0); + } + } + + void appendLogString(StringBuilder sb) { + sb.append(queueName).append(", ") + .append(current.getMemory()).append(", ") + .append(current.getVirtualCores()).append(", ") + .append(pending.getMemory()).append(", ") + .append(pending.getVirtualCores()).append(", ") + .append(guaranteed.getMemory()).append(", ") + .append(guaranteed.getVirtualCores()).append(", ") + .append(idealAssigned.getMemory()).append(", ") + .append(idealAssigned.getVirtualCores()).append(", ") + .append(toBePreempted.getMemory()).append(", ") + .append(toBePreempted.getVirtualCores() ).append(", ") + .append(actuallyToBePreempted.getMemory()).append(", ") + .append(actuallyToBePreempted.getVirtualCores()); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3729264c31c..88e39de1149 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1020,4 +1020,49 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public boolean getLazyPreemptionEnabled() { return getBoolean(LAZY_PREEMPTION_ENALBED, DEFAULT_LAZY_PREEMPTION_ENABLED); } + + /** If true, run the policy but do not affect the cluster with preemption and + * kill events. */ + public static final String PREEMPTION_OBSERVE_ONLY = + "yarn.resourcemanager.monitor.capacity.preemption.observe_only"; + public static final boolean DEFAULT_PREEMPTION_OBSERVE_ONLY = false; + + /** Time in milliseconds between invocations of this policy */ + public static final String PREEMPTION_MONITORING_INTERVAL = + "yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval"; + public static final long DEFAULT_PREEMPTION_MONITORING_INTERVAL = 3000L; + + /** Time in milliseconds between requesting a preemption from an application + * and killing the container. */ + public static final String PREEMPTION_WAIT_TIME_BEFORE_KILL = + "yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill"; + public static final long DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL = 15000L; + + /** Maximum percentage of resources preemptionCandidates in a single round. By + * controlling this value one can throttle the pace at which containers are + * reclaimed from the cluster. After computing the total desired preemption, + * the policy scales it back within this limit. */ + public static final String TOTAL_PREEMPTION_PER_ROUND = + "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round"; + public static final float DEFAULT_TOTAL_PREEMPTION_PER_ROUND = 0.1f; + + /** Maximum amount of resources above the target capacity ignored for + * preemption. This defines a deadzone around the target capacity that helps + * prevent thrashing and oscillations around the computed target balance. + * High values would slow the time to capacity and (absent natural + * completions) it might prevent convergence to guaranteed capacity. */ + public static final String PREEMPTION_MAX_IGNORED_OVER_CAPACITY = + "yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity"; + public static final float DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY = 0.1f; + /** + * Given a computed preemption target, account for containers naturally + * expiring and preempt only this percentage of the delta. This determines + * the rate of geometric convergence into the deadzone ({@link + * #PREEMPTION_MAX_IGNORED_OVER_CAPACITY}). For example, a termination factor of 0.5 + * will reclaim almost 95% of resources within 5 * {@link + * #PREEMPTION_WAIT_TIME_BEFORE_KILL}, even absent natural termination. */ + public static final String PREEMPTION_NATURAL_TERMINATION_FACTOR = + "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor"; + public static final float DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR = + 0.2f; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java index 19148d78cb9..fefb56ad730 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java @@ -86,12 +86,6 @@ public class PreemptableQueue { return res == null ? Resources.none() : res; } - @SuppressWarnings("unchecked") - public Map getKillableContainers(String partition) { - Map map = killableContainers.get(partition); - return map == null ? Collections.EMPTY_MAP : map; - } - public Map> getKillableContainers() { return killableContainers; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java index a9f02a54d04..76fcd4a9f81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java @@ -146,7 +146,7 @@ public class PreemptionManager { } } - public Map getShallowCopyOfPreemptableEntities() { + public Map getShallowCopyOfPreemptableQueues() { try { readLock.lock(); Map map = new HashMap<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index e9129de3c66..3db4782050b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -17,38 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MAX_IGNORED_OVER_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MONITORING_INTERVAL; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.OBSERVE_ONLY; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Deque; -import java.util.LinkedList; -import java.util.List; -import java.util.NavigableSet; -import java.util.Random; -import java.util.StringTokenizer; -import java.util.TreeSet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -63,7 +31,6 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; -import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TempQueuePerPartition; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -95,6 +62,32 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.NavigableSet; +import java.util.Random; +import java.util.StringTokenizer; +import java.util.TreeSet; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + public class TestProportionalCapacityPreemptionPolicy { static final long TS = 3141592653L; @@ -105,11 +98,10 @@ public class TestProportionalCapacityPreemptionPolicy { float setAMResourcePercent = 0.0f; Random rand = null; Clock mClock = null; - Configuration conf = null; + CapacitySchedulerConfiguration conf = null; CapacityScheduler mCS = null; RMContext rmContext = null; RMNodeLabelsManager lm = null; - CapacitySchedulerConfiguration schedConf = null; EventHandler mDisp = null; ResourceCalculator rc = new DefaultResourceCalculator(); Resource clusterResources = null; @@ -132,7 +124,7 @@ public class TestProportionalCapacityPreemptionPolicy { AMCONTAINER(0), CONTAINER(1), LABELEDCONTAINER(2); int value; - private priority(int value) { + priority(int value) { this.value = value; } @@ -146,12 +138,17 @@ public class TestProportionalCapacityPreemptionPolicy { @Before @SuppressWarnings("unchecked") public void setup() { - conf = new Configuration(false); - conf.setLong(WAIT_TIME_BEFORE_KILL, 10000); - conf.setLong(MONITORING_INTERVAL, 3000); + conf = new CapacitySchedulerConfiguration(new Configuration(false)); + conf.setLong( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000); + conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + 3000); // report "ideal" preempt - conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0); - conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0); + conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, + 1.0f); + conf.setFloat( + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + 1.0f); conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName()); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); @@ -164,8 +161,7 @@ public class TestProportionalCapacityPreemptionPolicy { mCS = mock(CapacityScheduler.class); when(mCS.getResourceCalculator()).thenReturn(rc); lm = mock(RMNodeLabelsManager.class); - schedConf = new CapacitySchedulerConfiguration(); - when(mCS.getConfiguration()).thenReturn(schedConf); + when(mCS.getConfiguration()).thenReturn(conf); rmContext = mock(RMContext.class); when(mCS.getRMContext()).thenReturn(rmContext); when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager()); @@ -271,7 +267,9 @@ public class TestProportionalCapacityPreemptionPolicy { { -1, 1, 1, 1 }, // req granularity { 3, 0, 0, 0 }, // subqueues }; - conf.setLong(WAIT_TIME_BEFORE_KILL, killTime); + conf.setLong( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + killTime); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); // ensure all pending rsrc from A get preempted from other queues @@ -308,7 +306,9 @@ public class TestProportionalCapacityPreemptionPolicy { { -1, 1, 1, 1 }, // req granularity { 3, 0, 0, 0 }, // subqueues }; - conf.setFloat(MAX_IGNORED_OVER_CAPACITY, (float) 0.1); + conf.setFloat( + CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY, + (float) 0.1); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // ignore 10% overcapacity to avoid jitter @@ -330,7 +330,7 @@ public class TestProportionalCapacityPreemptionPolicy { { 3, 0, 0, 0 }, // subqueues }; - schedConf.setPreemptionDisabled("root.queueB", true); + conf.setPreemptionDisabled("root.queueB", true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); @@ -343,7 +343,7 @@ public class TestProportionalCapacityPreemptionPolicy { // event handler will count only events from the following test and not the // previous one. setup(); - schedConf.setPreemptionDisabled("root.queueB", false); + conf.setPreemptionDisabled("root.queueB", false); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); @@ -382,7 +382,7 @@ public class TestProportionalCapacityPreemptionPolicy { // Need to call setup() again to reset mDisp setup(); // Turn off preemption for queueB and it's children - schedConf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled("root.queueA.queueB", true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); ApplicationAttemptId expectedAttemptOnQueueC = @@ -429,7 +429,7 @@ public class TestProportionalCapacityPreemptionPolicy { // Need to call setup() again to reset mDisp setup(); // Turn off preemption for queueB(appA) - schedConf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled("root.queueA.queueB", true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); // Now that queueB(appA) is not preemptable, verify that resources come @@ -439,8 +439,8 @@ public class TestProportionalCapacityPreemptionPolicy { setup(); // Turn off preemption for two of the 3 queues with over-capacity. - schedConf.setPreemptionDisabled("root.queueD.queueE", true); - schedConf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled("root.queueD.queueE", true); + conf.setPreemptionDisabled("root.queueA.queueB", true); ProportionalCapacityPreemptionPolicy policy3 = buildPolicy(qData); policy3.editSchedule(); @@ -481,7 +481,7 @@ public class TestProportionalCapacityPreemptionPolicy { // Turn off preemption for queueA and it's children. queueF(appC)'s request // should starve. setup(); // Call setup() to reset mDisp - schedConf.setPreemptionDisabled("root.queueA", true); + conf.setPreemptionDisabled("root.queueA", true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA))); // queueC @@ -505,7 +505,7 @@ public class TestProportionalCapacityPreemptionPolicy { { -1, -1, 1, 1, 1, -1, 1, 1, 1 }, // req granularity { 2, 3, 0, 0, 0, 3, 0, 0, 0 }, // subqueues }; - schedConf.setPreemptionDisabled("root.queueA.queueC", true); + conf.setPreemptionDisabled("root.queueA.queueC", true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // Although queueC(appB) is way over capacity and is untouchable, @@ -529,7 +529,7 @@ public class TestProportionalCapacityPreemptionPolicy { { 3, 2, 0, 0, 2, 0, 0, 2, 0, 0 }, // subqueues }; - schedConf.setPreemptionDisabled("root", true); + conf.setPreemptionDisabled("root", true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // All queues should be non-preemptable, so request should starve. @@ -556,7 +556,7 @@ public class TestProportionalCapacityPreemptionPolicy { { 2, 2, 0, 0, 2, 0, 0 }, // subqueues }; // QueueE inherits non-preemption from QueueD - schedConf.setPreemptionDisabled("root.queueD", true); + conf.setPreemptionDisabled("root.queueD", true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // appC is running on QueueE. QueueE is over absMaxCap, but is not @@ -596,7 +596,10 @@ public class TestProportionalCapacityPreemptionPolicy { { -1, 1, 1, 0 }, // req granularity { 3, 0, 0, 0 }, // subqueues }; - conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 0.1); + conf.setFloat( + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + (float) 0.1); + ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // ignore 10% imbalance between over-capacity queues @@ -616,7 +619,10 @@ public class TestProportionalCapacityPreemptionPolicy { { -1, 1, 1, 0 }, // req granularity { 3, 0, 0, 0 }, // subqueues }; - conf.setBoolean(OBSERVE_ONLY, true); + conf.setBoolean(CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY, + true); + when(mCS.getConfiguration()).thenReturn( + new CapacitySchedulerConfiguration(conf)); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // verify even severe imbalance not affected @@ -735,7 +741,7 @@ public class TestProportionalCapacityPreemptionPolicy { containers.add(rm4); // sort them - ProportionalCapacityPreemptionPolicy.sortContainers(containers); + FifoCandidatesSelector.sortContainers(containers); // verify the "priority"-first, "reverse container-id"-second // ordering is enforced correctly @@ -957,7 +963,7 @@ public class TestProportionalCapacityPreemptionPolicy { ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) { ProportionalCapacityPreemptionPolicy policy = new ProportionalCapacityPreemptionPolicy( - conf, rmContext, mCS, mClock); + rmContext, mCS, mClock); clusterResources = Resource.newInstance( leafAbsCapacities(qData[0], qData[7]), 0); ParentQueue mRoot = buildMockRootQueue(rand, qData); @@ -967,11 +973,6 @@ public class TestProportionalCapacityPreemptionPolicy { return policy; } - ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData, - String[][] resData) { - return buildPolicy(qData, resData, false); - } - ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData, String[][] resData, boolean useDominantResourceCalculator) { if (useDominantResourceCalculator) { @@ -979,7 +980,7 @@ public class TestProportionalCapacityPreemptionPolicy { new DominantResourceCalculator()); } ProportionalCapacityPreemptionPolicy policy = - new ProportionalCapacityPreemptionPolicy(conf, rmContext, mCS, mClock); + new ProportionalCapacityPreemptionPolicy(rmContext, mCS, mClock); clusterResources = leafAbsCapacities(parseResourceDetails(resData[0]), qData[2]); ParentQueue mRoot = buildMockRootQueue(rand, resData, qData); @@ -1124,7 +1125,7 @@ public class TestProportionalCapacityPreemptionPolicy { String qName = ""; while(tokenizer.hasMoreTokens()) { qName += tokenizer.nextToken(); - preemptionDisabled = schedConf.getPreemptionDisabled(qName, preemptionDisabled); + preemptionDisabled = conf.getPreemptionDisabled(qName, preemptionDisabled); qName += "."; } return preemptionDisabled; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java index 21ea4953a21..b266665dce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java @@ -18,29 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MONITORING_INTERVAL; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND; -import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.eq; -import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -81,6 +58,25 @@ import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestProportionalCapacityPreemptionPolicyForNodePartitions { private static final Log LOG = LogFactory.getLog(TestProportionalCapacityPreemptionPolicyForNodePartitions.class); @@ -94,8 +90,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { private ResourceCalculator rc = new DefaultResourceCalculator(); private Clock mClock = null; - private Configuration conf = null; - private CapacitySchedulerConfiguration csConf = null; + private CapacitySchedulerConfiguration conf = null; private CapacityScheduler cs = null; private EventHandler mDisp = null; private ProportionalCapacityPreemptionPolicy policy = null; @@ -107,24 +102,23 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { org.apache.log4j.Logger.getRootLogger().setLevel( org.apache.log4j.Level.DEBUG); - conf = new Configuration(false); - conf.setLong(WAIT_TIME_BEFORE_KILL, 10000); - conf.setLong(MONITORING_INTERVAL, 3000); + conf = new CapacitySchedulerConfiguration(new Configuration(false)); + conf.setLong( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000); + conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + 3000); // report "ideal" preempt - conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0); - conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0); - conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, - ProportionalCapacityPreemptionPolicy.class.getCanonicalName()); - conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); - // FairScheduler doesn't support this test, - // Set CapacityScheduler as the scheduler for this test. - conf.set("yarn.resourcemanager.scheduler.class", - CapacityScheduler.class.getName()); + conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, + (float) 1.0); + conf.setFloat( + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + (float) 1.0); mClock = mock(Clock.class); cs = mock(CapacityScheduler.class); when(cs.getResourceCalculator()).thenReturn(rc); when(cs.getPreemptionManager()).thenReturn(new PreemptionManager()); + when(cs.getConfiguration()).thenReturn(conf); nlm = mock(RMNodeLabelsManager.class); mDisp = mock(EventHandler.class); @@ -134,11 +128,9 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { Dispatcher disp = mock(Dispatcher.class); when(rmContext.getDispatcher()).thenReturn(disp); when(disp.getEventHandler()).thenReturn(mDisp); - csConf = new CapacitySchedulerConfiguration(); - when(cs.getConfiguration()).thenReturn(csConf); when(cs.getRMContext()).thenReturn(rmContext); - policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock); + policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock); partitionToResource = new HashMap<>(); nodeIdToSchedulerNodes = new HashMap<>(); nameToCSQueues = new HashMap<>(); @@ -576,7 +568,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { "c\t" // app3 in c + "(1,1,n1,x,20,false)"; // 20x in n1 - csConf.setPreemptionDisabled("root.b", true); + conf.setPreemptionDisabled("root.b", true); buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); policy.editSchedule(); @@ -901,7 +893,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { when(cs.getClusterResource()).thenReturn(clusterResource); mockApplications(appsConfig); - policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, + policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock); } @@ -1235,7 +1227,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions { // Setup preemption disabled when(queue.getPreemptionDisabled()).thenReturn( - csConf.getPreemptionDisabled(queuePath, false)); + conf.getPreemptionDisabled(queuePath, false)); nameToCSQueues.put(queueName, queue); when(cs.getQueue(eq(queueName))).thenReturn(queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java index bea7797b917..216ebabbeda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.util.ArrayList; @@ -81,14 +82,15 @@ public class TestCapacitySchedulerPreemption { conf = TestUtils.getConfigurationWithMultipleQueues(this.conf); // Set preemption related configurations - conf.setInt(ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL, + conf.setInt(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 0); conf.setBoolean(CapacitySchedulerConfiguration.LAZY_PREEMPTION_ENALBED, true); + conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, + 1.0f); conf.setFloat( - ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND, 1.0f); - conf.setFloat( - ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR, 1.0f); + CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, + 1.0f); mgr = new NullRMNodeLabelsManager(); mgr.init(this.conf); clock = mock(Clock.class); @@ -484,6 +486,10 @@ public class TestCapacitySchedulerPreemption { .isEmpty()); } + /* + * Ignore this test now because it could be a premature optimization + */ + @Ignore @Test (timeout = 60000) public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded() throws Exception { From 37e23ce45c592f3c9c48a08a52a5f46787f6c0e9 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Wed, 30 Mar 2016 13:37:37 -0700 Subject: [PATCH 19/75] HDFS-10223. peerFromSocketAndKey performs SASL exchange before setting connection timeouts (cmccabe) --- .../org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 ++- .../erasurecode/ErasureCodingWorker.java | 3 +- .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../sasl/TestSaslDataTransfer.java | 48 +++++++++++++++++++ 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 88bd21909d2..d4e3187a53b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2734,9 +2734,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, NetUtils.connect(sock, addr, getRandomLocalInterfaceAddr(), socketTimeout); peer = DFSUtilClient.peerFromSocketAndKey(saslClient, sock, this, - blockToken, datanodeId); - peer.setReadTimeout(socketTimeout); - peer.setWriteTimeout(socketTimeout); + blockToken, datanodeId, socketTimeout); success = true; return peer; } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 880234e1610..b9f4dce2d4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -587,12 +587,14 @@ public class DFSUtilClient { public static Peer peerFromSocketAndKey( SaslDataTransferClient saslClient, Socket s, DataEncryptionKeyFactory keyFactory, - Token blockToken, DatanodeID datanodeId) - throws IOException { + Token blockToken, DatanodeID datanodeId, + int socketTimeoutMs) throws IOException { Peer peer = null; boolean success = false; try { peer = peerFromSocket(s); + peer.setReadTimeout(socketTimeoutMs); + peer.setWriteTimeout(socketTimeoutMs); peer = saslClient.peerSend(peer, keyFactory, blockToken, datanodeId); success = true; return peer; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 74fb3e1134d..4bcb291b288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -875,8 +875,7 @@ public final class ErasureCodingWorker { NetUtils.connect(sock, addr, socketTimeout); peer = DFSUtilClient.peerFromSocketAndKey(datanode.getSaslClient(), sock, datanode.getDataEncryptionKeyFactoryForBlock(b), - blockToken, datanodeId); - peer.setReadTimeout(socketTimeout); + blockToken, datanodeId, socketTimeout); success = true; return peer; } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index d3be9b445e7..291ba5673bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -952,7 +952,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { s.setSoTimeout(HdfsConstants.READ_TIMEOUT); peer = DFSUtilClient.peerFromSocketAndKey( dfs.getSaslDataTransferClient(), s, NamenodeFsck.this, - blockToken, datanodeId); + blockToken, datanodeId, HdfsConstants.READ_TIMEOUT); } finally { if (peer == null) { IOUtils.closeQuietly(s); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java index 2d4eb0d2b43..8555e5d0ad9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java @@ -25,6 +25,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.BlockLocation; @@ -32,12 +36,18 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.Peer; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver; +import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.junit.After; @@ -197,4 +207,42 @@ public class TestSaslDataTransfer extends SaslDataTransferTestCase { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); } + + /** + * Verifies that peerFromSocketAndKey honors socket read timeouts. + */ + @Test(timeout=60000) + public void TestPeerFromSocketAndKeyReadTimeout() throws Exception { + HdfsConfiguration conf = createSecureConfig( + "authentication,integrity,privacy"); + AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false); + SaslDataTransferClient saslClient = new SaslDataTransferClient( + conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), + TrustedChannelResolver.getInstance(conf), fallbackToSimpleAuth); + DatanodeID fakeDatanodeId = new DatanodeID("127.0.0.1", "localhost", + "beefbeef-beef-beef-beef-beefbeefbeef", 1, 2, 3, 4); + DataEncryptionKeyFactory dataEncKeyFactory = + new DataEncryptionKeyFactory() { + @Override + public DataEncryptionKey newDataEncryptionKey() { + return new DataEncryptionKey(123, "456", new byte[8], + new byte[8], 1234567, "fakeAlgorithm"); + } + }; + ServerSocket serverSocket = null; + Socket socket = null; + try { + serverSocket = new ServerSocket(0, -1); + socket = new Socket(serverSocket.getInetAddress(), + serverSocket.getLocalPort()); + Peer peer = DFSUtilClient.peerFromSocketAndKey(saslClient, socket, + dataEncKeyFactory, new Token(), fakeDatanodeId, 1); + peer.close(); + Assert.fail("Expected DFSClient#peerFromSocketAndKey to time out."); + } catch (SocketTimeoutException e) { + GenericTestUtils.assertExceptionContains("Read timed out", e); + } finally { + IOUtils.cleanup(null, socket, serverSocket); + } + } } From e4fc609d5d3739b7809057954c5233cfd1d1117b Mon Sep 17 00:00:00 2001 From: Zhe ZHang Date: Wed, 30 Mar 2016 14:13:11 -0700 Subject: [PATCH 20/75] HADOOP-12886. Exclude weak ciphers in SSLFactory through ssl-server.xml. Contributed by Wei-Chiu Chuang. --- .../hadoop/security/ssl/SSLFactory.java | 42 +++++- .../hadoop/security/ssl/TestSSLFactory.java | 139 +++++++++++++++++- 2 files changed, 175 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java index ea658480d2e..95cba809894 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java @@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import javax.net.ssl.HostnameVerifier; @@ -34,6 +36,11 @@ import javax.net.ssl.SSLSocketFactory; import java.io.IOException; import java.net.HttpURLConnection; import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; /** * Factory that creates SSLEngine and SSLSocketFactory instances using @@ -48,6 +55,7 @@ import java.security.GeneralSecurityException; @InterfaceAudience.Private @InterfaceStability.Evolving public class SSLFactory implements ConnectionConfigurator { + static final Logger LOG = LoggerFactory.getLogger(SSLFactory.class); @InterfaceAudience.Private public static enum Mode { CLIENT, SERVER } @@ -60,7 +68,7 @@ public class SSLFactory implements ConnectionConfigurator { "hadoop.ssl.client.conf"; public static final String SSL_SERVER_CONF_KEY = "hadoop.ssl.server.conf"; - public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509"; + public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509"; public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false; @@ -71,6 +79,8 @@ public class SSLFactory implements ConnectionConfigurator { "hadoop.ssl.enabled.protocols"; public static final String DEFAULT_SSL_ENABLED_PROTOCOLS = "TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2"; + public static final String SSL_SERVER_EXCLUDE_CIPHER_LIST = + "ssl.server.exclude.cipher.list"; private Configuration conf; private Mode mode; @@ -80,6 +90,7 @@ public class SSLFactory implements ConnectionConfigurator { private KeyStoresFactory keystoresFactory; private String[] enabledProtocols = null; + private List excludeCiphers; /** * Creates an SSLFactory. @@ -105,6 +116,14 @@ public class SSLFactory implements ConnectionConfigurator { enabledProtocols = conf.getStrings(SSL_ENABLED_PROTOCOLS, DEFAULT_SSL_ENABLED_PROTOCOLS); + String excludeCiphersConf = + sslConf.get(SSL_SERVER_EXCLUDE_CIPHER_LIST, ""); + if (excludeCiphersConf.isEmpty()) { + excludeCiphers = new LinkedList(); + } else { + LOG.debug("will exclude cipher suites: {}", excludeCiphersConf); + excludeCiphers = Arrays.asList(excludeCiphersConf.split(",")); + } } private Configuration readSSLConfiguration(Mode mode) { @@ -195,11 +214,32 @@ public class SSLFactory implements ConnectionConfigurator { } else { sslEngine.setUseClientMode(false); sslEngine.setNeedClientAuth(requireClientCert); + disableExcludedCiphers(sslEngine); } sslEngine.setEnabledProtocols(enabledProtocols); return sslEngine; } + private void disableExcludedCiphers(SSLEngine sslEngine) { + String[] cipherSuites = sslEngine.getEnabledCipherSuites(); + + ArrayList defaultEnabledCipherSuites = + new ArrayList(Arrays.asList(cipherSuites)); + Iterator iterator = excludeCiphers.iterator(); + + while(iterator.hasNext()) { + String cipherName = (String)iterator.next(); + if(defaultEnabledCipherSuites.contains(cipherName)) { + defaultEnabledCipherSuites.remove(cipherName); + LOG.debug("Disabling cipher suite {}.", cipherName); + } + } + + cipherSuites = defaultEnabledCipherSuites.toArray( + new String[defaultEnabledCipherSuites.size()]); + sslEngine.setEnabledCipherSuites(cipherSuites); + } + /** * Returns a configured SSLServerSocketFactory. * diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java index 004888c7b71..b8a09ed92e7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -17,24 +17,31 @@ */ package org.apache.hadoop.security.ssl; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLEngineResult; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLSession; import java.io.File; import java.net.URL; +import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -42,13 +49,21 @@ import java.util.Collections; import java.util.Map; public class TestSSLFactory { - + private static final Logger LOG = LoggerFactory + .getLogger(TestSSLFactory.class); private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" + TestSSLFactory.class.getSimpleName(); private static final String KEYSTORES_DIR = new File(BASEDIR).getAbsolutePath(); private String sslConfsDir; + private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA," + + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA," + + "SSL_RSA_WITH_DES_CBC_SHA," + + "SSL_DHE_RSA_WITH_DES_CBC_SHA," + + "SSL_RSA_EXPORT_WITH_RC4_40_MD5," + + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA," + + "SSL_RSA_WITH_RC4_128_MD5"; @BeforeClass public static void setUp() throws Exception { @@ -62,7 +77,7 @@ public class TestSSLFactory { throws Exception { Configuration conf = new Configuration(); KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf, - clientCert, trustStore); + clientCert, trustStore, excludeCiphers); return conf; } @@ -125,6 +140,120 @@ public class TestSSLFactory { serverMode(true, false); } + private void runDelegatedTasks(SSLEngineResult result, SSLEngine engine) + throws Exception { + Runnable runnable; + if (result.getHandshakeStatus() == + SSLEngineResult.HandshakeStatus.NEED_TASK) { + while ((runnable = engine.getDelegatedTask()) != null) { + LOG.info("running delegated task..."); + runnable.run(); + } + SSLEngineResult.HandshakeStatus hsStatus = engine.getHandshakeStatus(); + if (hsStatus == SSLEngineResult.HandshakeStatus.NEED_TASK) { + throw new Exception("handshake shouldn't need additional tasks"); + } + } + } + + private static boolean isEngineClosed(SSLEngine engine) { + return engine.isOutboundDone() && engine.isInboundDone(); + } + + private static void checkTransfer(ByteBuffer a, ByteBuffer b) + throws Exception { + a.flip(); + b.flip(); + assertTrue("transfer did not complete", a.equals(b)); + + a.position(a.limit()); + b.position(b.limit()); + a.limit(a.capacity()); + b.limit(b.capacity()); + } + @Test + public void testServerWeakCiphers() throws Exception { + // a simple test case to verify that SSL server rejects weak cipher suites, + // inspired by https://docs.oracle.com/javase/8/docs/technotes/guides/ + // security/jsse/samples/sslengine/SSLEngineSimpleDemo.java + + // set up a client and a server SSLEngine object, and let them exchange + // data over ByteBuffer instead of network socket. + GenericTestUtils.setLogLevel(SSLFactory.LOG, Level.DEBUG); + final Configuration conf = createConfiguration(true, true); + + SSLFactory serverSSLFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); + SSLFactory clientSSLFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + + serverSSLFactory.init(); + clientSSLFactory.init(); + + SSLEngine serverSSLEngine = serverSSLFactory.createSSLEngine(); + SSLEngine clientSSLEngine = clientSSLFactory.createSSLEngine(); + // client selects cipher suites excluded by server + clientSSLEngine.setEnabledCipherSuites(excludeCiphers.split(",")); + + // use the same buffer size for server and client. + SSLSession session = clientSSLEngine.getSession(); + int appBufferMax = session.getApplicationBufferSize(); + int netBufferMax = session.getPacketBufferSize(); + + ByteBuffer clientOut = ByteBuffer.wrap("client".getBytes()); + ByteBuffer clientIn = ByteBuffer.allocate(appBufferMax); + ByteBuffer serverOut = ByteBuffer.wrap("server".getBytes()); + ByteBuffer serverIn = ByteBuffer.allocate(appBufferMax); + + // send data from client to server + ByteBuffer cTOs = ByteBuffer.allocateDirect(netBufferMax); + // send data from server to client + ByteBuffer sTOc = ByteBuffer.allocateDirect(netBufferMax); + + boolean dataDone = false; + try { + /** + * Server and client engines call wrap()/unwrap() to perform handshaking, + * until both engines are closed. + */ + while (!isEngineClosed(clientSSLEngine) || + !isEngineClosed(serverSSLEngine)) { + LOG.info("client wrap " + wrap(clientSSLEngine, clientOut, cTOs)); + LOG.info("server wrap " + wrap(serverSSLEngine, serverOut, sTOc)); + cTOs.flip(); + sTOc.flip(); + LOG.info("client unwrap " + unwrap(clientSSLEngine, sTOc, clientIn)); + LOG.info("server unwrap " + unwrap(serverSSLEngine, cTOs, serverIn)); + cTOs.compact(); + sTOc.compact(); + if (!dataDone && (clientOut.limit() == serverIn.position()) && + (serverOut.limit() == clientIn.position())) { + checkTransfer(serverOut, clientIn); + checkTransfer(clientOut, serverIn); + + LOG.info("closing client"); + clientSSLEngine.closeOutbound(); + dataDone = true; + } + } + Assert.fail("The exception was not thrown"); + } catch (SSLHandshakeException e) { + GenericTestUtils.assertExceptionContains("no cipher suites in common", e); + } + } + + private SSLEngineResult wrap(SSLEngine engine, ByteBuffer from, + ByteBuffer to) throws Exception { + SSLEngineResult result = engine.wrap(from, to); + runDelegatedTasks(result, engine); + return result; + } + + private SSLEngineResult unwrap(SSLEngine engine, ByteBuffer from, + ByteBuffer to) throws Exception { + SSLEngineResult result = engine.unwrap(from, to); + runDelegatedTasks(result, engine); + return result; + } + @Test public void validHostnameVerifier() throws Exception { Configuration conf = createConfiguration(false, true); From 32c0c3ecdf72e89a63f4aee5e75d1c5a12714b89 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 31 Mar 2016 09:04:09 +0900 Subject: [PATCH 21/75] HDFS-10221. Add .json to the rat exclusions. Contributed by Ming Ma. --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 4e1901b59cf..668bbfe4752 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -381,6 +381,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> src/test/all-tests src/test/resources/*.tgz src/test/resources/data* + **/*.json src/test/resources/editsStored* src/test/resources/empty-file src/main/webapps/datanode/robots.txt From acca149ec96f2932bebc492452a63a159de9ce47 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 31 Mar 2016 16:04:47 +0900 Subject: [PATCH 22/75] HADOOP-12902. JavaDocs for SignerSecretProvider are out-of-date in AuthenticationFilter. Contributed by Gabor Liptak. --- .../server/AuthenticationFilter.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 4bdc80826aa..5c93fd37374 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -61,9 +61,9 @@ import java.util.*; *

  • [#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the * {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link KerberosAuthenticationHandler}, otherwise * the full class name of the {@link AuthenticationHandler} must be specified.
    • - *
  • [#PREFIX#.]signature.secret: when signer.secret.provider is set to - * "string" or not specified, this is the value for the secret used to sign the - * HTTP cookie.
    • + *
  • [#PREFIX#.]signature.secret.file: when signer.secret.provider is set to + * "file" or not specified, this is the location of file including the secret + * used to sign the HTTP cookie.
    • *
  • [#PREFIX#.]token.validity: time -in seconds- that the generated token is * valid before a new authentication is triggered, default value is * 3600 seconds. This is also used for the rollover interval for @@ -79,17 +79,16 @@ import java.util.*; *

    *

    * Out of the box it provides 3 signer secret provider implementations: - * "string", "random", and "zookeeper" + * "file", "random" and "zookeeper" *

    * Additional signer secret providers are supported via the * {@link SignerSecretProvider} class. *

    * For the HTTP cookies mentioned above, the SignerSecretProvider is used to * determine the secret to use for signing the cookies. Different - * implementations can have different behaviors. The "string" implementation - * simply uses the string set in the [#PREFIX#.]signature.secret property - * mentioned above. The "random" implementation uses a randomly generated - * secret that rolls over at the interval specified by the + * implementations can have different behaviors. The "file" implementation + * loads the secret from a specified file. The "random" implementation uses a + * randomly generated secret that rolls over at the interval specified by the * [#PREFIX#.]token.validity mentioned above. The "zookeeper" implementation * is like the "random" one, except that it synchronizes the random secret * and rollovers between multiple servers; it's meant for HA services. @@ -97,12 +96,12 @@ import java.util.*; * The relevant configuration properties are: *

      *
    • signer.secret.provider: indicates the name of the SignerSecretProvider - * class to use. Possible values are: "string", "random", "zookeeper", or a - * classname. If not specified, the "string" implementation will be used with - * [#PREFIX#.]signature.secret; and if that's not specified, the "random" + * class to use. Possible values are: "file", "random", "zookeeper", or a + * classname. If not specified, the "file" implementation will be used with + * [#PREFIX#.]signature.secret.file; and if that's not specified, the "random" * implementation will be used.
      • - *
    • [#PREFIX#.]signature.secret: When the "string" implementation is - * specified, this value is used as the secret.
      • + *
    • [#PREFIX#.]signature.secret.file: When the "file" implementation is + * specified, this content of this file is used as the secret.
      • *
    • [#PREFIX#.]token.validity: When the "random" or "zookeeper" * implementations are specified, this value is used as the rollover * interval.
      • @@ -176,10 +175,10 @@ public class AuthenticationFilter implements Filter { /** * Constant for the configuration property that indicates the name of the * SignerSecretProvider class to use. - * Possible values are: "string", "random", "zookeeper", or a classname. - * If not specified, the "string" implementation will be used with - * SIGNATURE_SECRET; and if that's not specified, the "random" implementation - * will be used. + * Possible values are: "file", "random", "zookeeper", or a classname. + * If not specified, the "file" implementation will be used with + * SIGNATURE_SECRET_FILE; and if that's not specified, the "random" + * implementation will be used. */ public static final String SIGNER_SECRET_PROVIDER = "signer.secret.provider"; From f1b8f6b2c16403869f78a54268ae1165982a7050 Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Thu, 31 Mar 2016 14:01:48 +0530 Subject: [PATCH 23/75] YARN-4884. Fix missing documentation about rmadmin command regarding node labels. Contributed by Kai Sasaki. --- .../src/site/markdown/YarnCommands.md | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 5941988e34c..40704f021a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -217,31 +217,46 @@ Start the ResourceManager Usage: ``` - yarn rmadmin [-refreshQueues] - [-refreshNodes] - [-refreshUserToGroupsMapping] - [-refreshSuperUserGroupsConfiguration] - [-refreshAdminAcls] - [-refreshServiceAcl] - [-getGroups [username]] - [-transitionToActive [--forceactive] [--forcemanual] ] - [-transitionToStandby [--forcemanual] ] - [-failover [--forcefence] [--forceactive] ] - [-getServiceState ] - [-checkHealth ] - [-help [cmd]] + Usage: yarn rmadmin + -refreshQueues + -refreshNodes [-g [timeout in seconds]] + -refreshNodesResources + -refreshSuperUserGroupsConfiguration + -refreshUserToGroupsMappings + -refreshAdminAcls + -refreshServiceAcl + -getGroups [username] + -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> + -removeFromClusterNodeLabels (label splitted by ",") + -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> + -directlyAccessNodeLabelStore + -refreshClusterMaxPriority + -updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout]) + -transitionToActive [--forceactive] + -transitionToStandby + -failover [--forcefence] [--forceactive] + -getServiceState + -checkHealth + -help [cmd] ``` | COMMAND\_OPTIONS | Description | |:---- |:---- | | -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. | | -refreshNodes | Refresh the hosts information at the ResourceManager. | -| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. | +| -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. | | -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. | +| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. | | -refreshAdminAcls | Refresh acls for administration of ResourceManager | | -refreshServiceAcl | Reload the service-level authorization policy file ResourceManager will reload the authorization policy file. | | -getGroups [username] | Get groups the specified user belongs to. | -| -transitionToActive [--forceactive] [--forcemanual] \ | Transitions the service into Active state. Try to make the target active without checking that there is no active node if the --forceactive option is used. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. | +| -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> | Add to cluster node labels. Default exclusivity is true. | +| -removeFromClusterNodeLabels (label splitted by ",") | Remove from cluster node labels. | +| -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> | Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) | +| -directlyAccessNodeLabelStore | This is DEPRECATED, will be removed in future releases. Directly access node label store, with this option, all node label related operations will not connect RM. Instead, they will access/modify stored node labels directly. By default, it is false (access via RM). AND PLEASE NOTE: if you configured yarn.node-labels.fs-store.root-dir to a local directory (instead of NFS or HDFS), this option will only work when the command run on the machine where RM is running. | +| -refreshClusterMaxPriority | Refresh cluster max priority | +| -updateNodeResource [NodeID] [MemSize] [vCores] \([OvercommitTimeout]\) | Update resource on specific node. | +| -transitionToActive [--forceactive] [--forcemanual] \ | Transitions the service into Active state. Try to make the target active without checking that there is no active node if the --forceactive option is used. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. This command can not be used if automatic failover is enabled.| | -transitionToStandby [--forcemanual] \ | Transitions the service into Standby state. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. | | -failover [--forceactive] \ \ | Initiate a failover from serviceId1 to serviceId2. Try to failover to the target service even if it is not ready if the --forceactive option is used. This command can not be used if automatic failover is enabled. | | -getServiceState \ | Returns the state of the service. | From 0064cba169d1bb761f6e81ee86830be598d7c500 Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Thu, 31 Mar 2016 14:05:49 +0530 Subject: [PATCH 24/75] YARN-4857. Add missing default configuration regarding preemption of CapacityScheduler. Contributed by Kai Sasaki. --- .../src/main/resources/yarn-default.xml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index ea1afe48ae0..33cd9193f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -908,6 +908,64 @@ 600000 + + + If true, run the policy but do not affect the cluster with preemption and kill events. + + yarn.resourcemanager.monitor.capacity.preemption.observe_only + false + + + + + Time in milliseconds between invocations of this ProportionalCapacityPreemptionPolicy + policy. + + yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval + 3000 + + + + + Time in milliseconds between requesting a preemption from an application and killing + the container. + + yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill + 15000 + + + + + Maximum percentage of resources preempted in a single round. By controlling this valueone + can throttle the pace at which containers are reclaimed from the cluster. After computing + the total desired preemption, the policy scales it back within this limit. + + yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round + 0.1 + + + + + Maximum amount of resources above the target capacity ignored for preemption. + This defines a deadzone around the target capacity that helps prevent thrashing and + oscillations around the computed target balance. High values would slow the time to capacity + and (absent natural.completions) it might prevent convergence to guaranteed capacity. + + yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity + 0.1 + + + + + Given a computed preemption target, account for containers naturally expiring and preempt + only this percentage of the delta. This determines the rate of geometric convergence into + the deadzone (MAX_IGNORED_OVER_CAPACITY). For example, a termination factor of 0.5 will reclaim + almost 95% of resources within 5 * #WAIT_TIME_BEFORE_KILL, even absent natural termination. + + yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor + 0.2 + + From 0a74610d1c7c7f183d2b2d0b7a775add53cf6c94 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 24 Mar 2016 08:47:00 -0700 Subject: [PATCH 25/75] HADOOP-11393. Revert HADOOP_PREFIX, go back to HADOOP_HOME (aw) --- .../hadoop-common/src/main/bin/hadoop | 12 +++--- .../src/main/bin/hadoop-config.sh | 6 ++- .../src/main/bin/hadoop-daemon.sh | 6 +-- .../src/main/bin/hadoop-daemons.sh | 6 +-- .../src/main/bin/hadoop-functions.sh | 37 +++++++++-------- .../src/main/bin/hadoop-layout.sh.example | 14 +++---- .../hadoop-common/src/main/bin/slaves.sh | 6 +-- .../hadoop-common/src/main/bin/start-all.sh | 4 +- .../hadoop-common/src/main/bin/stop-all.sh | 4 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 10 ++--- .../org/apache/hadoop/tracing/TraceUtils.java | 4 +- .../src/site/markdown/ClusterSetup.md | 40 +++++++++---------- .../src/site/markdown/CommandsManual.md | 2 +- .../src/site/markdown/UnixShellGuide.md | 2 +- .../scripts/hadoop-functions_test_helper.bash | 3 +- .../src/test/scripts/hadoop_basic_init.bats | 2 +- .../src/test/scripts/hadoop_bootstrap.bats | 4 +- .../src/test/scripts/hadoop_confdir.bats | 24 +++++------ .../src/test/scripts/hadoop_finalize.bats | 2 +- .../hadoop-kms/src/main/conf/kms-env.sh | 4 +- .../hadoop-kms/src/main/libexec/kms-config.sh | 8 ++-- .../hadoop-kms/src/main/sbin/kms.sh | 4 +- .../src/main/conf/httpfs-env.sh | 4 +- .../src/main/libexec/httpfs-config.sh | 8 ++-- .../src/main/sbin/httpfs.sh | 4 +- .../src/main/native/fuse-dfs/doc/README | 6 +-- .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 12 +++--- .../src/main/bin/distribute-exclude.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs | 4 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 6 +-- .../src/main/bin/refresh-namenodes.sh | 4 +- .../src/main/bin/start-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 4 +- .../src/main/bin/start-secure-dns.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 4 +- .../src/main/bin/stop-secure-dns.sh | 4 +- .../src/site/markdown/Federation.md | 18 ++++----- .../markdown/HDFSHighAvailabilityWithNFS.md | 4 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 4 +- .../src/site/markdown/HdfsNfsGateway.md | 8 ++-- .../apache/hadoop/tracing/TestTraceAdmin.java | 2 +- .../TestTracingShortCircuitLocalRead.java | 4 +- hadoop-mapreduce-project/bin/mapred | 4 +- hadoop-mapreduce-project/bin/mapred-config.sh | 6 +-- .../bin/mr-jobhistory-daemon.sh | 4 +- .../apache/hadoop/mapred/pipes/Submitter.java | 2 +- .../java/org/apache/hadoop/fs/DFSCIOTest.java | 2 +- .../apache/hadoop/mapred/ReliabilityTest.java | 2 +- .../hadoop/tools/HadoopArchiveLogs.java | 4 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 4 +- .../hadoop/contrib/utils/join/README.txt | 2 +- .../native/pipes/debug/pipes-default-script | 5 ++- .../hadoop-sls/src/main/bin/rumen2sls.sh | 4 +- .../hadoop-sls/src/main/bin/slsrun.sh | 8 ++-- .../hadoop/streaming/DumpTypedBytes.java | 2 +- .../hadoop/streaming/HadoopStreaming.java | 2 +- .../hadoop/streaming/LoadTypedBytes.java | 2 +- .../apache/hadoop/streaming/StreamJob.java | 16 ++++---- .../hadoop-yarn/bin/start-yarn.sh | 4 +- .../hadoop-yarn/bin/stop-yarn.sh | 4 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +- .../hadoop-yarn/bin/yarn-config.sh | 6 +-- .../hadoop-yarn/bin/yarn-daemon.sh | 4 +- .../hadoop-yarn/bin/yarn-daemons.sh | 4 +- .../TestDockerContainerExecutorWithMocks.java | 2 +- .../markdown/DockerContainerExecutor.md.vm | 2 +- 67 files changed, 211 insertions(+), 208 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 46eaf273687..07569872e76 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -47,8 +47,8 @@ function hadoop_usage # This script runs the hadoop core commands. # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" @@ -84,9 +84,9 @@ case ${COMMAND} in # shellcheck disable=SC2086 exec "${HADOOP_HDFS_HOME}/bin/hdfs" \ --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@" - elif [[ -f "${HADOOP_PREFIX}/bin/hdfs" ]]; then + elif [[ -f "${HADOOP_HOME}/bin/hdfs" ]]; then # shellcheck disable=SC2086 - exec "${HADOOP_PREFIX}/bin/hdfs" \ + exec "${HADOOP_HOME}/bin/hdfs" \ --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@" else hadoop_error "HADOOP_HDFS_HOME not found!" @@ -104,8 +104,8 @@ case ${COMMAND} in if [[ -f "${HADOOP_MAPRED_HOME}/bin/mapred" ]]; then exec "${HADOOP_MAPRED_HOME}/bin/mapred" \ --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@" - elif [[ -f "${HADOOP_PREFIX}/bin/mapred" ]]; then - exec "${HADOOP_PREFIX}/bin/mapred" \ + elif [[ -f "${HADOOP_HOME}/bin/mapred" ]]; then + exec "${HADOOP_HOME}/bin/mapred" \ --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@" else hadoop_error "HADOOP_MAPRED_HOME not found!" diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index 0b52895aec3..fd2c83e2369 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -63,6 +63,8 @@ else exit 1 fi +hadoop_deprecate_envvar HADOOP_PREFIX HADOOP_HOME + # allow overrides of the above and pre-defines of the below if [[ -n "${HADOOP_COMMON_HOME}" ]] && [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh" ]]; then @@ -128,8 +130,8 @@ fi hadoop_shellprofiles_init # get the native libs in there pretty quick -hadoop_add_javalibpath "${HADOOP_PREFIX}/build/native" -hadoop_add_javalibpath "${HADOOP_PREFIX}/${HADOOP_COMMON_LIB_NATIVE_DIR}" +hadoop_add_javalibpath "${HADOOP_HOME}/build/native" +hadoop_add_javalibpath "${HADOOP_HOME}/${HADOOP_COMMON_LIB_NATIVE_DIR}" hadoop_shellprofiles_nativelib diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh index 5f094d631b0..8118f541452 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh @@ -21,8 +21,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) @@ -47,7 +47,7 @@ daemonmode=$1 shift if [[ -z "${HADOOP_HDFS_HOME}" ]]; then - hdfsscript="${HADOOP_PREFIX}/bin/hdfs" + hdfsscript="${HADOOP_HOME}/bin/hdfs" else hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs" fi diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh index 604eb7eb339..ae1e3248238 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh @@ -27,8 +27,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi @@ -51,7 +51,7 @@ daemonmode=$1 shift if [[ -z "${HADOOP_HDFS_HOME}" ]]; then - hdfsscript="${HADOOP_PREFIX}/bin/hdfs" + hdfsscript="${HADOOP_HOME}/bin/hdfs" else hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs" fi diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 7f293b64754..6c4c3459f3c 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -278,7 +278,7 @@ function hadoop_bootstrap # By now, HADOOP_LIBEXEC_DIR should have been defined upstream # We can piggyback off of that to figure out where the default # HADOOP_FREFIX should be. This allows us to run without - # HADOOP_PREFIX ever being defined by a human! As a consequence + # HADOOP_HOME ever being defined by a human! As a consequence # HADOOP_LIBEXEC_DIR now becomes perhaps the single most powerful # env var within Hadoop. if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then @@ -286,8 +286,8 @@ function hadoop_bootstrap exit 1 fi HADOOP_DEFAULT_PREFIX=$(cd -P -- "${HADOOP_LIBEXEC_DIR}/.." >/dev/null && pwd -P) - HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX} - export HADOOP_PREFIX + HADOOP_HOME=${HADOOP_HOME:-$HADOOP_DEFAULT_PREFIX} + export HADOOP_HOME # # short-cuts. vendors may redefine these as well, preferably @@ -302,7 +302,7 @@ function hadoop_bootstrap YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"} MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"} MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"} - HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_PREFIX}} + HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}} HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"} HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"} @@ -326,12 +326,12 @@ function hadoop_find_confdir # An attempt at compatibility with some Hadoop 1.x # installs. - if [[ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]]; then + if [[ -e "${HADOOP_HOME}/conf/hadoop-env.sh" ]]; then conf_dir="conf" else conf_dir="etc/hadoop" fi - export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_PREFIX}/${conf_dir}}" + export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_HOME}/${conf_dir}}" hadoop_debug "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" } @@ -524,8 +524,8 @@ function hadoop_basic_init hadoop_debug "Initialize CLASSPATH" if [[ -z "${HADOOP_COMMON_HOME}" ]] && - [[ -d "${HADOOP_PREFIX}/${HADOOP_COMMON_DIR}" ]]; then - export HADOOP_COMMON_HOME="${HADOOP_PREFIX}" + [[ -d "${HADOOP_HOME}/${HADOOP_COMMON_DIR}" ]]; then + export HADOOP_COMMON_HOME="${HADOOP_HOME}" fi # default policy file for service-level authorization @@ -533,20 +533,20 @@ function hadoop_basic_init # define HADOOP_HDFS_HOME if [[ -z "${HADOOP_HDFS_HOME}" ]] && - [[ -d "${HADOOP_PREFIX}/${HDFS_DIR}" ]]; then - export HADOOP_HDFS_HOME="${HADOOP_PREFIX}" + [[ -d "${HADOOP_HOME}/${HDFS_DIR}" ]]; then + export HADOOP_HDFS_HOME="${HADOOP_HOME}" fi # define HADOOP_YARN_HOME if [[ -z "${HADOOP_YARN_HOME}" ]] && - [[ -d "${HADOOP_PREFIX}/${YARN_DIR}" ]]; then - export HADOOP_YARN_HOME="${HADOOP_PREFIX}" + [[ -d "${HADOOP_HOME}/${YARN_DIR}" ]]; then + export HADOOP_YARN_HOME="${HADOOP_HOME}" fi # define HADOOP_MAPRED_HOME if [[ -z "${HADOOP_MAPRED_HOME}" ]] && - [[ -d "${HADOOP_PREFIX}/${MAPRED_DIR}" ]]; then - export HADOOP_MAPRED_HOME="${HADOOP_PREFIX}" + [[ -d "${HADOOP_HOME}/${MAPRED_DIR}" ]]; then + export HADOOP_MAPRED_HOME="${HADOOP_HOME}" fi if [[ ! -d "${HADOOP_COMMON_HOME}" ]]; then @@ -573,7 +573,7 @@ function hadoop_basic_init # let's define it as 'hadoop' HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER} HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-hadoop} - HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_PREFIX}/logs"} + HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_HOME}/logs"} HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log} HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO} HADOOP_NICENESS=${HADOOP_NICENESS:-0} @@ -1219,7 +1219,6 @@ function hadoop_finalize_hadoop_opts hadoop_translate_cygwin_path HADOOP_LOG_DIR hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR}" hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE}" - HADOOP_HOME=${HADOOP_PREFIX} hadoop_translate_cygwin_path HADOOP_HOME export HADOOP_HOME hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}" @@ -1252,11 +1251,11 @@ function hadoop_finalize_catalina_opts local prefix=${HADOOP_CATALINA_PREFIX} - hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}" + hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}" if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}" fi - hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_PREFIX}" + hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_HOME}" hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}" hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}" hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}" @@ -1282,7 +1281,7 @@ function hadoop_finalize hadoop_finalize_hadoop_heap hadoop_finalize_hadoop_opts - hadoop_translate_cygwin_path HADOOP_PREFIX + hadoop_translate_cygwin_path HADOOP_HOME hadoop_translate_cygwin_path HADOOP_CONF_DIR hadoop_translate_cygwin_path HADOOP_COMMON_HOME hadoop_translate_cygwin_path HADOOP_HDFS_HOME diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example index faa431740c0..efba10cc1ec 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example @@ -26,8 +26,8 @@ ## ## If you move HADOOP_LIBEXEC_DIR from some location that ## isn't bin/../libexec, you MUST define either HADOOP_LIBEXEC_DIR -## or have HADOOP_PREFIX/libexec/hadoop-config.sh and -## HADOOP_PREFIX/libexec/hadoop-layout.sh (this file) exist. +## or have HADOOP_HOME/libexec/hadoop-config.sh and +## HADOOP_HOME/libexec/hadoop-layout.sh (this file) exist. ## NOTE: ## @@ -44,7 +44,7 @@ #### # Default location for the common/core Hadoop project -# export HADOOP_COMMON_HOME=${HADOOP_PREFIX} +# export HADOOP_COMMON_HOME=${HADOOP_HOME} # Relative locations where components under HADOOP_COMMON_HOME are located # export HADOOP_COMMON_DIR="share/hadoop/common" @@ -56,7 +56,7 @@ #### # Default location for the HDFS subproject -# export HADOOP_HDFS_HOME=${HADOOP_PREFIX} +# export HADOOP_HDFS_HOME=${HADOOP_HOME} # Relative locations where components under HADOOP_HDFS_HOME are located # export HDFS_DIR="share/hadoop/hdfs" @@ -67,7 +67,7 @@ #### # Default location for the YARN subproject -# export HADOOP_YARN_HOME=${HADOOP_PREFIX} +# export HADOOP_YARN_HOME=${HADOOP_HOME} # Relative locations where components under HADOOP_YARN_HOME are located # export YARN_DIR="share/hadoop/yarn" @@ -78,7 +78,7 @@ #### # Default location for the MapReduce subproject -# export HADOOP_MAPRED_HOME=${HADOOP_PREFIX} +# export HADOOP_MAPRED_HOME=${HADOOP_HOME} # Relative locations where components under HADOOP_MAPRED_HOME are located # export MAPRED_DIR="share/hadoop/mapreduce" @@ -92,6 +92,6 @@ # note that this path only gets added for certain commands and not # part of the general classpath unless HADOOP_OPTIONAL_TOOLS is used # to configure them in -# export HADOOP_TOOLS_HOME=${HADOOP_PREFIX} +# export HADOOP_TOOLS_HOME=${HADOOP_HOME} # export HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"} # export HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"} diff --git a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh index 5859da03b2d..34bf0ebb2b8 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh @@ -22,7 +22,7 @@ # # HADOOP_SLAVES File naming remote hosts. # Default is ${HADOOP_CONF_DIR}/slaves. -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. # HADOOP_SSH_OPTS Options passed to ssh when running remote commands. ## @@ -33,8 +33,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh index 845ca378cc9..142064209d3 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh @@ -21,8 +21,8 @@ exit 1 # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh index df7ae8d7c5e..ee1f6eb5691 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh @@ -22,8 +22,8 @@ echo "This script is deprecated. Use stop-dfs.sh and stop-yarn.sh instead." exit 1 # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index 3c554aa8d02..3f19e459c4d 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -55,14 +55,14 @@ # Location of Hadoop. By default, Hadoop will attempt to determine # this location based upon its execution path. -# export HADOOP_PREFIX= +# export HADOOP_HOME= # Location of Hadoop's configuration information. i.e., where this # file is probably living. Many sites will also set this in the # same location where JAVA_HOME is defined. If this is not defined # Hadoop will attempt to locate it based upon its execution # path. -# export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop +# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop # The maximum amount of heap to use (Java -Xmx). If no unit # is provided, it will be converted to MB. Daemons will @@ -186,10 +186,10 @@ esac # non-secure) # -# Where (primarily) daemon log files are stored. # $HADOOP_PREFIX/logs -# by default. +# Where (primarily) daemon log files are stored. +# ${HADOOP_HOME}/logs by default. # Java property: hadoop.log.dir -# export HADOOP_LOG_DIR=${HADOOP_PREFIX}/logs +# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs # A string representing this instance of hadoop. $USER by default. # This is used in writing log and pid files, so keep that in mind! diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java index 09acb35bcd7..0ae6d03933f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java @@ -32,7 +32,7 @@ import org.apache.htrace.core.HTraceConfiguration; @InterfaceAudience.Private public class TraceUtils { private static List EMPTY = Collections.emptyList(); - static final String DEFAULT_HADOOP_PREFIX = "hadoop.htrace."; + static final String DEFAULT_HADOOP_TRACE_PREFIX = "hadoop.htrace."; public static HTraceConfiguration wrapHadoopConf(final String prefix, final Configuration conf) { @@ -52,7 +52,7 @@ public class TraceUtils { if (ret != null) { return ret; } - return getInternal(DEFAULT_HADOOP_PREFIX + key); + return getInternal(DEFAULT_HADOOP_TRACE_PREFIX + key); } @Override diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md index bf9419a7ae6..d2479e76110 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md @@ -86,10 +86,10 @@ Other useful configuration parameters that you can customize include: In most cases, you should specify the `HADOOP_PID_DIR` and `HADOOP_LOG_DIR` directories such that they can only be written to by the users that are going to run the hadoop daemons. Otherwise there is the potential for a symlink attack. -It is also traditional to configure `HADOOP_PREFIX` in the system-wide shell environment configuration. For example, a simple script inside `/etc/profile.d`: +It is also traditional to configure `HADOOP_HOME` in the system-wide shell environment configuration. For example, a simple script inside `/etc/profile.d`: - HADOOP_PREFIX=/path/to/hadoop - export HADOOP_PREFIX + HADOOP_HOME=/path/to/hadoop + export HADOOP_HOME | Daemon | Environment Variable | |:---- |:---- | @@ -243,73 +243,73 @@ To start a Hadoop cluster you will need to start both the HDFS and YARN cluster. The first time you bring up HDFS, it must be formatted. Format a new distributed filesystem as *hdfs*: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format + [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format Start the HDFS NameNode with the following command on the designated node as *hdfs*: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start namenode Start a HDFS DataNode with the following command on each designated node as *hdfs*: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start datanode + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start datanode If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes can be started with a utility script. As *hdfs*: - [hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh + [hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh Start the YARN with the following command, run on the designated ResourceManager as *yarn*: - [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start resourcemanager + [yarn]$ $HADOOP_HOME/bin/yarn --daemon start resourcemanager Run a script to start a NodeManager on each designated host as *yarn*: - [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start nodemanager + [yarn]$ $HADOOP_HOME/bin/yarn --daemon start nodemanager Start a standalone WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them: - [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start proxyserver + [yarn]$ $HADOOP_HOME/bin/yarn --daemon start proxyserver If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be started with a utility script. As *yarn*: - [yarn]$ $HADOOP_PREFIX/sbin/start-yarn.sh + [yarn]$ $HADOOP_HOME/sbin/start-yarn.sh Start the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*: - [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon start historyserver + [mapred]$ $HADOOP_HOME/bin/mapred --daemon start historyserver ### Hadoop Shutdown Stop the NameNode with the following command, run on the designated NameNode as *hdfs*: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop namenode + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop namenode Run a script to stop a DataNode as *hdfs*: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop datanode + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop datanode If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes may be stopped with a utility script. As *hdfs*: - [hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh + [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh Stop the ResourceManager with the following command, run on the designated ResourceManager as *yarn*: - [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop resourcemanager + [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop resourcemanager Run a script to stop a NodeManager on a slave as *yarn*: - [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop nodemanager + [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop nodemanager If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be stopped with a utility script. As *yarn*: - [yarn]$ $HADOOP_PREFIX/sbin/stop-yarn.sh + [yarn]$ $HADOOP_HOME/sbin/stop-yarn.sh Stop the WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them: - [yarn]$ $HADOOP_PREFIX/bin/yarn stop proxyserver + [yarn]$ $HADOOP_HOME/bin/yarn stop proxyserver Stop the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*: - [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon stop historyserver + [mapred]$ $HADOOP_HOME/bin/mapred --daemon stop historyserver Web Interfaces -------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md index 59ea19896bc..365a8448e69 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md @@ -39,7 +39,7 @@ All of the shell commands will accept a common set of options. For some commands | SHELL\_OPTION | Description | |:---- |:---- | | `--buildpaths` | Enables developer versions of jars. | -| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_PREFIX/etc/hadoop`. | +| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_HOME/etc/hadoop`. | | `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. For commands that do not support daemonization, this option is ignored. | | `--debug` | Enables shell level configuration debugging information | | `--help` | Shell script usage information. | diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md index ffab5a0a93f..caa3aa761a2 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md @@ -83,7 +83,7 @@ Apache Hadoop allows for third parties to easily add new features through a vari Core to this functionality is the concept of a shell profile. Shell profiles are shell snippets that can do things such as add jars to the classpath, configure Java system properties and more. -Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_PREFIX}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime. +Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_HOME}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime. An example of a shell profile is in the libexec directory. diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash index f7183455e9a..be2d7f58b27 100755 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash @@ -27,6 +27,7 @@ setup() { # shellcheck disable=SC2034 HADOOP_SHELL_SCRIPT_DEBUG=true unset HADOOP_CONF_DIR + # we unset both of these for bw compat unset HADOOP_HOME unset HADOOP_PREFIX @@ -53,4 +54,4 @@ strstr() { else echo false fi -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats index ae20248dadf..79ede4273f4 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats @@ -45,7 +45,7 @@ basicinitsetup () { unset ${j} done - HADOOP_PREFIX=${TMP} + HADOOP_HOME=${TMP} } check_var_values () { diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats index 9114c707336..de4edd493a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats @@ -22,7 +22,7 @@ load hadoop-functions_test_helper } @test "hadoop_bootstrap (libexec)" { - unset HADOOP_PREFIX + unset HADOOP_HOME unset HADOOP_COMMON_DIR unset HADOOP_COMMON_LIB_JARS_DIR unset HDFS_DIR @@ -39,7 +39,7 @@ load hadoop-functions_test_helper hadoop_bootstrap # all of these should be set - [ -n ${HADOOP_PREFIX} ] + [ -n ${HADOOP_HOME} ] [ -n ${HADOOP_COMMON_DIR} ] [ -n ${HADOOP_COMMON_LIB_JARS_DIR} ] [ -n ${HDFS_DIR} ] diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats index 3e42da936c5..1f0c706ca7d 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats @@ -16,10 +16,10 @@ load hadoop-functions_test_helper create_fake_dirs () { - HADOOP_PREFIX=${TMP} + HADOOP_HOME=${TMP} for j in conf etc/hadoop; do - mkdir -p "${HADOOP_PREFIX}/${j}" - echo "unittest=${j}" > "${HADOOP_PREFIX}/${j}/hadoop-env.sh" + mkdir -p "${HADOOP_HOME}/${j}" + echo "unittest=${j}" > "${HADOOP_HOME}/${j}/hadoop-env.sh" done } @@ -32,27 +32,27 @@ create_fake_dirs () { @test "hadoop_find_confdir (bw compat: conf)" { create_fake_dirs hadoop_find_confdir - echo ">${HADOOP_CONF_DIR}< >${HADOOP_PREFIX}/conf<" - [ "${HADOOP_CONF_DIR}" = ${HADOOP_PREFIX}/conf ] + echo ">${HADOOP_CONF_DIR}< >${HADOOP_HOME}/conf<" + [ "${HADOOP_CONF_DIR}" = ${HADOOP_HOME}/conf ] } @test "hadoop_find_confdir (etc/hadoop)" { create_fake_dirs - rm -rf "${HADOOP_PREFIX}/conf" + rm -rf "${HADOOP_HOME}/conf" hadoop_find_confdir - [ "${HADOOP_CONF_DIR}" = ${HADOOP_PREFIX}/etc/hadoop ] + [ "${HADOOP_CONF_DIR}" = ${HADOOP_HOME}/etc/hadoop ] } @test "hadoop_verify_confdir (negative) " { create_fake_dirs - HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf + HADOOP_CONF_DIR=${HADOOP_HOME}/conf run hadoop_verify_confdir [ -n "${output}" ] } @test "hadoop_verify_confdir (positive) " { create_fake_dirs - HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf + HADOOP_CONF_DIR=${HADOOP_HOME}/conf touch "${HADOOP_CONF_DIR}/log4j.properties" run hadoop_verify_confdir [ -z "${output}" ] @@ -60,7 +60,7 @@ create_fake_dirs () { @test "hadoop_exec_hadoopenv (positive) " { create_fake_dirs - HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf + HADOOP_CONF_DIR=${HADOOP_HOME}/conf hadoop_exec_hadoopenv [ -n "${HADOOP_ENV_PROCESSED}" ] [ "${unittest}" = conf ] @@ -68,7 +68,7 @@ create_fake_dirs () { @test "hadoop_exec_hadoopenv (negative) " { create_fake_dirs - HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf + HADOOP_CONF_DIR=${HADOOP_HOME}/conf HADOOP_ENV_PROCESSED=true hadoop_exec_hadoopenv [ -z "${unittest}" ] @@ -76,7 +76,7 @@ create_fake_dirs () { @test "hadoop_exec_userfuncs" { create_fake_dirs - HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf + HADOOP_CONF_DIR=${HADOOP_HOME}/conf echo "unittest=userfunc" > "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" hadoop_exec_userfuncs [ "${unittest}" = "userfunc" ] diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats index 668c115825b..b9339f3c569 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats @@ -100,7 +100,7 @@ load hadoop-functions_test_helper hadoop_finalize_hadoop_heap () { true; } hadoop_finalize_hadoop_opts () { true; } hadoop_translate_cygwin_path () { - if [ $1 = HADOOP_PREFIX ]; then + if [ $1 = HADOOP_HOME ]; then testvar=prefix; fi } diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh index c3bc772b261..7044fa86704 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh @@ -20,7 +20,7 @@ # KMS temporary directory # -# export KMS_TEMP=${HADOOP_PREFIX}/temp +# export KMS_TEMP=${HADOOP_HOME}/temp # The HTTP port used by KMS # @@ -59,7 +59,7 @@ # # Location of tomcat # -# export KMS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/kms/tomcat +# export KMS_CATALINA_HOME=${HADOOP_HOME}/share/hadoop/kms/tomcat # Java System properties for KMS should be specified in this variable. # The java.library.path and hadoop.home.dir properties are automatically diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh index c88aa879625..5e1ffa40c9d 100644 --- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh +++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh @@ -28,7 +28,7 @@ function hadoop_subproject_init export HADOOP_CATALINA_PREFIX=kms - export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_PREFIX}/temp}" + export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_HOME}/temp}" hadoop_deprecate_envvar KMS_CONFIG HADOOP_CONF_DIR @@ -49,7 +49,7 @@ function hadoop_subproject_init # shellcheck disable=SC2086 export KMS_SSL_TRUSTSTORE_PASS=${KMS_SSL_TRUSTSTORE_PASS:-"$(echo ${CATALINA_OPTS} | grep -o 'trustStorePassword=[^ ]*' | cut -f2 -d= )"} - export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/kms/tomcat}" + export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_HOME}/share/hadoop/kms/tomcat}" export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}" export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}" @@ -69,8 +69,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] && . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then - . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "ERROR: Hadoop common not found." 2>&1 exit 1 diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh index 4ed4725aeeb..6708cd917db 100755 --- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh +++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh @@ -30,8 +30,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh index a4edef625f7..f01245372f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh @@ -20,7 +20,7 @@ # HTTPFS temporary directory # -# export HTTPFS_TEMP=${HADOOP_PREFIX}/temp +# export HTTPFS_TEMP=${HADOOP_HOME}/temp # The HTTP port used by HTTPFS # @@ -53,7 +53,7 @@ # # Location of tomcat # -# export HTTPFS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat +# export HTTPFS_CATALINA_HOME=${HADOOP_HOME}/share/hadoop/httpfs/tomcat # Java System properties for HTTPFS should be specified in this variable. # The java.library.path and hadoop.home.dir properties are automatically diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh index 767bd6e70db..ba4b4068dfc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh @@ -28,7 +28,7 @@ function hadoop_subproject_init export HADOOP_CATALINA_PREFIX=httpfs - export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_PREFIX}/temp}" + export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_HOME}/temp}" hadoop_deprecate_envvar HTTPFS_CONFIG HADOOP_CONF_DIR @@ -47,7 +47,7 @@ function hadoop_subproject_init export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${HTTPFS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}" - export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat}" + export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_HOME}/share/hadoop/httpfs/tomcat}" export HADOOP_CATALINA_HOME="${HTTPFS_CATALINA_HOME:-${CATALINA_BASE}}" export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out}" @@ -67,8 +67,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] && . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then - . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "ERROR: Hadoop common not found." 2>&1 exit 1 diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh index 18c1af0a382..3e7cdf8d698 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh @@ -30,8 +30,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README index 672265e1a71..e8cc0e509f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README @@ -35,9 +35,9 @@ Requirements BUILDING fuse-dfs executable can be built by setting `require.fuse` option to true using Maven. For example: - in HADOOP_PREFIX: `mvn package -Pnative -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true` + in HADOOP_HOME: `mvn package -Pnative -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true` - The executable `fuse_dfs` will be located at HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/ + The executable `fuse_dfs` will be located at HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/ Common build problems include not finding the libjvm.so in JAVA_HOME/jre/lib/OS_ARCH/server or not finding fuse in FUSE_HOME or /usr/local. @@ -109,7 +109,7 @@ NOTE - you cannot export this with a FUSE module built into the kernel RECOMMENDATIONS -1. From /bin, `ln -s HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs* .` +1. From /bin, `ln -s HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs* .` 2. Always start with debug on so you can see if you are missing a classpath or something like that. diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh index 26dfd19005f..c52c5f96065 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh @@ -16,12 +16,12 @@ # limitations under the License. # -if [ "$HADOOP_PREFIX" = "" ]; then - echo "HADOOP_PREFIX is empty. Set it to the root directory of Hadoop source code" +if [ "$HADOOP_HOME" = "" ]; then + echo "HADOOP_HOME is empty. Set it to the root directory of Hadoop source code" exit 1 fi -export FUSEDFS_PATH="$HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs" -export LIBHDFS_PATH="$HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/usr/local/lib" +export FUSEDFS_PATH="$HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs" +export LIBHDFS_PATH="$HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/usr/local/lib" if [ "$OS_ARCH" = "" ]; then export OS_ARCH=amd64 @@ -38,12 +38,12 @@ fi while IFS= read -r -d '' file do export CLASSPATH=$CLASSPATH:$file -done < <(find "$HADOOP_PREFIX/hadoop-client" -name "*.jar" -print0) +done < <(find "$HADOOP_HOME/hadoop-client" -name "*.jar" -print0) while IFS= read -r -d '' file do export CLASSPATH=$CLASSPATH:$file -done < <(find "$HADOOP_PREFIX/hhadoop-hdfs-project" -name "*.jar" -print0) +done < <(find "$HADOOP_HOME/hhadoop-hdfs-project" -name "*.jar" -print0) export CLASSPATH=$HADOOP_CONF_DIR:$CLASSPATH export PATH=$FUSEDFS_PATH:$PATH diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh index cfd44e390a9..97f04f708a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh @@ -52,8 +52,8 @@ if [ ! -f "$excludeFilenameLocal" ] ; then exit 1 fi -namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -namenodes) -excludeFilenameRemote=$("$HADOOP_PREFIX/bin/hdfs" getconf -excludeFile) +namenodes=$("$HADOOP_HOME/bin/hdfs" getconf -namenodes) +excludeFilenameRemote=$("$HADOOP_HOME/bin/hdfs" getconf -excludeFile) if [ "$excludeFilenameRemote" = '' ] ; then echo \ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index bcd04d16085..c365250f59f 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -60,8 +60,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh index 244e5a9da5f..d440210992f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh @@ -49,7 +49,7 @@ function hadoop_subproject_init hadoop_deprecate_envvar HADOOP_HDFS_IDENT_STRING HADOOP_IDENT_STRING - HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_PREFIX}" + HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME}" # turn on the defaults export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} @@ -71,8 +71,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] && . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then - . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" +elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "ERROR: Hadoop common not found." 2>&1 exit 1 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh index 318a2826a01..f51dd0fc57e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh @@ -21,8 +21,8 @@ # for dfsadmin to support multiple namenodes. # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh index 32ca2b23d48..df044fe48ba 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh @@ -30,8 +30,8 @@ function hadoop_usage bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh index 9c5a172c874..1e35e7d8a67 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh @@ -29,8 +29,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh index f904640d549..3fce34572de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh @@ -26,8 +26,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh index bb51a8a5a17..ec94080ece5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh @@ -28,8 +28,8 @@ function hadoop_usage bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh index cc0d11d6522..e6933742113 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh @@ -28,8 +28,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh index 816a3e31c97..2a973b1e839 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh @@ -26,8 +26,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md index 38c10703a84..99a41a29010 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md @@ -150,13 +150,13 @@ Here is an example configuration with two Namenodes: **Step 1**: Format a Namenode using the following command: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format [-clusterId ] + [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format [-clusterId ] Choose a unique cluster\_id which will not conflict other clusters in your environment. If a cluster\_id is not provided, then a unique one is auto generated. **Step 2**: Format additional Namenodes using the following command: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format -clusterId + [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format -clusterId Note that the cluster\_id in step 2 must be same as that of the cluster\_id in step 1. If they are different, the additional Namenodes will not be part of the federated cluster. @@ -164,7 +164,7 @@ Note that the cluster\_id in step 2 must be same as that of the cluster\_id in s Older releases only support a single Namenode. Upgrade the cluster to newer release in order to enable federation During upgrade you can provide a ClusterID as follows: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode -upgrade -clusterId + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start namenode -upgrade -clusterId If cluster\_id is not provided, it is auto generated. @@ -187,7 +187,7 @@ Perform the following steps: * Refresh the Datanodes to pickup the newly added Namenode by running the following command against all the Datanodes in the cluster: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNamenodes : + [hdfs]$ $HADOOP_HOME/bin/hdfs dfsadmin -refreshNamenodes : Managing the cluster -------------------- @@ -196,11 +196,11 @@ Managing the cluster To start the cluster run the following command: - [hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh + [hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh To stop the cluster run the following command: - [hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh + [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh These commands can be run from any node where the HDFS configuration is available. The command uses the configuration to determine the Namenodes in the cluster and then starts the Namenode process on those nodes. The Datanodes are started on the nodes specified in the `slaves` file. The script can be used as a reference for building your own scripts to start and stop the cluster. @@ -208,7 +208,7 @@ These commands can be run from any node where the HDFS configuration is availabl The Balancer has been changed to work with multiple Namenodes. The Balancer can be run using the command: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start balancer [-policy ] + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start balancer [-policy ] The policy parameter can be any of the following: @@ -227,11 +227,11 @@ Decommissioning is similar to prior releases. The nodes that need to be decomiss **Step 1**: To distribute an exclude file to all the Namenodes, use the following command: - [hdfs]$ $HADOOP_PREFIX/sbin/distribute-exclude.sh + [hdfs]$ $HADOOP_HOME/sbin/distribute-exclude.sh **Step 2**: Refresh all the Namenodes to pick up the new exclude file: - [hdfs]$ $HADOOP_PREFIX/sbin/refresh-namenodes.sh + [hdfs]$ $HADOOP_HOME/sbin/refresh-namenodes.sh The above command uses HDFS configuration to determine the configured Namenodes in the cluster and refreshes them to pick up the new exclude file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md index 51a88c9fba8..f888966ed0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md @@ -475,7 +475,7 @@ There are also several other configuration parameters which may be set to contro After the configuration keys have been added, the next step is to initialize required state in ZooKeeper. You can do so by running the following command from one of the NameNode hosts. - [hdfs]$ $HADOOP_PREFIX/bin/zkfc -formatZK + [hdfs]$ $HADOOP_HOME/bin/zkfc -formatZK This will create a znode in ZooKeeper inside of which the automatic failover system stores its data. @@ -487,7 +487,7 @@ Since automatic failover has been enabled in the configuration, the `start-dfs.s If you manually manage the services on your cluster, you will need to manually start the `zkfc` daemon on each of the machines that runs a NameNode. You can start the daemon by running: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start zkfc ### Securing access to ZooKeeper diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md index 8b42386046c..9a97add2b22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md @@ -523,7 +523,7 @@ There are also several other configuration parameters which may be set to contro After the configuration keys have been added, the next step is to initialize required state in ZooKeeper. You can do so by running the following command from one of the NameNode hosts. - [hdfs]$ $HADOOP_PREFIX/bin/hdfs zkfc -formatZK + [hdfs]$ $HADOOP_HOME/bin/hdfs zkfc -formatZK This will create a znode in ZooKeeper inside of which the automatic failover system stores its data. @@ -535,7 +535,7 @@ Since automatic failover has been enabled in the configuration, the `start-dfs.s If you manually manage the services on your cluster, you will need to manually start the `zkfc` daemon on each of the machines that runs a NameNode. You can start the daemon by running: - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start zkfc ### Securing access to ZooKeeper diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md index 7dc2fe46fe1..67311891586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md @@ -215,7 +215,7 @@ Three daemons are required to provide NFS service: rpcbind (or portmap), mountd 2. Start Hadoop's portmap (needs root privileges): - [root]> $HADOOP_PREFIX/bin/hdfs --daemon start portmap + [root]> $HADOOP_HOME/bin/hdfs --daemon start portmap 3. Start mountd and nfsd. @@ -224,12 +224,12 @@ Three daemons are required to provide NFS service: rpcbind (or portmap), mountd While in secure mode, any user can start NFS gateway as long as the user has read access to the Kerberos keytab defined in "nfs.keytab.file". - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start nfs3 + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start nfs3 4. Stop NFS gateway services. - [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop nfs3 - [root]> $HADOOP_PREFIX/bin/hdfs --daemon stop portmap + [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop nfs3 + [root]> $HADOOP_HOME/bin/hdfs --daemon stop portmap Optionally, you can forgo running the Hadoop-provided portmap daemon and instead use the system portmap daemon on all operating systems if you start the NFS Gateway as root. This will allow the HDFS NFS Gateway to work around the aforementioned bug and still register using the system portmap daemon. To do so, just start the NFS gateway daemon as you normally would, but make sure to do so as the "root" user, and also set the "HADOOP\_PRIVILEGED\_NFS\_USER" environment variable to an unprivileged user. In this mode the NFS Gateway will start as root to perform its initial registration with the system portmap, and then will drop privileges back to the user specified by the HADOOP\_PRIVILEGED\_NFS\_USER afterward and for the rest of the duration of the lifetime of the NFS Gateway process. Note that if you choose this route, you should skip steps 1 and 2 above. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java index 198dafb3412..71c9c567125 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java @@ -62,7 +62,7 @@ public class TestTraceAdmin { public void testCreateAndDestroySpanReceiver() throws Exception { Configuration conf = new Configuration(); conf = new Configuration(); - conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX + + conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX + Tracer.SPAN_RECEIVER_CLASSES_KEY, ""); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java index 37c09d1890e..b3cf4021107 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java @@ -65,10 +65,10 @@ public class TestTracingShortCircuitLocalRead { public void testShortCircuitTraceHooks() throws IOException { assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS); conf = new Configuration(); - conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX + + conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX + Tracer.SPAN_RECEIVER_CLASSES_KEY, SetSpanReceiver.class.getName()); - conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX + + conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX + Tracer.SAMPLER_CLASSES_KEY, "AlwaysSampler"); conf.setLong("dfs.blocksize", 100 * 1024); diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index fab5b87f2e0..f280f311f2b 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -37,8 +37,8 @@ function hadoop_usage bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh index fe6e9e6cfed..a9897929c59 100644 --- a/hadoop-mapreduce-project/bin/mapred-config.sh +++ b/hadoop-mapreduce-project/bin/mapred-config.sh @@ -47,7 +47,7 @@ function hadoop_subproject_init hadoop_deprecate_envvar HADOOP_MAPRED_ROOT_LOGGER HADOOP_ROOT_LOGGER - HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}" + HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_HOME}" hadoop_deprecate_envvar HADOOP_MAPRED_IDENT_STRING HADOOP_IDENT_STRING } @@ -62,8 +62,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] && . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then - . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" +elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "ERROR: Hadoop common not found." 2>&1 exit 1 diff --git a/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh b/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh index 57b1ebd01d7..998ca90dca7 100644 --- a/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh +++ b/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh @@ -21,8 +21,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java index 4f5b6a1c9c4..ae457829ad1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java @@ -315,7 +315,7 @@ public class Submitter extends Configured implements Tool { // # if (exec.contains("#")) { // set default gdb commands for map and reduce task - String defScript = "$HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-script"; + String defScript = "$HADOOP_HOME/src/c++/pipes/debug/pipes-default-script"; setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java index 12bec0869f6..b01954e1c87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java @@ -450,7 +450,7 @@ public class DFSCIOTest { } //Copy the executables over to the remote filesystem - String hadoopHome = System.getenv("HADOOP_PREFIX"); + String hadoopHome = System.getenv("HADOOP_HOME"); fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/libhdfs.so." + HDFS_LIB_VERSION), HDFS_SHLIB); fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_read"), HDFS_READ); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java index e6e12ebbe4f..ecac83af5b3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java @@ -179,7 +179,7 @@ public class ReliabilityTest extends Configured implements Tool { private String normalizeCommandPath(String command) { final String hadoopHome; - if ((hadoopHome = System.getenv("HADOOP_PREFIX")) != null) { + if ((hadoopHome = System.getenv("HADOOP_HOME")) != null) { command = hadoopHome + "/" + command; } return command; diff --git a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java index 6b8af97e9c4..c502ffd173d 100644 --- a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java @@ -450,7 +450,7 @@ public class HadoopArchiveLogs implements Tool { fi export HADOOP_CLIENT_OPTS="-Xmx1024m" export HADOOP_CLASSPATH=/dist/share/hadoop/tools/lib/hadoop-archive-logs-2.8.0-SNAPSHOT.jar:/dist/share/hadoop/tools/lib/hadoop-archives-2.8.0-SNAPSHOT.jar - "$HADOOP_PREFIX"/bin/hadoop org.apache.hadoop.tools.HadoopArchiveLogsRunner -appId "$appId" -user "$user" -workingDir /tmp/logs/archive-logs-work -remoteRootLogDir /tmp/logs -suffix logs + "$HADOOP_HOME"/bin/hadoop org.apache.hadoop.tools.HadoopArchiveLogsRunner -appId "$appId" -user "$user" -workingDir /tmp/logs/archive-logs-work -remoteRootLogDir /tmp/logs -suffix logs */ @VisibleForTesting void generateScript(File localScript, Path workingDir, @@ -484,7 +484,7 @@ public class HadoopArchiveLogs implements Tool { fw.write("m\"\n"); fw.write("export HADOOP_CLASSPATH="); fw.write(classpath); - fw.write("\n\"$HADOOP_PREFIX\"/bin/hadoop "); + fw.write("\n\"$HADOOP_HOME\"/bin/hadoop "); fw.write(HadoopArchiveLogsRunner.class.getName()); fw.write(" -appId \"$appId\" -user \"$user\" -workingDir "); fw.write(workingDir.toString()); diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java index 7fcb0bfad80..d2d7801caf2 100644 --- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java @@ -294,14 +294,14 @@ public class TestHadoopArchiveLogs { Assert.assertTrue(lines[14].startsWith("export HADOOP_CLASSPATH=")); if (proxy) { Assert.assertEquals( - "\"$HADOOP_PREFIX\"/bin/hadoop org.apache.hadoop.tools." + + "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir " + workingDir.toString() + " -remoteRootLogDir " + remoteRootLogDir.toString() + " -suffix " + suffix, lines[15]); } else { Assert.assertEquals( - "\"$HADOOP_PREFIX\"/bin/hadoop org.apache.hadoop.tools." + + "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." + "HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " + "-workingDir " + workingDir.toString() + " -remoteRootLogDir " + remoteRootLogDir.toString() + " -suffix " + suffix + " -noProxy", diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt index 73fd6ef15a8..47ef31c4677 100644 --- a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt +++ b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt @@ -20,7 +20,7 @@ B.a31 B.a32 ***************************** *** Invoke SampleDataJoin *** ***************************** -[:~]$ $HADOOP_PREFIX/bin/hadoop jar hadoop-datajoin-examples.jar org.apache.hadoop.contrib.utils.join.DataJoinJob datajoin/input datajoin/output Text 1 org.apache.hadoop.contrib.utils.join.SampleDataJoinMapper org.apache.hadoop.contrib.utils.join.SampleDataJoinReducer org.apache.hadoop.contrib.utils.join.SampleTaggedMapOutput Text +[:~]$ $HADOOP_HOME/bin/hadoop jar hadoop-datajoin-examples.jar org.apache.hadoop.contrib.utils.join.DataJoinJob datajoin/input datajoin/output Text 1 org.apache.hadoop.contrib.utils.join.SampleDataJoinMapper org.apache.hadoop.contrib.utils.join.SampleDataJoinReducer org.apache.hadoop.contrib.utils.join.SampleTaggedMapOutput Text Using TextInputFormat: Text Using TextOutputFormat: Text 07/06/01 19:58:23 INFO mapred.FileInputFormat: Total input paths to process : 2 diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script index 6bacc437e43..7b74fb67a1f 100644 --- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script +++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -9,6 +10,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -core=`find . -name 'core*'` +core=$(find . -name 'core*') #Only pipes programs have 5th argument as program name. -gdb -quiet $5 -c $core -x $HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-gdb-commands.txt +gdb -quiet "${5}" -c "${core}" -x "${HADOOP_HOME}/src/c++/pipes/debug/pipes-default-gdb-commands.txt" diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh index f9bfaef2db0..0bd291bb8f3 100644 --- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh +++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh @@ -77,8 +77,8 @@ function run_sls_generator() } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh index 30fd60a4440..403c4bb05f6 100644 --- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh +++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh @@ -71,8 +71,8 @@ function parse_args() function calculate_classpath { hadoop_add_to_classpath_tools hadoop-sls - hadoop_debug "Injecting ${HADOOP_PREFIX}/share/hadoop/tools/sls/html into CLASSPATH" - hadoop_add_classpath "${HADOOP_PREFIX}/share/hadoop/tools/sls/html" + hadoop_debug "Injecting ${HADOOP_TOOLS_DIR}/sls/html into CLASSPATH" + hadoop_add_classpath "${HADOOP_TOOLS_DIR}/sls/html" } function run_simulation() { @@ -105,8 +105,8 @@ function run_simulation() { } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java index 42007a07560..5a07cc325ca 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java @@ -91,7 +91,7 @@ public class DumpTypedBytes implements Tool { } private void printUsage() { - System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar" + System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar" + " dumptb "); System.out.println(" Dumps all files that match the given pattern to " + "standard output as typed bytes."); diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java index 5d0112458ff..eabf46c83c0 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java @@ -56,7 +56,7 @@ public class HadoopStreaming { } private static void printUsage() { - System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar" + System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar" + " [options]"); System.out.println("Options:"); System.out.println(" dumptb Dumps all files that match the" diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java index 6470393cba5..a7a001cff6c 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java @@ -89,7 +89,7 @@ public class LoadTypedBytes implements Tool { } private void printUsage() { - System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar" + System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar" + " loadtb "); System.out.println(" Reads typed bytes from standard input" + " and stores them in a sequence file in"); diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java index 118e0fb8998..9f5b293b369 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java @@ -502,7 +502,7 @@ public class StreamJob implements Tool { } private void printUsage(boolean detailed) { - System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar" + System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar" + " [options]"); System.out.println("Options:"); System.out.println(" -input DFS input file(s) for the Map" @@ -551,7 +551,7 @@ public class StreamJob implements Tool { System.out.println(); System.out.println("For more details about these options:"); System.out.println("Use " + - "$HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar -info"); + "$HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar -info"); return; } System.out.println(); @@ -611,7 +611,7 @@ public class StreamJob implements Tool { System.out.println(" -D stream.non.zero.exit.is.failure=false"); System.out.println("Use a custom hadoop streaming build along with standard" + " hadoop install:"); - System.out.println(" $HADOOP_PREFIX/bin/hadoop jar " + + System.out.println(" $HADOOP_HOME/bin/hadoop jar " + "/path/my-hadoop-streaming.jar [...]\\"); System.out.println(" [...] -D stream.shipped.hadoopstreaming=" + "/path/my-hadoop-streaming.jar"); @@ -625,7 +625,7 @@ public class StreamJob implements Tool { System.out.println(" -cmdenv EXAMPLE_DIR=/home/example/dictionaries/"); System.out.println(); System.out.println("Shortcut:"); - System.out.println(" setenv HSTREAMING \"$HADOOP_PREFIX/bin/hadoop jar " + + System.out.println(" setenv HSTREAMING \"$HADOOP_HOME/bin/hadoop jar " + "hadoop-streaming.jar\""); System.out.println(); System.out.println("Example: $HSTREAMING -mapper " + @@ -648,9 +648,9 @@ public class StreamJob implements Tool { // -------------------------------------------- protected String getHadoopClientHome() { - String h = env_.getProperty("HADOOP_PREFIX"); // standard Hadoop + String h = env_.getProperty("HADOOP_HOME"); // standard Hadoop if (h == null) { - //fail("Missing required environment variable: HADOOP_PREFIX"); + //fail("Missing required environment variable: HADOOP_HOME"); h = "UNDEF"; } return h; @@ -674,8 +674,8 @@ public class StreamJob implements Tool { // usually found in: build/contrib or build/hadoop--dev-streaming.jar // First try an explicit spec: it's too hard to find our own location in this case: - // $HADOOP_PREFIX/bin/hadoop jar /not/first/on/classpath/custom-hadoop-streaming.jar - // where findInClasspath() would find the version of hadoop-streaming.jar in $HADOOP_PREFIX + // $HADOOP_HOME/bin/hadoop jar /not/first/on/classpath/custom-hadoop-streaming.jar + // where findInClasspath() would find the version of hadoop-streaming.jar in $HADOOP_HOME String runtimeClasses = config_.get("stream.shipped.hadoopstreaming"); // jar or class dir if (runtimeClasses == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh index f462fad61df..3b41299630d 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh @@ -26,8 +26,8 @@ function hadoop_usage bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh index 33059894dc7..358f0c90118 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh @@ -26,8 +26,8 @@ function hadoop_usage bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index cb2364b3957..cac3bb6c0cc 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -51,8 +51,8 @@ function hadoop_usage # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh index 7df9fa1019e..d7fa4066f60 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh @@ -53,7 +53,7 @@ function hadoop_subproject_init hadoop_deprecate_envvar YARN_SLAVES HADOOP_SLAVES - HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}" + HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_HOME}" # YARN-1429 added the completely superfluous YARN_USER_CLASSPATH # env var. We're going to override HADOOP_USER_CLASSPATH to keep @@ -74,8 +74,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] && . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then - . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" +elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else echo "ERROR: Hadoop common not found." 2>&1 exit 1 diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh index 2f886f2a473..a195c60d006 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh @@ -21,8 +21,8 @@ function hadoop_usage } # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh index 4ef08648245..958c8bd9754 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh @@ -25,8 +25,8 @@ this="${BASH_SOURCE-$0}" bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) # let's locate libexec... -if [[ -n "${HADOOP_PREFIX}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java index c4500c4f88e..27ff4383e27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java @@ -169,7 +169,7 @@ public class TestDockerContainerExecutorWithMocks { String appSubmitter = "nobody"; String appId = "APP_ID"; String containerId = "CONTAINER_ID"; - String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*"; + String testImage = "testrepo.com/test-image rm -rf $HADOOP_HOME/*"; Container container = mock(Container.class, RETURNS_DEEP_STUBS); ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm index fbfe04b8c25..6a7db436ec5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm @@ -142,7 +142,7 @@ Step 2. Pick a custom Docker image if you want. In this example, we'll use seque Step 3. Run. ```bash -hadoop jar $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar \ +hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar \ teragen \ -Dmapreduce.map.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \ -Dyarn.app.mapreduce.am.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \ From d95c6eb32cec7768ac418fb467b1198ccf3cf0dc Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Thu, 31 Mar 2016 08:42:57 -0700 Subject: [PATCH 26/75] HADOOP-12916. Allow RPC scheduler/callqueue backoff using response times. Contributed by Xiaoyu Yao. --- .../org/apache/hadoop/conf/Configuration.java | 13 + .../hadoop/fs/CommonConfigurationKeys.java | 14 +- .../apache/hadoop/ipc/CallQueueManager.java | 124 +++++- .../apache/hadoop/ipc/DecayRpcScheduler.java | 392 ++++++++++++++---- .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 2 + .../hadoop/ipc/DefaultRpcScheduler.java | 45 ++ .../org/apache/hadoop/ipc/FairCallQueue.java | 45 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 8 +- .../org/apache/hadoop/ipc/RpcScheduler.java | 8 +- .../org/apache/hadoop/ipc/Schedulable.java | 5 +- .../java/org/apache/hadoop/ipc/Server.java | 77 +++- .../apache/hadoop/ipc/WritableRpcEngine.java | 45 +- .../hadoop/ipc/TestCallQueueManager.java | 147 ++++++- .../hadoop/ipc/TestDecayRpcScheduler.java | 42 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 79 ++-- .../hadoop/ipc/TestIdentityProviders.java | 18 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 92 +++- 17 files changed, 891 insertions(+), 265 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 8355d9694ee..4c8f27b9802 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1626,6 +1626,10 @@ public class Configuration implements Iterable>, return defaultValue; } vStr = vStr.trim(); + return getTimeDurationHelper(name, vStr, unit); + } + + private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) { ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr); if (null == vUnit) { LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit); @@ -1636,6 +1640,15 @@ public class Configuration implements Iterable>, return unit.convert(Long.parseLong(vStr), vUnit.unit()); } + public long[] getTimeDurations(String name, TimeUnit unit) { + String[] strings = getTrimmedStrings(name); + long[] durations = new long[strings.length]; + for (int i = 0; i < strings.length; i++) { + durations[i] = getTimeDurationHelper(name, strings[i], unit); + } + return durations; + } + /** * Get the value of the name property as a Pattern. * If no such property is specified, or if the specified value is not a valid diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 9b4069a422a..a708900e630 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -90,14 +90,22 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** * CallQueue related settings. These are not used directly, but rather * combined with a namespace and port. For instance: - * IPC_CALLQUEUE_NAMESPACE + ".8020." + IPC_CALLQUEUE_IMPL_KEY + * IPC_NAMESPACE + ".8020." + IPC_CALLQUEUE_IMPL_KEY */ - public static final String IPC_CALLQUEUE_NAMESPACE = "ipc"; + public static final String IPC_NAMESPACE = "ipc"; public static final String IPC_CALLQUEUE_IMPL_KEY = "callqueue.impl"; - public static final String IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY = "identity-provider.impl"; + public static final String IPC_SCHEDULER_IMPL_KEY = "scheduler.impl"; + public static final String IPC_IDENTITY_PROVIDER_KEY = "identity-provider.impl"; public static final String IPC_BACKOFF_ENABLE = "backoff.enable"; public static final boolean IPC_BACKOFF_ENABLE_DEFAULT = false; + /** + * IPC scheduler priority levels. + */ + public static final String IPC_SCHEDULER_PRIORITY_LEVELS_KEY = + "scheduler.priority.levels"; + public static final int IPC_SCHEDULER_PRIORITY_LEVELS_DEFAULT_KEY = 4; + /** This is for specifying the implementation for the mappings from * hostnames to the racks they belong to */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java index 2ee15d3809e..1a7782aa63c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; /** * Abstracts queue operations for different blocking queues. @@ -43,6 +44,13 @@ public class CallQueueManager { Class queueClass, Class elementClass) { return (Class>)queueClass; } + + @SuppressWarnings("unchecked") + static Class convertSchedulerClass( + Class schedulerClass) { + return (Class)schedulerClass; + } + private final boolean clientBackOffEnabled; // Atomic refs point to active callQueue @@ -50,25 +58,76 @@ public class CallQueueManager { private final AtomicReference> putRef; private final AtomicReference> takeRef; + private RpcScheduler scheduler; + public CallQueueManager(Class> backingClass, + Class schedulerClass, boolean clientBackOffEnabled, int maxQueueSize, String namespace, Configuration conf) { + int priorityLevels = parseNumLevels(namespace, conf); + this.scheduler = createScheduler(schedulerClass, priorityLevels, + namespace, conf); BlockingQueue bq = createCallQueueInstance(backingClass, - maxQueueSize, namespace, conf); + priorityLevels, maxQueueSize, namespace, conf); this.clientBackOffEnabled = clientBackOffEnabled; this.putRef = new AtomicReference>(bq); this.takeRef = new AtomicReference>(bq); - LOG.info("Using callQueue " + backingClass); + LOG.info("Using callQueue: " + backingClass + " scheduler: " + + schedulerClass); + } + + private static T createScheduler( + Class theClass, int priorityLevels, String ns, Configuration conf) { + // Used for custom, configurable scheduler + try { + Constructor ctor = theClass.getDeclaredConstructor(int.class, + String.class, Configuration.class); + return ctor.newInstance(priorityLevels, ns, conf); + } catch (RuntimeException e) { + throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); + } catch (Exception e) { + } + + try { + Constructor ctor = theClass.getDeclaredConstructor(int.class); + return ctor.newInstance(priorityLevels); + } catch (RuntimeException e) { + throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); + } catch (Exception e) { + } + + // Last attempt + try { + Constructor ctor = theClass.getDeclaredConstructor(); + return ctor.newInstance(); + } catch (RuntimeException e) { + throw e; + } catch (InvocationTargetException e) { + throw new RuntimeException(theClass.getName() + + " could not be constructed.", e.getCause()); + } catch (Exception e) { + } + + // Nothing worked + throw new RuntimeException(theClass.getName() + + " could not be constructed."); } private > T createCallQueueInstance( - Class theClass, int maxLen, String ns, Configuration conf) { + Class theClass, int priorityLevels, int maxLen, String ns, + Configuration conf) { // Used for custom, configurable callqueues try { - Constructor ctor = theClass.getDeclaredConstructor(int.class, String.class, - Configuration.class); - return ctor.newInstance(maxLen, ns, conf); + Constructor ctor = theClass.getDeclaredConstructor(int.class, + int.class, String.class, Configuration.class); + return ctor.newInstance(priorityLevels, maxLen, ns, conf); } catch (RuntimeException e) { throw e; } catch (InvocationTargetException e) { @@ -110,6 +169,22 @@ public class CallQueueManager { return clientBackOffEnabled; } + // Based on policy to determine back off current call + boolean shouldBackOff(Schedulable e) { + return scheduler.shouldBackOff(e); + } + + void addResponseTime(String name, int priorityLevel, int queueTime, + int processingTime) { + scheduler.addResponseTime(name, priorityLevel, queueTime, processingTime); + } + + // This should be only called once per call and cached in the call object + // each getPriorityLevel call will increment the counter for the caller + int getPriorityLevel(Schedulable e) { + return scheduler.getPriorityLevel(e); + } + /** * Insert e into the backing queue or block until we can. * If we block and the queue changes on us, we will insert while the @@ -146,15 +221,46 @@ public class CallQueueManager { return takeRef.get().size(); } + /** + * Read the number of levels from the configuration. + * This will affect the FairCallQueue's overall capacity. + * @throws IllegalArgumentException on invalid queue count + */ + @SuppressWarnings("deprecation") + private static int parseNumLevels(String ns, Configuration conf) { + // Fair call queue levels (IPC_CALLQUEUE_PRIORITY_LEVELS_KEY) + // takes priority over the scheduler level key + // (IPC_SCHEDULER_PRIORITY_LEVELS_KEY) + int retval = conf.getInt(ns + "." + + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 0); + if (retval == 0) { // No FCQ priority level configured + retval = conf.getInt(ns + "." + + CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY, + CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_DEFAULT_KEY); + } else { + LOG.warn(ns + "." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY + + " is deprecated. Please use " + ns + "." + + CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY + "."); + } + if(retval < 1) { + throw new IllegalArgumentException("numLevels must be at least 1"); + } + return retval; + } + /** * Replaces active queue with the newly requested one and transfers * all calls to the newQ before returning. */ public synchronized void swapQueue( + Class schedulerClass, Class> queueClassToUse, int maxSize, String ns, Configuration conf) { - BlockingQueue newQ = createCallQueueInstance(queueClassToUse, maxSize, - ns, conf); + int priorityLevels = parseNumLevels(ns, conf); + RpcScheduler newScheduler = createScheduler(schedulerClass, priorityLevels, + ns, conf); + BlockingQueue newQ = createCallQueueInstance(queueClassToUse, + priorityLevels, maxSize, ns, conf); // Our current queue becomes the old queue BlockingQueue oldQ = putRef.get(); @@ -168,6 +274,8 @@ public class CallQueueManager { // Swap takeRef to handle new calls takeRef.set(newQ); + this.scheduler = newScheduler; + LOG.info("Old Queue: " + stringRepr(oldQ) + ", " + "Replacement: " + stringRepr(newQ)); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java index a6a14d08beb..42373395ee0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java @@ -27,17 +27,21 @@ import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.AtomicReference; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import com.google.common.util.concurrent.AtomicDoubleArray; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.metrics2.util.MBeans; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The decay RPC scheduler counts incoming requests in a map, then @@ -49,22 +53,28 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean /** * Period controls how many milliseconds between each decay sweep. */ - public static final String IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY = + public static final String IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY = + "decay-scheduler.period-ms"; + public static final long IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT = + 5000; + @Deprecated + public static final String IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY = "faircallqueue.decay-scheduler.period-ms"; - public static final long IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT = - 5000L; /** * Decay factor controls how much each count is suppressed by on each sweep. * Valid numbers are > 0 and < 1. Decay factor works in tandem with period * to control how long the scheduler remembers an identity. */ - public static final String IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY = + public static final String IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY = + "decay-scheduler.decay-factor"; + public static final double IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT = + 0.5; + @Deprecated + public static final String IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY = "faircallqueue.decay-scheduler.decay-factor"; - public static final double IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT = - 0.5; - /** + /** * Thresholds are specified as integer percentages, and specify which usage * range each queue will be allocated to. For instance, specifying the list * 10, 40, 80 @@ -74,15 +84,31 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean * - q1 from 10 up to 40 * - q0 otherwise. */ - public static final String IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY = - "faircallqueue.decay-scheduler.thresholds"; + public static final String IPC_DECAYSCHEDULER_THRESHOLDS_KEY = + "decay-scheduler.thresholds"; + @Deprecated + public static final String IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY = + "faircallqueue.decay-scheduler.thresholds"; // Specifies the identity to use when the IdentityProvider cannot handle // a schedulable. public static final String DECAYSCHEDULER_UNKNOWN_IDENTITY = - "IdentityProvider.Unknown"; + "IdentityProvider.Unknown"; - public static final Log LOG = LogFactory.getLog(DecayRpcScheduler.class); + public static final String + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY = + "decay-scheduler.backoff.responsetime.enable"; + public static final Boolean + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT = false; + + // Specifies the average response time (ms) thresholds of each + // level to trigger backoff + public static final String + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY = + "decay-scheduler.backoff.responsetime.thresholds"; + + public static final Logger LOG = + LoggerFactory.getLogger(DecayRpcScheduler.class); // Track the number of calls for each schedulable identity private final ConcurrentHashMap callCounts = @@ -91,6 +117,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean // Should be the sum of all AtomicLongs in callCounts private final AtomicLong totalCalls = new AtomicLong(); + // Track total call count and response time in current decay window + private final AtomicLongArray responseTimeCountInCurrWindow; + private final AtomicLongArray responseTimeTotalInCurrWindow; + + // Track average response time in previous decay window + private final AtomicDoubleArray responseTimeAvgInLastWindow; + private final AtomicLongArray responseTimeCountInLastWindow; + // Pre-computed scheduling decisions during the decay sweep are // atomically swapped in as a read-only map private final AtomicReference> scheduleCacheRef = @@ -98,10 +132,12 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean // Tune the behavior of the scheduler private final long decayPeriodMillis; // How long between each tick - private final double decayFactor; // nextCount = currentCount / decayFactor - private final int numQueues; // affects scheduling decisions, from 0 to numQueues - 1 + private final double decayFactor; // nextCount = currentCount * decayFactor + private final int numLevels; private final double[] thresholds; private final IdentityProvider identityProvider; + private final boolean backOffByResponseTimeEnabled; + private final long[] backOffResponseTimeThresholds; /** * This TimerTask will call decayCurrentCounts until @@ -132,35 +168,46 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean /** * Create a decay scheduler. - * @param numQueues number of queues to schedule for + * @param numLevels number of priority levels * @param ns config prefix, so that we can configure multiple schedulers * in a single instance. * @param conf configuration to use. */ - public DecayRpcScheduler(int numQueues, String ns, Configuration conf) { - if (numQueues < 1) { - throw new IllegalArgumentException("number of queues must be > 0"); + public DecayRpcScheduler(int numLevels, String ns, Configuration conf) { + if(numLevels < 1) { + throw new IllegalArgumentException("Number of Priority Levels must be " + + "at least 1"); } - - this.numQueues = numQueues; + this.numLevels = numLevels; this.decayFactor = parseDecayFactor(ns, conf); this.decayPeriodMillis = parseDecayPeriodMillis(ns, conf); this.identityProvider = this.parseIdentityProvider(ns, conf); - this.thresholds = parseThresholds(ns, conf, numQueues); + this.thresholds = parseThresholds(ns, conf, numLevels); + this.backOffByResponseTimeEnabled = parseBackOffByResponseTimeEnabled(ns, + conf); + this.backOffResponseTimeThresholds = + parseBackOffResponseTimeThreshold(ns, conf, numLevels); // Setup delay timer Timer timer = new Timer(); DecayTask task = new DecayTask(this, timer); timer.scheduleAtFixedRate(task, decayPeriodMillis, decayPeriodMillis); - MetricsProxy prox = MetricsProxy.getInstance(ns); + // Setup response time metrics + responseTimeTotalInCurrWindow = new AtomicLongArray(numLevels); + responseTimeCountInCurrWindow = new AtomicLongArray(numLevels); + responseTimeAvgInLastWindow = new AtomicDoubleArray(numLevels); + responseTimeCountInLastWindow = new AtomicLongArray(numLevels); + + MetricsProxy prox = MetricsProxy.getInstance(ns, numLevels); prox.setDelegate(this); } // Load configs - private IdentityProvider parseIdentityProvider(String ns, Configuration conf) { + private IdentityProvider parseIdentityProvider(String ns, + Configuration conf) { List providers = conf.getInstances( - ns + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY, + ns + "." + CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY, IdentityProvider.class); if (providers.size() < 1) { @@ -174,10 +221,16 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean private static double parseDecayFactor(String ns, Configuration conf) { double factor = conf.getDouble(ns + "." + - IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, - IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT - ); - + IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, 0.0); + if (factor == 0.0) { + factor = conf.getDouble(ns + "." + + IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY, + IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT); + } else if ((factor > 0.0) && (factor < 1)) { + LOG.warn(IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY + + " is deprecated. Please use " + + IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY + "."); + } if (factor <= 0 || factor >= 1) { throw new IllegalArgumentException("Decay Factor " + "must be between 0 and 1"); @@ -188,10 +241,17 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean private static long parseDecayPeriodMillis(String ns, Configuration conf) { long period = conf.getLong(ns + "." + - IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, - IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT - ); - + IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, + 0); + if (period == 0) { + period = conf.getLong(ns + "." + + IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY, + IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT); + } else if (period > 0) { + LOG.warn((IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY + + " is deprecated. Please use " + + IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY)); + } if (period <= 0) { throw new IllegalArgumentException("Period millis must be >= 0"); } @@ -200,15 +260,24 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean } private static double[] parseThresholds(String ns, Configuration conf, - int numQueues) { + int numLevels) { int[] percentages = conf.getInts(ns + "." + - IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY); + IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY); if (percentages.length == 0) { - return getDefaultThresholds(numQueues); - } else if (percentages.length != numQueues-1) { + percentages = conf.getInts(ns + "." + IPC_DECAYSCHEDULER_THRESHOLDS_KEY); + if (percentages.length == 0) { + return getDefaultThresholds(numLevels); + } + } else { + LOG.warn(IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY + + " is deprecated. Please use " + + IPC_DECAYSCHEDULER_THRESHOLDS_KEY); + } + + if (percentages.length != numLevels-1) { throw new IllegalArgumentException("Number of thresholds should be " + - (numQueues-1) + ". Was: " + percentages.length); + (numLevels-1) + ". Was: " + percentages.length); } // Convert integer percentages to decimals @@ -223,14 +292,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean /** * Generate default thresholds if user did not specify. Strategy is * to halve each time, since queue usage tends to be exponential. - * So if numQueues is 4, we would generate: double[]{0.125, 0.25, 0.5} + * So if numLevels is 4, we would generate: double[]{0.125, 0.25, 0.5} * which specifies the boundaries between each queue's usage. - * @param numQueues number of queues to compute for - * @return array of boundaries of length numQueues - 1 + * @param numLevels number of levels to compute for + * @return array of boundaries of length numLevels - 1 */ - private static double[] getDefaultThresholds(int numQueues) { - double[] ret = new double[numQueues - 1]; - double div = Math.pow(2, numQueues - 1); + private static double[] getDefaultThresholds(int numLevels) { + double[] ret = new double[numLevels - 1]; + double div = Math.pow(2, numLevels - 1); for (int i = 0; i < ret.length; i++) { ret[i] = Math.pow(2, i)/div; @@ -238,39 +307,89 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean return ret; } + private static long[] parseBackOffResponseTimeThreshold(String ns, + Configuration conf, int numLevels) { + long[] responseTimeThresholds = conf.getTimeDurations(ns + "." + + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY, + TimeUnit.MILLISECONDS); + // backoff thresholds not specified + if (responseTimeThresholds.length == 0) { + return getDefaultBackOffResponseTimeThresholds(numLevels); + } + // backoff thresholds specified but not match with the levels + if (responseTimeThresholds.length != numLevels) { + throw new IllegalArgumentException( + "responseTimeThresholds must match with the number of priority " + + "levels"); + } + // invalid thresholds + for (long responseTimeThreshold: responseTimeThresholds) { + if (responseTimeThreshold <= 0) { + throw new IllegalArgumentException( + "responseTimeThreshold millis must be >= 0"); + } + } + return responseTimeThresholds; + } + + // 10s for level 0, 20s for level 1, 30s for level 2, ... + private static long[] getDefaultBackOffResponseTimeThresholds(int numLevels) { + long[] ret = new long[numLevels]; + for (int i = 0; i < ret.length; i++) { + ret[i] = 10000*(i+1); + } + return ret; + } + + private static Boolean parseBackOffByResponseTimeEnabled(String ns, + Configuration conf) { + return conf.getBoolean(ns + "." + + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY, + IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT); + } + /** * Decay the stored counts for each user and clean as necessary. * This method should be called periodically in order to keep * counts current. */ private void decayCurrentCounts() { - long total = 0; - Iterator> it = - callCounts.entrySet().iterator(); + try { + long total = 0; + Iterator> it = + callCounts.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = it.next(); - AtomicLong count = entry.getValue(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + AtomicLong count = entry.getValue(); - // Compute the next value by reducing it by the decayFactor - long currentValue = count.get(); - long nextValue = (long)(currentValue * decayFactor); - total += nextValue; - count.set(nextValue); + // Compute the next value by reducing it by the decayFactor + long currentValue = count.get(); + long nextValue = (long) (currentValue * decayFactor); + total += nextValue; + count.set(nextValue); - if (nextValue == 0) { - // We will clean up unused keys here. An interesting optimization might - // be to have an upper bound on keyspace in callCounts and only - // clean once we pass it. - it.remove(); + if (nextValue == 0) { + // We will clean up unused keys here. An interesting optimization + // might be to have an upper bound on keyspace in callCounts and only + // clean once we pass it. + it.remove(); + } } + + // Update the total so that we remain in sync + totalCalls.set(total); + + // Now refresh the cache of scheduling decisions + recomputeScheduleCache(); + + // Update average response time with decay + updateAverageResponseTime(true); + } catch (Exception ex) { + LOG.error("decayCurrentCounts exception: " + + ExceptionUtils.getFullStackTrace(ex)); + throw ex; } - - // Update the total so that we remain in sync - totalCalls.set(total); - - // Now refresh the cache of scheduling decisions - recomputeScheduleCache(); } /** @@ -324,7 +443,7 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean /** * Given the number of occurrences, compute a scheduling decision. * @param occurrences how many occurrences - * @return scheduling decision from 0 to numQueues - 1 + * @return scheduling decision from 0 to numLevels - 1 */ private int computePriorityLevel(long occurrences) { long totalCallSnapshot = totalCalls.get(); @@ -334,14 +453,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean proportion = (double) occurrences / totalCallSnapshot; } - // Start with low priority queues, since they will be most common - for(int i = (numQueues - 1); i > 0; i--) { + // Start with low priority levels, since they will be most common + for(int i = (numLevels - 1); i > 0; i--) { if (proportion >= this.thresholds[i - 1]) { - return i; // We've found our queue number + return i; // We've found our level number } } - // If we get this far, we're at queue 0 + // If we get this far, we're at level 0 return 0; } @@ -349,7 +468,7 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean * Returns the priority level for a given identity by first trying the cache, * then computing it. * @param identity an object responding to toString and hashCode - * @return integer scheduling decision from 0 to numQueues - 1 + * @return integer scheduling decision from 0 to numLevels - 1 */ private int cachedOrComputedPriorityLevel(Object identity) { try { @@ -360,22 +479,29 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean if (scheduleCache != null) { Integer priority = scheduleCache.get(identity); if (priority != null) { + LOG.debug("Cache priority for: {} with priority: {}", identity, + priority); return priority; } } // Cache was no good, compute it - return computePriorityLevel(occurrences); + int priority = computePriorityLevel(occurrences); + LOG.debug("compute priority for " + identity + " priority " + priority); + return priority; + } catch (InterruptedException ie) { - LOG.warn("Caught InterruptedException, returning low priority queue"); - return numQueues - 1; + LOG.warn("Caught InterruptedException, returning low priority level"); + LOG.debug("Fallback priority for: {} with priority: {}", identity, + numLevels - 1); + return numLevels - 1; } } /** * Compute the appropriate priority for a schedulable based on past requests. * @param obj the schedulable obj to query and remember - * @return the queue index which we recommend scheduling in + * @return the level index which we recommend scheduling in */ @Override public int getPriorityLevel(Schedulable obj) { @@ -389,6 +515,73 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean return cachedOrComputedPriorityLevel(identity); } + @Override + public boolean shouldBackOff(Schedulable obj) { + Boolean backOff = false; + if (backOffByResponseTimeEnabled) { + int priorityLevel = obj.getPriorityLevel(); + if (LOG.isDebugEnabled()) { + double[] responseTimes = getAverageResponseTime(); + LOG.debug("Current Caller: {} Priority: {} ", + obj.getUserGroupInformation().getUserName(), + obj.getPriorityLevel()); + for (int i = 0; i < numLevels; i++) { + LOG.debug("Queue: {} responseTime: {} backoffThreshold: {}", i, + responseTimes[i], backOffResponseTimeThresholds[i]); + } + } + // High priority rpc over threshold triggers back off of low priority rpc + for (int i = 0; i < priorityLevel + 1; i++) { + if (responseTimeAvgInLastWindow.get(i) > + backOffResponseTimeThresholds[i]) { + backOff = true; + break; + } + } + } + return backOff; + } + + @Override + public void addResponseTime(String name, int priorityLevel, int queueTime, + int processingTime) { + responseTimeCountInCurrWindow.getAndIncrement(priorityLevel); + responseTimeTotalInCurrWindow.getAndAdd(priorityLevel, + queueTime+processingTime); + if (LOG.isDebugEnabled()) { + LOG.debug("addResponseTime for call: {} priority: {} queueTime: {} " + + "processingTime: {} ", name, priorityLevel, queueTime, + processingTime); + } + } + + // Update the cached average response time at the end of decay window + void updateAverageResponseTime(boolean enableDecay) { + for (int i = 0; i < numLevels; i++) { + double averageResponseTime = 0; + long totalResponseTime = responseTimeTotalInCurrWindow.get(i); + long responseTimeCount = responseTimeCountInCurrWindow.get(i); + if (responseTimeCount > 0) { + averageResponseTime = (double) totalResponseTime / responseTimeCount; + } + final double lastAvg = responseTimeAvgInLastWindow.get(i); + if (enableDecay && lastAvg > 0.0) { + final double decayed = decayFactor * lastAvg + averageResponseTime; + responseTimeAvgInLastWindow.set(i, decayed); + } else { + responseTimeAvgInLastWindow.set(i, averageResponseTime); + } + responseTimeCountInLastWindow.set(i, responseTimeCount); + if (LOG.isDebugEnabled()) { + LOG.debug("updateAverageResponseTime queue: {} Average: {} Count: {}", + i, averageResponseTime, responseTimeCount); + } + // Reset for next decay window + responseTimeTotalInCurrWindow.set(i, 0); + responseTimeCountInCurrWindow.set(i, 0); + } + } + // For testing @VisibleForTesting public double getDecayFactor() { return decayFactor; } @@ -429,16 +622,21 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean // Weakref for delegate, so we don't retain it forever if it can be GC'd private WeakReference delegate; + private double[] averageResponseTimeDefault; + private long[] callCountInLastWindowDefault; - private MetricsProxy(String namespace) { + private MetricsProxy(String namespace, int numLevels) { + averageResponseTimeDefault = new double[numLevels]; + callCountInLastWindowDefault = new long[numLevels]; MBeans.register(namespace, "DecayRpcScheduler", this); } - public static synchronized MetricsProxy getInstance(String namespace) { + public static synchronized MetricsProxy getInstance(String namespace, + int numLevels) { MetricsProxy mp = INSTANCES.get(namespace); if (mp == null) { // We must create one - mp = new MetricsProxy(namespace); + mp = new MetricsProxy(namespace, numLevels); INSTANCES.put(namespace, mp); } return mp; @@ -487,6 +685,25 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean return scheduler.getTotalCallVolume(); } } + + @Override + public double[] getAverageResponseTime() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return averageResponseTimeDefault; + } else { + return scheduler.getAverageResponseTime(); + } + } + + public long[] getResponseTimeCountInLastWindow() { + DecayRpcScheduler scheduler = delegate.get(); + if (scheduler == null) { + return callCountInLastWindowDefault; + } else { + return scheduler.getResponseTimeCountInLastWindow(); + } + } } public int getUniqueIdentityCount() { @@ -497,6 +714,23 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean return totalCalls.get(); } + public long[] getResponseTimeCountInLastWindow() { + long[] ret = new long[responseTimeCountInLastWindow.length()]; + for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) { + ret[i] = responseTimeCountInLastWindow.get(i); + } + return ret; + } + + @Override + public double[] getAverageResponseTime() { + double[] ret = new double[responseTimeAvgInLastWindow.length()]; + for (int i = 0; i < responseTimeAvgInLastWindow.length(); i++) { + ret[i] = responseTimeAvgInLastWindow.get(i); + } + return ret; + } + public String getSchedulingDecisionSummary() { Map decisions = scheduleCacheRef.get(); if (decisions == null) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java index 3481f19449d..fab9b93b1b3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java @@ -27,4 +27,6 @@ public interface DecayRpcSchedulerMXBean { String getCallVolumeSummary(); int getUniqueIdentityCount(); long getTotalCallVolume(); + double[] getAverageResponseTime(); + long[] getResponseTimeCountInLastWindow(); } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java new file mode 100644 index 00000000000..08f74d4b1c0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import org.apache.hadoop.conf.Configuration; + +/** + * No op default RPC scheduler. + */ +public class DefaultRpcScheduler implements RpcScheduler { + @Override + public int getPriorityLevel(Schedulable obj) { + return 0; + } + + @Override + public boolean shouldBackOff(Schedulable obj) { + return false; + } + + @Override + public void addResponseTime(String name, int priorityLevel, int queueTime, + int processingTime) { + } + + public DefaultRpcScheduler(int priorityLevels, String namespace, + Configuration conf) { + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java index 0b56243db58..435c454176b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java @@ -44,8 +44,9 @@ import org.apache.hadoop.metrics2.util.MBeans; */ public class FairCallQueue extends AbstractQueue implements BlockingQueue { - // Configuration Keys + @Deprecated public static final int IPC_CALLQUEUE_PRIORITY_LEVELS_DEFAULT = 4; + @Deprecated public static final String IPC_CALLQUEUE_PRIORITY_LEVELS_KEY = "faircallqueue.priority-levels"; @@ -66,9 +67,6 @@ public class FairCallQueue extends AbstractQueue } } - /* Scheduler picks which queue to place in */ - private RpcScheduler scheduler; - /* Multiplexer picks which queue to draw from */ private RpcMultiplexer multiplexer; @@ -83,8 +81,13 @@ public class FairCallQueue extends AbstractQueue * Notes: the FairCallQueue has no fixed capacity. Rather, it has a minimum * capacity of `capacity` and a maximum capacity of `capacity * number_queues` */ - public FairCallQueue(int capacity, String ns, Configuration conf) { - int numQueues = parseNumQueues(ns, conf); + public FairCallQueue(int priorityLevels, int capacity, String ns, + Configuration conf) { + if(priorityLevels < 1) { + throw new IllegalArgumentException("Number of Priority Levels must be " + + "at least 1"); + } + int numQueues = priorityLevels; LOG.info("FairCallQueue is in use with " + numQueues + " queues."); this.queues = new ArrayList>(numQueues); @@ -95,28 +98,12 @@ public class FairCallQueue extends AbstractQueue this.overflowedCalls.add(new AtomicLong(0)); } - this.scheduler = new DecayRpcScheduler(numQueues, ns, conf); this.multiplexer = new WeightedRoundRobinMultiplexer(numQueues, ns, conf); - // Make this the active source of metrics MetricsProxy mp = MetricsProxy.getInstance(ns); mp.setDelegate(this); } - /** - * Read the number of queues from the configuration. - * This will affect the FairCallQueue's overall capacity. - * @throws IllegalArgumentException on invalid queue count - */ - private static int parseNumQueues(String ns, Configuration conf) { - int retval = conf.getInt(ns + "." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, - IPC_CALLQUEUE_PRIORITY_LEVELS_DEFAULT); - if(retval < 1) { - throw new IllegalArgumentException("numQueues must be at least 1"); - } - return retval; - } - /** * Returns the first non-empty queue with equal or lesser priority * than startIdx. Wraps around, searching a maximum of N @@ -144,7 +131,7 @@ public class FairCallQueue extends AbstractQueue /** * Put and offer follow the same pattern: - * 1. Get a priorityLevel from the scheduler + * 1. Get the assigned priorityLevel from the call by scheduler * 2. Get the nth sub-queue matching this priorityLevel * 3. delegate the call to this sub-queue. * @@ -154,7 +141,7 @@ public class FairCallQueue extends AbstractQueue */ @Override public void put(E e) throws InterruptedException { - int priorityLevel = scheduler.getPriorityLevel(e); + int priorityLevel = e.getPriorityLevel(); final int numLevels = this.queues.size(); while (true) { @@ -185,7 +172,7 @@ public class FairCallQueue extends AbstractQueue @Override public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { - int priorityLevel = scheduler.getPriorityLevel(e); + int priorityLevel = e.getPriorityLevel(); BlockingQueue q = this.queues.get(priorityLevel); boolean ret = q.offer(e, timeout, unit); @@ -196,7 +183,7 @@ public class FairCallQueue extends AbstractQueue @Override public boolean offer(E e) { - int priorityLevel = scheduler.getPriorityLevel(e); + int priorityLevel = e.getPriorityLevel(); BlockingQueue q = this.queues.get(priorityLevel); boolean ret = q.offer(e); @@ -436,12 +423,6 @@ public class FairCallQueue extends AbstractQueue return calls; } - // For testing - @VisibleForTesting - public void setScheduler(RpcScheduler newScheduler) { - this.scheduler = newScheduler; - } - @VisibleForTesting public void setMultiplexer(RpcMultiplexer newMux) { this.multiplexer = newMux; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index 692d2b6e384..071e2e827b4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -654,13 +654,7 @@ public class ProtobufRpcEngine implements RpcEngine { String detailedMetricsName = (exception == null) ? methodName : exception.getClass().getSimpleName(); - server.rpcMetrics.addRpcQueueTime(qTime); - server.rpcMetrics.addRpcProcessingTime(processingTime); - server.rpcDetailedMetrics.addProcessingTime(detailedMetricsName, - processingTime); - if (server.isLogSlowRPC()) { - server.logSlowRpcCalls(methodName, processingTime); - } + server.updateMetrics(detailedMetricsName, qTime, processingTime); } return new RpcResponseWrapper(result); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java index a1557061809..6f93b22eed1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java @@ -19,11 +19,17 @@ package org.apache.hadoop.ipc; /** - * Implement this interface to be used for RPC scheduling in the fair call queues. + * Implement this interface to be used for RPC scheduling and backoff. + * */ public interface RpcScheduler { /** * Returns priority level greater than zero as a hint for scheduling. */ int getPriorityLevel(Schedulable obj); + + boolean shouldBackOff(Schedulable obj); + + void addResponseTime(String name, int priorityLevel, int queueTime, + int processingTime); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java index 38f3518df98..3b28d85428b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java @@ -18,11 +18,8 @@ package org.apache.hadoop.ipc; -import java.nio.ByteBuffer; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.io.Writable; /** * Interface which allows extracting information necessary to @@ -31,4 +28,6 @@ import org.apache.hadoop.io.Writable; @InterfaceAudience.Private public interface Schedulable { public UserGroupInformation getUserGroupInformation(); + + int getPriorityLevel(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 1d928659ea9..eb28ad57d11 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -396,6 +396,15 @@ public abstract class Server { return CurCall.get() != null; } + /** + * Return the priority level assigned by call queue to an RPC + * Returns 0 in case no priority is assigned. + */ + public static int getPriorityLevel() { + Call call = CurCall.get(); + return call != null? call.getPriorityLevel() : 0; + } + private String bindAddress; private int port; // port we listen on private int handlerCount; // number of handler threads @@ -482,6 +491,18 @@ public abstract class Server { } } + void updateMetrics(String name, int queueTime, int processingTime) { + rpcMetrics.addRpcQueueTime(queueTime); + rpcMetrics.addRpcProcessingTime(processingTime); + rpcDetailedMetrics.addProcessingTime(name, processingTime); + callQueue.addResponseTime(name, getPriorityLevel(), queueTime, + processingTime); + + if (isLogSlowRPC()) { + logSlowRpcCalls(name, processingTime); + } + } + /** * A convenience method to bind to a given address and report * better exceptions if the address is not a valid host. @@ -578,6 +599,10 @@ public abstract class Server { return serviceAuthorizationManager; } + private String getQueueClassPrefix() { + return CommonConfigurationKeys.IPC_NAMESPACE + "." + port; + } + static Class> getQueueClass( String prefix, Configuration conf) { String name = prefix + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY; @@ -585,8 +610,29 @@ public abstract class Server { return CallQueueManager.convertQueueClass(queueClass, Call.class); } - private String getQueueClassPrefix() { - return CommonConfigurationKeys.IPC_CALLQUEUE_NAMESPACE + "." + port; + static Class getSchedulerClass( + String prefix, Configuration conf) { + String schedulerKeyname = prefix + "." + CommonConfigurationKeys + .IPC_SCHEDULER_IMPL_KEY; + Class schedulerClass = conf.getClass(schedulerKeyname, null); + // Patch the configuration for legacy fcq configuration that does not have + // a separate scheduler setting + if (schedulerClass == null) { + String queueKeyName = prefix + "." + CommonConfigurationKeys + .IPC_CALLQUEUE_IMPL_KEY; + Class queueClass = conf.getClass(queueKeyName, null); + if (queueClass != null) { + if (queueClass.getCanonicalName().equals( + FairCallQueue.class.getCanonicalName())) { + conf.setClass(schedulerKeyname, DecayRpcScheduler.class, + RpcScheduler.class); + } + } + } + schedulerClass = conf.getClass(schedulerKeyname, + DefaultRpcScheduler.class); + + return CallQueueManager.convertSchedulerClass(schedulerClass); } /* @@ -595,7 +641,8 @@ public abstract class Server { public synchronized void refreshCallQueue(Configuration conf) { // Create the next queue String prefix = getQueueClassPrefix(); - callQueue.swapQueue(getQueueClass(prefix, conf), maxQueueSize, prefix, conf); + callQueue.swapQueue(getSchedulerClass(prefix, conf), + getQueueClass(prefix, conf), maxQueueSize, prefix, conf); } /** @@ -623,6 +670,8 @@ public abstract class Server { private final byte[] clientId; private final TraceScope traceScope; // the HTrace scope on the server side private final CallerContext callerContext; // the call context + private int priorityLevel; + // the priority level assigned by scheduler, 0 by default private Call(Call call) { this(call.callId, call.retryCount, call.rpcRequest, call.connection, @@ -709,7 +758,16 @@ public abstract class Server { @Override public UserGroupInformation getUserGroupInformation() { return connection.user; - } + } + + @Override + public int getPriorityLevel() { + return this.priorityLevel; + } + + public void setPriorityLevel(int priorityLevel) { + this.priorityLevel = priorityLevel; + } } /** Listens on the socket. Creates jobs for the handler threads*/ @@ -2151,6 +2209,9 @@ public abstract class Server { rpcRequest, this, ProtoUtil.convert(header.getRpcKind()), header.getClientId().toByteArray(), traceScope, callerContext); + // Save the priority level assignment by the scheduler + call.setPriorityLevel(callQueue.getPriorityLevel(call)); + if (callQueue.isClientBackoffEnabled()) { // if RPC queue is full, we will ask the RPC client to back off by // throwing RetriableException. Whether RPC client will honor @@ -2166,9 +2227,10 @@ public abstract class Server { private void queueRequestOrAskClientToBackOff(Call call) throws WrappedRpcServerException, InterruptedException { - // If rpc queue is full, we will ask the client to back off. - boolean isCallQueued = callQueue.offer(call); - if (!isCallQueued) { + // If rpc scheduler indicates back off based on performance + // degradation such as response time or rpc queue is full, + // we will ask the client to back off. + if (callQueue.shouldBackOff(call) || !callQueue.offer(call)) { rpcMetrics.incrClientBackoff(); RetriableException retriableException = new RetriableException("Server is too busy."); @@ -2513,6 +2575,7 @@ public abstract class Server { // Setup appropriate callqueue final String prefix = getQueueClassPrefix(); this.callQueue = new CallQueueManager(getQueueClass(prefix, conf), + getSchedulerClass(prefix, conf), getClientBackoffEnable(prefix, conf), maxQueueSize, prefix, conf); this.secretManager = (SecretManager) secretManager; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java index a1db6be7c43..a9dbb41fd98 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java @@ -34,7 +34,6 @@ import org.apache.hadoop.io.*; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.RPC.RpcInvoker; -import org.apache.hadoop.ipc.VersionedProtocol; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.TokenIdentifier; @@ -502,13 +501,12 @@ public class WritableRpcEngine implements RpcEngine { } } } - - // Invoke the protocol method - long startTime = Time.now(); - int qTime = (int) (startTime-receivedTime); - Exception exception = null; - try { + // Invoke the protocol method + long startTime = Time.now(); + int qTime = (int) (startTime-receivedTime); + Exception exception = null; + try { Method method = protocolImpl.protocolClass.getMethod(call.getMethodName(), call.getParameterClasses()); @@ -539,27 +537,20 @@ public class WritableRpcEngine implements RpcEngine { exception = ioe; throw ioe; } finally { - int processingTime = (int) (Time.now() - startTime); - if (LOG.isDebugEnabled()) { - String msg = "Served: " + call.getMethodName() + - " queueTime= " + qTime + - " procesingTime= " + processingTime; - if (exception != null) { - msg += " exception= " + exception.getClass().getSimpleName(); - } - LOG.debug(msg); - } - String detailedMetricsName = (exception == null) ? - call.getMethodName() : - exception.getClass().getSimpleName(); - server.rpcMetrics.addRpcQueueTime(qTime); - server.rpcMetrics.addRpcProcessingTime(processingTime); - server.rpcDetailedMetrics.addProcessingTime(detailedMetricsName, - processingTime); - if (server.isLogSlowRPC()) { - server.logSlowRpcCalls(call.getMethodName(), processingTime); + int processingTime = (int) (Time.now() - startTime); + if (LOG.isDebugEnabled()) { + String msg = "Served: " + call.getMethodName() + + " queueTime= " + qTime + " procesingTime= " + processingTime; + if (exception != null) { + msg += " exception= " + exception.getClass().getSimpleName(); + } + LOG.debug(msg); } - } + String detailedMetricsName = (exception == null) ? + call.getMethodName() : + exception.getClass().getSimpleName(); + server.updateMetrics(detailedMetricsName, qTime, processingTime); + } } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java index 4d659acd46d..af9ce1b08bc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java @@ -27,17 +27,37 @@ import java.util.HashMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.Test; public class TestCallQueueManager { private CallQueueManager manager; + private Configuration conf = new Configuration(); - public class FakeCall { + public class FakeCall implements Schedulable { public final int tag; // Can be used for unique identification - + private int priorityLevel; + UserGroupInformation fakeUgi = UserGroupInformation.createRemoteUser + ("fakeUser"); public FakeCall(int tag) { this.tag = tag; } + + @Override + public UserGroupInformation getUserGroupInformation() { + return fakeUgi; + } + + @Override + public int getPriorityLevel() { + return priorityLevel; + } + + public void setPriorityLevel(int level) { + this.priorityLevel = level; + } } /** @@ -62,7 +82,9 @@ public class TestCallQueueManager { try { // Fill up to max (which is infinite if maxCalls < 0) while (isRunning && (callsAdded < maxCalls || maxCalls < 0)) { - cq.put(new FakeCall(this.tag)); + FakeCall call = new FakeCall(this.tag); + call.setPriorityLevel(cq.getPriorityLevel(call)); + cq.put(call); callsAdded++; } } catch (InterruptedException e) { @@ -135,7 +157,7 @@ public class TestCallQueueManager { t.start(); t.join(100); - assertEquals(putter.callsAdded, numberOfPuts); + assertEquals(numberOfPuts, putter.callsAdded); t.interrupt(); } @@ -143,23 +165,90 @@ public class TestCallQueueManager { private static final Class> queueClass = CallQueueManager.convertQueueClass(LinkedBlockingQueue.class, FakeCall.class); + private static final Class schedulerClass + = CallQueueManager.convertSchedulerClass(DefaultRpcScheduler.class); + @Test public void testCallQueueCapacity() throws InterruptedException { - manager = new CallQueueManager(queueClass, false, 10, "", null); + manager = new CallQueueManager(queueClass, schedulerClass, false, + 10, "", conf); assertCanPut(manager, 10, 20); // Will stop at 10 due to capacity } @Test public void testEmptyConsume() throws InterruptedException { - manager = new CallQueueManager(queueClass, false, 10, "", null); + manager = new CallQueueManager(queueClass, schedulerClass, false, + 10, "", conf); assertCanTake(manager, 0, 1); // Fails since it's empty } + static Class> getQueueClass( + String prefix, Configuration conf) { + String name = prefix + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY; + Class queueClass = conf.getClass(name, LinkedBlockingQueue.class); + return CallQueueManager.convertQueueClass(queueClass, FakeCall.class); + } + + @Test + public void testFcqBackwardCompatibility() throws InterruptedException { + // Test BackwardCompatibility to ensure existing FCQ deployment still + // work without explicitly specifying DecayRpcScheduler + Configuration conf = new Configuration(); + final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0"; + + final String queueClassName = "org.apache.hadoop.ipc.FairCallQueue"; + conf.setStrings(ns + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY, + queueClassName); + + // Specify only Fair Call Queue without a scheduler + // Ensure the DecayScheduler will be added to avoid breaking. + Class scheduler = Server.getSchedulerClass(ns, + conf); + assertTrue(scheduler.getCanonicalName(). + equals("org.apache.hadoop.ipc.DecayRpcScheduler")); + + Class> queue = + (Class>) getQueueClass(ns, conf); + assertTrue(queue.getCanonicalName().equals(queueClassName)); + + manager = new CallQueueManager(queue, scheduler, false, + 2, "", conf); + + // Default FCQ has 4 levels and the max capacity is 2 x 4 + assertCanPut(manager, 3, 3); + } + + @Test + public void testSchedulerWithoutFCQ() throws InterruptedException { + Configuration conf = new Configuration(); + // Test DecayedRpcScheduler without FCQ + // Ensure the default LinkedBlockingQueue can work with DecayedRpcScheduler + final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0"; + final String schedulerClassName = "org.apache.hadoop.ipc.DecayRpcScheduler"; + conf.setStrings(ns + "." + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY, + schedulerClassName); + + Class> queue = + (Class>) getQueueClass(ns, conf); + assertTrue(queue.getCanonicalName().equals("java.util.concurrent." + + "LinkedBlockingQueue")); + + manager = new CallQueueManager(queue, + Server.getSchedulerClass(ns, conf), false, + 3, "", conf); + + // LinkedBlockingQueue with a capacity of 3 can put 3 calls + assertCanPut(manager, 3, 3); + // LinkedBlockingQueue with a capacity of 3 can't put 1 more call + assertCanPut(manager, 0, 1); + } + @Test(timeout=60000) public void testSwapUnderContention() throws InterruptedException { - manager = new CallQueueManager(queueClass, false, 5000, "", null); + manager = new CallQueueManager(queueClass, schedulerClass, false, + 5000, "", conf); ArrayList producers = new ArrayList(); ArrayList consumers = new ArrayList(); @@ -188,7 +277,7 @@ public class TestCallQueueManager { Thread.sleep(500); for (int i=0; i < 5; i++) { - manager.swapQueue(queueClass, 5000, "", null); + manager.swapQueue(schedulerClass, queueClass, 5000, "", conf); } // Stop the producers @@ -223,24 +312,50 @@ public class TestCallQueueManager { } public static class ExceptionFakeCall { - public ExceptionFakeCall() { - throw new IllegalArgumentException("Exception caused by constructor.!!"); + throw new IllegalArgumentException("Exception caused by call queue " + + "constructor.!!"); } } - private static final Class> exceptionQueueClass = CallQueueManager - .convertQueueClass(ExceptionFakeCall.class, ExceptionFakeCall.class); + public static class ExceptionFakeScheduler { + public ExceptionFakeScheduler() { + throw new IllegalArgumentException("Exception caused by " + + "scheduler constructor.!!"); + } + } + + private static final Class + exceptionSchedulerClass = CallQueueManager.convertSchedulerClass( + ExceptionFakeScheduler.class); + + private static final Class> + exceptionQueueClass = CallQueueManager.convertQueueClass( + ExceptionFakeCall.class, ExceptionFakeCall.class); @Test - public void testInvocationException() throws InterruptedException { + public void testCallQueueConstructorException() throws InterruptedException { try { - new CallQueueManager(exceptionQueueClass, false, 10, - "", null); + new CallQueueManager(exceptionQueueClass, + schedulerClass, false, 10, "", new Configuration()); fail(); } catch (RuntimeException re) { assertTrue(re.getCause() instanceof IllegalArgumentException); - assertEquals("Exception caused by constructor.!!", re.getCause() + assertEquals("Exception caused by call queue constructor.!!", re + .getCause() + .getMessage()); + } + } + + @Test + public void testSchedulerConstructorException() throws InterruptedException { + try { + new CallQueueManager(queueClass, exceptionSchedulerClass, + false, 10, "", new Configuration()); + fail(); + } catch (RuntimeException re) { + assertTrue(re.getCause() instanceof IllegalArgumentException); + assertEquals("Exception caused by scheduler constructor.!!", re.getCause() .getMessage()); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java index edc3b0051ab..0b0408cbcdb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java @@ -18,12 +18,11 @@ package org.apache.hadoop.ipc; +import static java.lang.Thread.sleep; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.Arrays; import org.junit.Test; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -31,8 +30,6 @@ import static org.mockito.Mockito.when; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; - public class TestDecayRpcScheduler { private Schedulable mockCall(String id) { Schedulable mockCall = mock(Schedulable.class); @@ -57,30 +54,32 @@ public class TestDecayRpcScheduler { } @Test + @SuppressWarnings("deprecation") public void testParsePeriod() { // By default scheduler = new DecayRpcScheduler(1, "", new Configuration()); - assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT, + assertEquals(DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT, scheduler.getDecayPeriodMillis()); // Custom Configuration conf = new Configuration(); - conf.setLong("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, + conf.setLong("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, 1058); scheduler = new DecayRpcScheduler(1, "ns", conf); assertEquals(1058L, scheduler.getDecayPeriodMillis()); } @Test + @SuppressWarnings("deprecation") public void testParseFactor() { // Default scheduler = new DecayRpcScheduler(1, "", new Configuration()); - assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT, + assertEquals(DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT, scheduler.getDecayFactor(), 0.00001); // Custom Configuration conf = new Configuration(); - conf.set("prefix." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, + conf.set("prefix." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, "0.125"); scheduler = new DecayRpcScheduler(1, "prefix", conf); assertEquals(0.125, scheduler.getDecayFactor(), 0.00001); @@ -94,6 +93,7 @@ public class TestDecayRpcScheduler { } @Test + @SuppressWarnings("deprecation") public void testParseThresholds() { // Defaults vary by number of queues Configuration conf = new Configuration(); @@ -111,16 +111,17 @@ public class TestDecayRpcScheduler { // Custom conf = new Configuration(); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY, + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY, "1, 10, 20, 50, 85"); scheduler = new DecayRpcScheduler(6, "ns", conf); assertEqualDecimalArrays(new double[]{0.01, 0.1, 0.2, 0.5, 0.85}, scheduler.getThresholds()); } @Test + @SuppressWarnings("deprecation") public void testAccumulate() { Configuration conf = new Configuration(); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush scheduler = new DecayRpcScheduler(1, "ns", conf); assertEquals(0, scheduler.getCallCountSnapshot().size()); // empty first @@ -138,10 +139,11 @@ public class TestDecayRpcScheduler { } @Test - public void testDecay() { + @SuppressWarnings("deprecation") + public void testDecay() throws Exception { Configuration conf = new Configuration(); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "999999999"); // Never - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5"); + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "999999999"); // Never + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, "0.5"); scheduler = new DecayRpcScheduler(1, "ns", conf); assertEquals(0, scheduler.getTotalCallSnapshot()); @@ -150,6 +152,8 @@ public class TestDecayRpcScheduler { scheduler.getPriorityLevel(mockCall("A")); } + sleep(1000); + for (int i = 0; i < 8; i++) { scheduler.getPriorityLevel(mockCall("B")); } @@ -184,10 +188,11 @@ public class TestDecayRpcScheduler { } @Test + @SuppressWarnings("deprecation") public void testPriority() { Configuration conf = new Configuration(); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY, + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY, "25, 50, 75"); scheduler = new DecayRpcScheduler(4, "ns", conf); @@ -204,10 +209,11 @@ public class TestDecayRpcScheduler { } @Test(timeout=2000) + @SuppressWarnings("deprecation") public void testPeriodic() throws InterruptedException { Configuration conf = new Configuration(); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "10"); - conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5"); + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "10"); + conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, "0.5"); scheduler = new DecayRpcScheduler(1, "ns", conf); assertEquals(10, scheduler.getDecayPeriodMillis()); @@ -219,7 +225,7 @@ public class TestDecayRpcScheduler { // It should eventually decay to zero while (scheduler.getTotalCallSnapshot() > 0) { - Thread.sleep(10); + sleep(10); } } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java index 2694ba3ab91..4a8ad3b9271 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java @@ -37,21 +37,24 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.conf.Configuration; import org.mockito.Matchers; -import static org.apache.hadoop.ipc.FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY; - public class TestFairCallQueue extends TestCase { private FairCallQueue fcq; - private Schedulable mockCall(String id) { + private Schedulable mockCall(String id, int priority) { Schedulable mockCall = mock(Schedulable.class); UserGroupInformation ugi = mock(UserGroupInformation.class); when(ugi.getUserName()).thenReturn(id); when(mockCall.getUserGroupInformation()).thenReturn(ugi); + when(mockCall.getPriorityLevel()).thenReturn(priority); return mockCall; } + private Schedulable mockCall(String id) { + return mockCall(id, 0); + } + // A scheduler which always schedules into priority zero private RpcScheduler alwaysZeroScheduler; { @@ -60,11 +63,12 @@ public class TestFairCallQueue extends TestCase { alwaysZeroScheduler = sched; } + @SuppressWarnings("deprecation") public void setUp() { Configuration conf = new Configuration(); - conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); + conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); - fcq = new FairCallQueue(5, "ns", conf); + fcq = new FairCallQueue(2, 5, "ns", conf); } // @@ -85,7 +89,6 @@ public class TestFairCallQueue extends TestCase { } public void testOfferSucceeds() { - fcq.setScheduler(alwaysZeroScheduler); for (int i = 0; i < 5; i++) { // We can fit 10 calls @@ -96,7 +99,6 @@ public class TestFairCallQueue extends TestCase { } public void testOfferFailsWhenFull() { - fcq.setScheduler(alwaysZeroScheduler); for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c"))); } assertFalse(fcq.offer(mockCall("c"))); // It's full @@ -107,11 +109,10 @@ public class TestFairCallQueue extends TestCase { public void testOfferSucceedsWhenScheduledLowPriority() { // Scheduler will schedule into queue 0 x 5, then queue 1 RpcScheduler sched = mock(RpcScheduler.class); - when(sched.getPriorityLevel(Matchers.any())).thenReturn(0, 0, 0, 0, 0, 1, 0); - fcq.setScheduler(sched); - for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c"))); } + int mockedPriorities[] = {0, 0, 0, 0, 0, 1, 0}; + for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c", mockedPriorities[i]))); } - assertTrue(fcq.offer(mockCall("c"))); + assertTrue(fcq.offer(mockCall("c", mockedPriorities[5]))); assertEquals(6, fcq.size()); } @@ -121,7 +122,7 @@ public class TestFairCallQueue extends TestCase { } public void testPeekNonDestructive() { - Schedulable call = mockCall("c"); + Schedulable call = mockCall("c", 0); assertTrue(fcq.offer(call)); assertEquals(call, fcq.peek()); @@ -130,8 +131,8 @@ public class TestFairCallQueue extends TestCase { } public void testPeekPointsAtHead() { - Schedulable call = mockCall("c"); - Schedulable next = mockCall("b"); + Schedulable call = mockCall("c", 0); + Schedulable next = mockCall("b", 0); fcq.offer(call); fcq.offer(next); @@ -139,15 +140,11 @@ public class TestFairCallQueue extends TestCase { } public void testPollTimeout() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); - assertNull(fcq.poll(10, TimeUnit.MILLISECONDS)); } public void testPollSuccess() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); - - Schedulable call = mockCall("c"); + Schedulable call = mockCall("c", 0); assertTrue(fcq.offer(call)); assertEquals(call, fcq.poll(10, TimeUnit.MILLISECONDS)); @@ -156,7 +153,6 @@ public class TestFairCallQueue extends TestCase { } public void testOfferTimeout() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c"), 10, TimeUnit.MILLISECONDS)); } @@ -166,13 +162,11 @@ public class TestFairCallQueue extends TestCase { assertEquals(5, fcq.size()); } + @SuppressWarnings("deprecation") public void testDrainTo() { Configuration conf = new Configuration(); - conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); - FairCallQueue fcq2 = new FairCallQueue(10, "ns", conf); - - fcq.setScheduler(alwaysZeroScheduler); - fcq2.setScheduler(alwaysZeroScheduler); + conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); + FairCallQueue fcq2 = new FairCallQueue(2, 10, "ns", conf); // Start with 3 in fcq, to be drained for (int i = 0; i < 3; i++) { @@ -185,13 +179,11 @@ public class TestFairCallQueue extends TestCase { assertEquals(3, fcq2.size()); } + @SuppressWarnings("deprecation") public void testDrainToWithLimit() { Configuration conf = new Configuration(); - conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); - FairCallQueue fcq2 = new FairCallQueue(10, "ns", conf); - - fcq.setScheduler(alwaysZeroScheduler); - fcq2.setScheduler(alwaysZeroScheduler); + conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2); + FairCallQueue fcq2 = new FairCallQueue(2, 10, "ns", conf); // Start with 3 in fcq, to be drained for (int i = 0; i < 3; i++) { @@ -209,27 +201,23 @@ public class TestFairCallQueue extends TestCase { } public void testFirstQueueFullRemainingCapacity() { - fcq.setScheduler(alwaysZeroScheduler); while (fcq.offer(mockCall("c"))) ; // Queue 0 will fill up first, then queue 1 assertEquals(5, fcq.remainingCapacity()); } public void testAllQueuesFullRemainingCapacity() { - RpcScheduler sched = mock(RpcScheduler.class); - when(sched.getPriorityLevel(Matchers.any())).thenReturn(0, 0, 0, 0, 0, 1, 1, 1, 1, 1); - fcq.setScheduler(sched); - while (fcq.offer(mockCall("c"))) ; + int[] mockedPriorities = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0}; + int i = 0; + while (fcq.offer(mockCall("c", mockedPriorities[i++]))) ; assertEquals(0, fcq.remainingCapacity()); assertEquals(10, fcq.size()); } public void testQueuesPartialFilledRemainingCapacity() { - RpcScheduler sched = mock(RpcScheduler.class); - when(sched.getPriorityLevel(Matchers.any())).thenReturn(0, 1, 0, 1, 0); - fcq.setScheduler(sched); - for (int i = 0; i < 5; i++) { fcq.offer(mockCall("c")); } + int[] mockedPriorities = {0, 1, 0, 1, 0}; + for (int i = 0; i < 5; i++) { fcq.offer(mockCall("c", mockedPriorities[i])); } assertEquals(5, fcq.remainingCapacity()); assertEquals(5, fcq.size()); @@ -351,16 +339,12 @@ public class TestFairCallQueue extends TestCase { // Make sure put will overflow into lower queues when the top is full public void testPutOverflows() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); - // We can fit more than 5, even though the scheduler suggests the top queue assertCanPut(fcq, 8, 8); assertEquals(8, fcq.size()); } public void testPutBlocksWhenAllFull() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); - assertCanPut(fcq, 10, 10); // Fill up assertEquals(10, fcq.size()); @@ -369,12 +353,10 @@ public class TestFairCallQueue extends TestCase { } public void testTakeBlocksWhenEmpty() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); assertCanTake(fcq, 0, 1); } public void testTakeRemovesCall() throws InterruptedException { - fcq.setScheduler(alwaysZeroScheduler); Schedulable call = mockCall("c"); fcq.offer(call); @@ -383,17 +365,14 @@ public class TestFairCallQueue extends TestCase { } public void testTakeTriesNextQueue() throws InterruptedException { - // Make a FCQ filled with calls in q 1 but empty in q 0 - RpcScheduler q1Scheduler = mock(RpcScheduler.class); - when(q1Scheduler.getPriorityLevel(Matchers.any())).thenReturn(1); - fcq.setScheduler(q1Scheduler); // A mux which only draws from q 0 RpcMultiplexer q0mux = mock(RpcMultiplexer.class); when(q0mux.getAndAdvanceCurrentIndex()).thenReturn(0); fcq.setMultiplexer(q0mux); - Schedulable call = mockCall("c"); + // Make a FCQ filled with calls in q 1 but empty in q 0 + Schedulable call = mockCall("c", 1); fcq.put(call); // Take from q1 even though mux said q0, since q0 empty diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java index 1fa0fff7055..263841246bf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java @@ -19,25 +19,15 @@ package org.apache.hadoop.ipc; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import org.junit.Assert; -import org.junit.Assume; import org.junit.Test; -import org.junit.Before; -import org.junit.After; import java.util.List; import java.io.IOException; -import java.nio.ByteBuffer; -import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.conf.Configuration; @@ -55,16 +45,20 @@ public class TestIdentityProviders { } } + @Override + public int getPriorityLevel() { + return 0; + } } @Test public void testPluggableIdentityProvider() { Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY, + conf.set(CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY, "org.apache.hadoop.ipc.UserIdentityProvider"); List providers = conf.getInstances( - CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY, + CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY, IdentityProvider.class); assertTrue(providers.size() == 1); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 99bfc61c2ea..2ebd1c5e4e1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -43,8 +43,10 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.MetricsAsserts; import org.apache.hadoop.test.MockitoUtil; +import org.apache.log4j.Level; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -956,7 +958,7 @@ public class TestRPC extends TestRpcBase { } /** - * Test RPC backoff. + * Test RPC backoff by queue full. */ @Test (timeout=30000) public void testClientBackOff() throws Exception { @@ -969,7 +971,7 @@ public class TestRPC extends TestRpcBase { final ExecutorService executorService = Executors.newFixedThreadPool(numClients); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); - conf.setBoolean(CommonConfigurationKeys.IPC_CALLQUEUE_NAMESPACE + + conf.setBoolean(CommonConfigurationKeys.IPC_NAMESPACE + ".0." + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true); RPC.Builder builder = newServerBuilder(conf) .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); @@ -1018,6 +1020,92 @@ public class TestRPC extends TestRpcBase { assertTrue("RetriableException not received", succeeded); } + /** + * Test RPC backoff by response time of each priority level. + */ + @Test (timeout=30000) + public void testClientBackOffByResponseTime() throws Exception { + Server server; + final TestRpcService proxy; + boolean succeeded = false; + final int numClients = 1; + final int queueSizePerHandler = 3; + + GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG); + + final List> res = new ArrayList>(); + final ExecutorService executorService = + Executors.newFixedThreadPool(numClients); + conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); + final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0."; + conf.setBoolean(ns + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true); + conf.setStrings(ns + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY, + "org.apache.hadoop.ipc.FairCallQueue"); + conf.setStrings(ns + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY, + "org.apache.hadoop.ipc.DecayRpcScheduler"); + conf.setInt(ns + CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY, + 2); + conf.setBoolean(ns + + DecayRpcScheduler.IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY, + true); + // set a small thresholds 2s and 4s for level 0 and level 1 for testing + conf.set(ns + + DecayRpcScheduler.IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY + , "2s, 4s"); + + // Set max queue size to 3 so that 2 calls from the test won't trigger + // back off because the queue is full. + RPC.Builder builder = newServerBuilder(conf) + .setQueueSizePerHandler(queueSizePerHandler).setNumHandlers(1) + .setVerbose(true); + server = setupTestServer(builder); + + @SuppressWarnings("unchecked") + CallQueueManager spy = spy((CallQueueManager) Whitebox + .getInternalState(server, "callQueue")); + Whitebox.setInternalState(server, "callQueue", spy); + + Exception lastException = null; + proxy = getClient(addr, conf); + try { + // start a sleep RPC call that sleeps 3s. + for (int i = 0; i < numClients; i++) { + res.add(executorService.submit( + new Callable() { + @Override + public Void call() throws ServiceException, InterruptedException { + proxy.sleep(null, newSleepRequest(3000)); + return null; + } + })); + verify(spy, timeout(500).times(i + 1)).offer(Mockito.anyObject()); + } + // Start another sleep RPC call and verify the call is backed off due to + // avg response time(3s) exceeds threshold (2s). + try { + // wait for the 1st response time update + Thread.sleep(5500); + proxy.sleep(null, newSleepRequest(100)); + } catch (ServiceException e) { + RemoteException re = (RemoteException) e.getCause(); + IOException unwrapExeption = re.unwrapRemoteException(); + if (unwrapExeption instanceof RetriableException) { + succeeded = true; + } else { + lastException = unwrapExeption; + } + } + } finally { + executorService.shutdown(); + stop(server, proxy); + } + if (lastException != null) { + LOG.error("Last received non-RetriableException:", lastException); + } + assertTrue("RetriableException not received", succeeded); + } + /** * Test RPC timeout. */ From 6d67420dbc5c6097216fa40fcec8ed626b2bae14 Mon Sep 17 00:00:00 2001 From: Sangjin Lee Date: Thu, 31 Mar 2016 10:49:03 -0700 Subject: [PATCH 27/75] YARN-4183. Clarify the behavior of timeline service config properties (Naganarasimha G R via sjlee) --- .../src/main/resources/yarn-default.xml | 13 ++++++++++--- .../src/site/markdown/TimelineServer.md | 6 +++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 33cd9193f0e..cb3c73a13e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1874,8 +1874,12 @@ - Indicate to clients whether timeline service is enabled or not. - If enabled, clients will put entities and events to the timeline server. + + In the server side it indicates whether timeline service is enabled or not. + And in the client side, users can enable it to indicate whether client wants + to use timeline service. If it's enabled in the client side along with + security, then yarn client tries to fetch the delegation tokens for the + timeline server. yarn.timeline-service.enabled false @@ -2027,7 +2031,10 @@ - Client policy for whether timeline operations are non-fatal + Client policy for whether timeline operations are non-fatal. + Should the failure to obtain a delegation token be considered an application + failure (option = false), or should the client attempt to continue to + publish information without it (option=true) yarn.timeline-service.client.best-effort false diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 8ef7d9a3d20..9283e5829a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -141,7 +141,7 @@ and cluster operators. | Configuration Property | Description | |:---- |:---- | -| `yarn.timeline-service.enabled` | Indicate to clients whether Timeline service is enabled or not. If enabled, the `TimelineClient` library used by applications will post entities and events to the Timeline server. Defaults to `false`. | +| `yarn.timeline-service.enabled` | In the server side it indicates whether timeline service is enabled or not. And in the client side, users can enable it to indicate whether client wants to use timeline service. If it's enabled in the client side along with security, then yarn client tries to fetch the delegation tokens for the timeline server. Defaults to `false`. | | `yarn.resourcemanager.system-metrics-publisher.enabled` | The setting that controls whether or not YARN system metrics are published on the timeline server by RM. Defaults to `false`. | | `yarn.timeline-service.generic-application-history.enabled` | Indicate to clients whether to query generic application data from timeline history-service or not. If not enabled then application data is queried only from Resource Manager. Defaults to `false`. | @@ -150,7 +150,7 @@ and cluster operators. | Configuration Property | Description | |:---- |:---- | | `yarn.timeline-service.store-class` | Store class name for timeline store. Defaults to `org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore`. | -| `yarn.timeline-service.leveldb-timeline-store.path` | Store file name for leveldb timeline store. Defaults to `${hadoop.tmp.dir}/yarn/timelin`e. | +| `yarn.timeline-service.leveldb-timeline-store.path` | Store file name for leveldb timeline store. Defaults to `${hadoop.tmp.dir}/yarn/timeline`. | | `yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms` | Length of time to wait between deletion cycles of leveldb timeline store in milliseconds. Defaults to `300000`. | | `yarn.timeline-service.leveldb-timeline-store.read-cache-size` | Size of read cache for uncompressed blocks for leveldb timeline store in bytes. Defaults to `104857600`. | | `yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size` | Size of cache for recently read entity start times for leveldb timeline store in number of entities. Defaults to `10000`. | @@ -215,7 +215,7 @@ to `kerberos`, after which the following configuration options are available: | `yarn.timeline-service.delegation.key.update-interval` | Defaults to `86400000` (1 day). | | `yarn.timeline-service.delegation.token.renew-interval` | Defaults to `86400000` (1 day). | | `yarn.timeline-service.delegation.token.max-lifetime` | Defaults to `604800000` (7 days). | -| `yarn.timeline-service.best-effort` | Should the failure to obtain a delegation token be considered an application failure (option = false), or should the client attempt to continue to publish information without it (option=true). Default: `false` | +| `yarn.timeline-service.client.best-effort` | Should the failure to obtain a delegation token be considered an application failure (option = false), or should the client attempt to continue to publish information without it (option=true). Default: `false` | #### Enabling the timeline service and the generic history service From 7a021471c376ce846090fbd1a315266bada048d4 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Thu, 31 Mar 2016 13:09:09 -0700 Subject: [PATCH 28/75] YARN-4639. Remove dead code in TestDelegationTokenRenewer added in YARN-3055 (templedf via rkanter) --- .../resourcemanager/security/TestDelegationTokenRenewer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index d85e928b791..1bfac8d9ffa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1090,9 +1090,6 @@ public class TestDelegationTokenRenewer { new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); - //MyFS fs = (MyFS)FileSystem.get(conf); - //MyToken token1 = fs.getDelegationToken("user123"); - // create Token1: Text userText1 = new Text("user"); DelegationTokenIdentifier dtId1 = From 0dd9bcab97ccdf24a2174636604110b74664cf80 Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 31 Mar 2016 14:28:13 -0700 Subject: [PATCH 29/75] YARN-4811. Generate histograms in ContainerMetrics for actual container resource usage --- .../hadoop/metrics2/lib/MutableQuantiles.java | 7 +- .../metrics2/util/QuantileEstimator.java | 32 +++++++++ .../hadoop/metrics2/util/SampleQuantiles.java | 2 +- .../hadoop-yarn-server-nodemanager/pom.xml | 5 ++ .../monitor/ContainerMetrics.java | 69 +++++++++++++++++++ .../monitor/TestContainerMetrics.java | 58 +++++++++++++++- 6 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java index 2e6053ff6ce..a4711db3503 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java @@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.util.Quantile; +import org.apache.hadoop.metrics2.util.QuantileEstimator; import org.apache.hadoop.metrics2.util.SampleQuantiles; import com.google.common.annotations.VisibleForTesting; @@ -54,7 +55,7 @@ public class MutableQuantiles extends MutableMetric { private final MetricsInfo[] quantileInfos; private final int interval; - private SampleQuantiles estimator; + private QuantileEstimator estimator; private long previousCount = 0; @VisibleForTesting @@ -134,6 +135,10 @@ public class MutableQuantiles extends MutableMetric { return interval; } + public synchronized void setEstimator(QuantileEstimator quantileEstimator) { + this.estimator = quantileEstimator; + } + /** * Runnable used to periodically roll over the internal * {@link SampleQuantiles} every interval. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java new file mode 100644 index 00000000000..075b879fc92 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.util; + +import java.util.Map; + +public interface QuantileEstimator { + + void insert(long value); + + Map snapshot(); + + long getCount(); + + void clear(); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java index d54003671d4..0c5d98f2374 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java @@ -47,7 +47,7 @@ import com.google.common.base.Preconditions; * */ @InterfaceAudience.Private -public class SampleQuantiles { +public class SampleQuantiles implements QuantileEstimator { /** * Total number of items in stream diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 051bb4e5fcd..59c333289db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -117,6 +117,11 @@ com.google.protobuf protobuf-java + + + com.codahale.metrics + metrics-core + From 256c82fe2981748cd0befc5490d8118d139908f9 Mon Sep 17 00:00:00 2001 From: Harsh J Date: Fri, 1 Apr 2016 14:18:10 +0530 Subject: [PATCH 35/75] HADOOP-11687. Ignore x-* and response headers when copying an Amazon S3 object. Contributed by Aaron Peterson and harsh. --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 70 ++++++++++++++++++- .../site/markdown/tools/hadoop-aws/index.md | 7 ++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 7ab6c796308..6afb05d04ab 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -26,6 +26,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -1128,7 +1129,7 @@ public class S3AFileSystem extends FileSystem { } ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); - final ObjectMetadata dstom = srcom.clone(); + ObjectMetadata dstom = cloneObjectMetadata(srcom); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { dstom.setSSEAlgorithm(serverSideEncryptionAlgorithm); } @@ -1234,6 +1235,73 @@ public class S3AFileSystem extends FileSystem { statistics.incrementWriteOps(1); } + /** + * Creates a copy of the passed {@link ObjectMetadata}. + * Does so without using the {@link ObjectMetadata#clone()} method, + * to avoid copying unnecessary headers. + * @param source the {@link ObjectMetadata} to copy + * @return a copy of {@link ObjectMetadata} with only relevant attributes + */ + private ObjectMetadata cloneObjectMetadata(ObjectMetadata source) { + // This approach may be too brittle, especially if + // in future there are new attributes added to ObjectMetadata + // that we do not explicitly call to set here + ObjectMetadata ret = new ObjectMetadata(); + + // Non null attributes + ret.setContentLength(source.getContentLength()); + + // Possibly null attributes + // Allowing nulls to pass breaks it during later use + if (source.getCacheControl() != null) { + ret.setCacheControl(source.getCacheControl()); + } + if (source.getContentDisposition() != null) { + ret.setContentDisposition(source.getContentDisposition()); + } + if (source.getContentEncoding() != null) { + ret.setContentEncoding(source.getContentEncoding()); + } + if (source.getContentMD5() != null) { + ret.setContentMD5(source.getContentMD5()); + } + if (source.getContentType() != null) { + ret.setContentType(source.getContentType()); + } + if (source.getExpirationTime() != null) { + ret.setExpirationTime(source.getExpirationTime()); + } + if (source.getExpirationTimeRuleId() != null) { + ret.setExpirationTimeRuleId(source.getExpirationTimeRuleId()); + } + if (source.getHttpExpiresDate() != null) { + ret.setHttpExpiresDate(source.getHttpExpiresDate()); + } + if (source.getLastModified() != null) { + ret.setLastModified(source.getLastModified()); + } + if (source.getOngoingRestore() != null) { + ret.setOngoingRestore(source.getOngoingRestore()); + } + if (source.getRestoreExpirationTime() != null) { + ret.setRestoreExpirationTime(source.getRestoreExpirationTime()); + } + if (source.getSSEAlgorithm() != null) { + ret.setSSEAlgorithm(source.getSSEAlgorithm()); + } + if (source.getSSECustomerAlgorithm() != null) { + ret.setSSECustomerAlgorithm(source.getSSECustomerAlgorithm()); + } + if (source.getSSECustomerKeyMd5() != null) { + ret.setSSECustomerKeyMd5(source.getSSECustomerKeyMd5()); + } + + for (Map.Entry e : source.getUserMetadata().entrySet()) { + ret.addUserMetadata(e.getKey(), e.getValue()); + } + return ret; + } + /** * Return the number of bytes that large input files should be optimally * be split into to minimize i/o time. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index af3541fafb1..7382029f3af 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -417,6 +417,13 @@ which pass in authentication details to the test runner These are both Hadoop XML configuration files, which must be placed into `hadoop-tools/hadoop-aws/src/test/resources`. +### `core-site.xml` + +This file pre-exists and sources the configurations created +under `auth-keys.xml`. + +For most purposes you will not need to edit this file unless you +need to apply a specific, non-default property change during the tests. ### `auth-keys.xml` From 82621e38a0445832998bc00693279e23a98605c1 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Fri, 1 Apr 2016 14:57:06 -0700 Subject: [PATCH 36/75] YARN-4895. Add subtractFrom method to ResourceUtilization class. Contributed by Konstantinos Karanasos. --- .../yarn/api/records/ResourceUtilization.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java index 5f52f85f0c6..2ae4872fbcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -44,6 +44,14 @@ public abstract class ResourceUtilization implements return utilization; } + @Public + @Unstable + public static ResourceUtilization newInstance( + ResourceUtilization resourceUtil) { + return newInstance(resourceUtil.getPhysicalMemory(), + resourceUtil.getVirtualMemory(), resourceUtil.getCPU()); + } + /** * Get used virtual memory. * @@ -147,4 +155,18 @@ public abstract class ResourceUtilization implements this.setVirtualMemory(this.getVirtualMemory() + vmem); this.setCPU(this.getCPU() + cpu); } + + /** + * Subtract utilization from the current one. + * @param pmem Physical memory to be subtracted. + * @param vmem Virtual memory to be subtracted. + * @param cpu CPU utilization to be subtracted. + */ + @Public + @Unstable + public void subtractFrom(int pmem, int vmem, float cpu) { + this.setPhysicalMemory(this.getPhysicalMemory() - pmem); + this.setVirtualMemory(this.getVirtualMemory() - vmem); + this.setCPU(this.getCPU() - cpu); + } } \ No newline at end of file From 5686caa9fcb59759c9286385575f31e407a97c16 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Fri, 1 Apr 2016 15:58:13 -0700 Subject: [PATCH 37/75] Missing file for YARN-4895. --- .../api/records/TestResourceUtilization.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java new file mode 100644 index 00000000000..5934846e2f3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.junit.Assert; +import org.junit.Test; + +public class TestResourceUtilization { + + @Test + public void testResourceUtilization() { + ResourceUtilization u1 = ResourceUtilization.newInstance(10, 20, 0.5f); + ResourceUtilization u2 = ResourceUtilization.newInstance(u1); + ResourceUtilization u3 = ResourceUtilization.newInstance(10, 20, 0.5f); + ResourceUtilization u4 = ResourceUtilization.newInstance(20, 20, 0.5f); + ResourceUtilization u5 = ResourceUtilization.newInstance(30, 40, 0.8f); + + Assert.assertEquals(u1, u2); + Assert.assertEquals(u1, u3); + Assert.assertNotEquals(u1, u4); + Assert.assertNotEquals(u2, u5); + Assert.assertNotEquals(u4, u5); + + Assert.assertTrue(u1.hashCode() == u2.hashCode()); + Assert.assertTrue(u1.hashCode() == u3.hashCode()); + Assert.assertFalse(u1.hashCode() == u4.hashCode()); + Assert.assertFalse(u2.hashCode() == u5.hashCode()); + Assert.assertFalse(u4.hashCode() == u5.hashCode()); + + Assert.assertTrue(u1.getPhysicalMemory() == 10); + Assert.assertFalse(u1.getVirtualMemory() == 10); + Assert.assertTrue(u1.getCPU() == 0.5f); + + Assert.assertEquals("", u1.toString()); + + u1.addTo(10, 0, 0.0f); + Assert.assertNotEquals(u1, u2); + Assert.assertEquals(u1, u4); + u1.addTo(10, 20, 0.3f); + Assert.assertEquals(u1, u5); + u1.subtractFrom(10, 20, 0.3f); + Assert.assertEquals(u1, u4); + u1.subtractFrom(10, 0, 0.0f); + Assert.assertEquals(u1, u3); + } +} From 81d04cae41182808ace5d86cdac7e4d71871eb1e Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 1 Apr 2016 16:19:54 -0700 Subject: [PATCH 38/75] YARN-4657. Javadoc comment is broken for Resources.multiplyByAndAddTo(). (Daniel Templeton via kasha) --- .../java/org/apache/hadoop/yarn/util/resource/Resources.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index b05d021ae27..558f96c7446 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -152,7 +152,7 @@ public class Resources { } /** - * Multiply @param rhs by @param by, and add the result to @param lhs + * Multiply {@code rhs} by {@code by}, and add the result to {@code lhs} * without creating any new {@link Resource} object */ public static Resource multiplyAndAddTo( From 54b2e78fd28c9def42bec7f0418833bad352686c Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Sun, 3 Apr 2016 13:27:49 +0530 Subject: [PATCH 39/75] HDFS-10253. Fix TestRefreshCallQueue failure (Contributed by Xiaoyu Yao) --- .../src/test/java/org/apache/hadoop/TestRefreshCallQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java index 1be2752fe0a..5cb7def8808 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java @@ -92,7 +92,7 @@ public class TestRefreshCallQueue { @SuppressWarnings("serial") public static class MockCallQueue extends LinkedBlockingQueue { - public MockCallQueue(int cap, String ns, Configuration conf) { + public MockCallQueue(int levels, int cap, String ns, Configuration conf) { super(cap); mockQueueConstructions++; } From 0ecdd4cffa51e34997321c384496efc249e3d8ff Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sun, 3 Apr 2016 16:39:14 +0100 Subject: [PATCH 40/75] HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse. --- .../AbstractContractGetFileStatusTest.java | 23 +++++++++++++++++++ .../apache/hadoop/fs/s3a/S3AFileSystem.java | 7 ++++-- .../src/test/resources/contract/s3a.xml | 5 ++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java index 7ed375e648c..3e5bb12f7ad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java @@ -19,10 +19,14 @@ package org.apache.hadoop.fs.contract; import java.io.FileNotFoundException; +import java.io.IOException; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends handleExpectedException(e); } } + + @Test + public void testListStatusEmptyDirectory() throws IOException { + // remove the test directory + FileSystem fs = getFileSystem(); + assertTrue(fs.delete(getContract().getTestPath(), true)); + + // create a - non-qualified - Path for a subdir + Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName()); + assertTrue(fs.mkdirs(subfolder)); + + // assert empty ls on the empty dir + assertEquals("ls on an empty directory not of length 0", 0, + fs.listStatus(subfolder).length); + + // assert non-empty ls on parent dir + assertTrue("ls on a non-empty directory of length 0", + fs.listStatus(getContract().getTestPath()).length > 0); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6afb05d04ab..fe705cef83c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -788,11 +788,14 @@ public class S3AFileSystem extends FileSystem { ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); + Path fQualified = f.makeQualified(uri, workingDir); + while (true) { for (S3ObjectSummary summary : objects.getObjectSummaries()) { Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir); // Skip over keys that are ourselves and old S3N _$folder$ files - if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) { + if (keyPath.equals(fQualified) || + summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) { if (LOG.isDebugEnabled()) { LOG.debug("Ignoring: " + keyPath); } @@ -807,7 +810,7 @@ public class S3AFileSystem extends FileSystem { } else { result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()), keyPath, - getDefaultBlockSize(f.makeQualified(uri, workingDir)))); + getDefaultBlockSize(fQualified))); if (LOG.isDebugEnabled()) { LOG.debug("Adding: fi: " + keyPath); } diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml index 4f9c0818ffa..be1e7ca6535 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml @@ -77,6 +77,11 @@ false + + fs.contract.supports-getfilestatus + true + + fs.contract.supports-seek true From 1e6f92977dc5431b117745feb5a3491e88a559c0 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Mon, 4 Apr 2016 08:09:29 +0530 Subject: [PATCH 41/75] YARN-4607. Pagination support for AppAttempt page TotalOutstandingResource Requests table. Contributed by Bibin A Chundatt --- .../yarn/server/webapp/WebPageUtils.java | 6 ++ .../webapp/AppAttemptPage.java | 4 +- .../webapp/RMAppAttemptBlock.java | 61 +++++++++++-------- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index a07baa2366f..3a26ae58890 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -95,4 +95,10 @@ public class WebPageUtils { .append(", 'mRender': parseHadoopID }]").toString(); } + public static String resourceRequestsTableInit() { + return tableInit().append(", 'aaData': resourceRequestsTableData") + .append(", bDeferRender: true").append(", bProcessing: true}") + .toString(); + } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java index df5fb9e8a59..45f188782d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java @@ -41,8 +41,10 @@ public class AppAttemptPage extends RmView { : join("Application Attempt ", $(YarnWebParams.APPLICATION_ATTEMPT_ID))); - set(DATATABLES_ID, "containers"); + set(DATATABLES_ID, "containers resourceRequests"); set(initID(DATATABLES, "containers"), WebPageUtils.containersTableInit()); + set(initID(DATATABLES, "resourceRequests"), + WebPageUtils.resourceRequestsTableInit()); setTableStyles(html, "containers", ".queue {width:6em}", ".ui {width:8em}"); set(YarnWebParams.WEB_UI_TYPE, YarnWebParams.RM_WEB_UI); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 668269e819a..6fef3671dd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH; import java.util.Collection; import java.util.List; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; @@ -46,6 +47,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -75,35 +77,44 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ } DIV div = html.div(_INFO_WRAP); - TABLE> table = - div.h3("Total Outstanding Resource Requests: " - + getTotalResource(resourceRequests)).table( - "#ResourceRequests"); + // Requests Table + TBODY>> tbody = div + .h3("Total Outstanding Resource Requests: " + + getTotalResource(resourceRequests)) + .table("#resourceRequests").thead().tr().th(".priority", "Priority") + .th(".resource", "ResourceName").th(".capacity", "Capability") + .th(".containers", "NumContainers") + .th(".relaxlocality", "RelaxLocality") + .th(".labelexpression", "NodeLabelExpression")._()._().tbody(); - table.tr(). - th(_TH, "Priority"). - th(_TH, "ResourceName"). - th(_TH, "Capability"). - th(_TH, "NumContainers"). - th(_TH, "RelaxLocality"). - th(_TH, "NodeLabelExpression"). - _(); - - boolean odd = false; - for (ResourceRequest request : resourceRequests) { - if (request.getNumContainers() == 0) { + StringBuilder resourceRequestTableData = new StringBuilder("[\n"); + for (ResourceRequest resourceRequest : resourceRequests) { + if (resourceRequest.getNumContainers() == 0) { continue; } - table.tr((odd = !odd) ? _ODD : _EVEN) - .td(String.valueOf(request.getPriority())) - .td(request.getResourceName()) - .td(String.valueOf(request.getCapability())) - .td(String.valueOf(request.getNumContainers())) - .td(String.valueOf(request.getRelaxLocality())) - .td(request.getNodeLabelExpression() == null ? "N/A" : request - .getNodeLabelExpression())._(); + resourceRequestTableData.append("[\"") + .append(String.valueOf(resourceRequest.getPriority())).append("\",\"") + .append(resourceRequest.getResourceName()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils + .escapeHtml(String.valueOf(resourceRequest.getCapability())))) + .append("\",\"") + .append(String.valueOf(resourceRequest.getNumContainers())) + .append("\",\"") + .append(String.valueOf(resourceRequest.getRelaxLocality())) + .append("\",\"") + .append(resourceRequest.getNodeLabelExpression() == null ? "N/A" + : resourceRequest.getNodeLabelExpression()) + .append("\"],\n"); } - table._(); + if (resourceRequestTableData + .charAt(resourceRequestTableData.length() - 2) == ',') { + resourceRequestTableData.delete(resourceRequestTableData.length() - 2, + resourceRequestTableData.length() - 1); + } + resourceRequestTableData.append("]"); + html.script().$type("text/javascript") + ._("var resourceRequestsTableData=" + resourceRequestTableData)._(); + tbody._()._(); div._(); } From da614ca5dc26562d7ecd5d7c5743fa52c3c17342 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 4 Apr 2016 17:46:56 +0900 Subject: [PATCH 42/75] HADOOP-12967. Remove FileUtil#copyMerge. Contributed by Brahma Reddy Battula. --- .../java/org/apache/hadoop/fs/FileUtil.java | 42 -------------- .../org/apache/hadoop/fs/TestFileUtil.java | 58 ------------------- 2 files changed, 100 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index b855c4851a5..e2d6ecdc07b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -23,7 +23,6 @@ import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.Map; @@ -381,47 +380,6 @@ public class FileUtil { } - @Deprecated - /** Copy all files in a directory to one output file (merge). */ - public static boolean copyMerge(FileSystem srcFS, Path srcDir, - FileSystem dstFS, Path dstFile, - boolean deleteSource, - Configuration conf, String addString) throws IOException { - dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false); - - if (!srcFS.getFileStatus(srcDir).isDirectory()) - return false; - - OutputStream out = dstFS.create(dstFile); - - try { - FileStatus contents[] = srcFS.listStatus(srcDir); - Arrays.sort(contents); - for (int i = 0; i < contents.length; i++) { - if (contents[i].isFile()) { - InputStream in = srcFS.open(contents[i].getPath()); - try { - IOUtils.copyBytes(in, out, conf, false); - if (addString!=null) - out.write(addString.getBytes("UTF-8")); - - } finally { - in.close(); - } - } - } - } finally { - out.close(); - } - - - if (deleteSource) { - return srcFS.delete(srcDir, true); - } else { - return true; - } - } - /** Copy local files to a FileSystem. */ public static boolean copy(File src, FileSystem dstFS, Path dst, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index f7464b7fce7..a9ef5c04280 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -19,11 +19,9 @@ package org.apache.hadoop.fs; import org.junit.*; -import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; import java.net.InetAddress; @@ -49,7 +47,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.tools.tar.TarEntry; import org.apache.tools.tar.TarOutputStream; -import javax.print.attribute.URISyntax; import static org.junit.Assert.*; import static org.mockito.Mockito.mock; @@ -526,61 +523,6 @@ public class TestFileUtil { validateAndSetWritablePermissions(false, ret); } - @Test (timeout = 30000) - public void testCopyMergeSingleDirectory() throws IOException { - setupDirs(); - boolean copyMergeResult = copyMerge("partitioned", "tmp/merged"); - Assert.assertTrue("Expected successful copyMerge result.", copyMergeResult); - File merged = new File(TEST_DIR, "tmp/merged"); - Assert.assertTrue("File tmp/merged must exist after copyMerge.", - merged.exists()); - BufferedReader rdr = new BufferedReader(new FileReader(merged)); - - try { - Assert.assertEquals("Line 1 of merged file must contain \"foo\".", - "foo", rdr.readLine()); - Assert.assertEquals("Line 2 of merged file must contain \"bar\".", - "bar", rdr.readLine()); - Assert.assertNull("Expected end of file reading merged file.", - rdr.readLine()); - } - finally { - rdr.close(); - } - } - - /** - * Calls FileUtil.copyMerge using the specified source and destination paths. - * Both source and destination are assumed to be on the local file system. - * The call will not delete source on completion and will not add an - * additional string between files. - * @param src String non-null source path. - * @param dst String non-null destination path. - * @return boolean true if the call to FileUtil.copyMerge was successful. - * @throws IOException if an I/O error occurs. - */ - @SuppressWarnings("deprecation") - private boolean copyMerge(String src, String dst) - throws IOException { - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.getLocal(conf); - final boolean result; - - try { - Path srcPath = new Path(TEST_ROOT_DIR, src); - Path dstPath = new Path(TEST_ROOT_DIR, dst); - boolean deleteSource = false; - String addString = null; - result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf, - addString); - } - finally { - fs.close(); - } - - return result; - } - /** * Test that getDU is able to handle cycles caused due to symbolic links * and that directory sizes are not added to the final calculated size From 5092c94195a63bd2c3e36d5a74b4c061cea1b847 Mon Sep 17 00:00:00 2001 From: naganarasimha Date: Mon, 4 Apr 2016 16:25:03 +0530 Subject: [PATCH 43/75] YARN-4746. yarn web services should convert parse failures of appId, appAttemptId and containerId to 400. Contributed by Bibin A Chundatt --- .../hadoop/yarn/util/ConverterUtils.java | 16 ++++++++-- .../hadoop/yarn/webapp/util/WebAppUtils.java | 22 +++++++++++++ .../yarn/server/webapp/WebServices.java | 22 ++++++++++--- .../nodemanager/webapp/NMWebServices.java | 6 ++-- .../webapp/TestNMWebServicesApps.java | 9 ++++-- .../resourcemanager/webapp/RMWebServices.java | 32 ++----------------- .../webapp/TestRMWebServicesApps.java | 24 ++++++++------ .../TestRMWebServicesAppsModification.java | 10 ++++-- 8 files changed, 87 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index e9674cfc147..acd29fb02d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -122,8 +122,20 @@ public class ConverterUtils { public static ApplicationId toApplicationId(RecordFactory recordFactory, String appIdStr) { Iterator it = _split(appIdStr).iterator(); - it.next(); // prefix. TODO: Validate application prefix - return toApplicationId(recordFactory, it); + if (!it.next().equals(APPLICATION_PREFIX)) { + throw new IllegalArgumentException("Invalid ApplicationId prefix: " + + appIdStr + ". The valid ApplicationId should start with prefix " + + APPLICATION_PREFIX); + } + try { + return toApplicationId(recordFactory, it); + } catch (NumberFormatException n) { + throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, + n); + } catch (NoSuchElementException e) { + throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, + e); + } } private static ApplicationId toApplicationId(RecordFactory recordFactory, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index f8e67ee1fb3..faf4a774447 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -33,9 +33,14 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.NotFoundException; @Private @Evolving @@ -378,4 +383,21 @@ public class WebAppUtils { } return password; } + + public static ApplicationId parseApplicationId(RecordFactory recordFactory, + String appId) { + if (appId == null || appId.isEmpty()) { + throw new NotFoundException("appId, " + appId + ", is empty or null"); + } + ApplicationId aid = null; + try { + aid = ConverterUtils.toApplicationId(recordFactory, appId); + } catch (Exception e) { + throw new BadRequestException(e); + } + if (aid == null) { + throw new NotFoundException("app with id " + appId + " not found"); + } + return aid; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 40e40c98421..19ea30136e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -429,7 +429,12 @@ public class WebServices { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } - ApplicationId aid = ConverterUtils.toApplicationId(appId); + ApplicationId aid = null; + try { + aid = ConverterUtils.toApplicationId(appId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (aid == null) { throw new NotFoundException("appId is null"); } @@ -442,8 +447,12 @@ public class WebServices { throw new NotFoundException("appAttemptId, " + appAttemptId + ", is empty or null"); } - ApplicationAttemptId aaid = - ConverterUtils.toApplicationAttemptId(appAttemptId); + ApplicationAttemptId aaid = null; + try { + aaid = ConverterUtils.toApplicationAttemptId(appAttemptId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (aaid == null) { throw new NotFoundException("appAttemptId is null"); } @@ -455,7 +464,12 @@ public class WebServices { throw new NotFoundException("containerId, " + containerId + ", is empty or null"); } - ContainerId cid = ConverterUtils.toContainerId(containerId); + ContainerId cid = null; + try { + cid = ConverterUtils.toContainerId(containerId); + } catch (Exception e) { + throw new BadRequestException(e); + } if (cid == null) { throw new NotFoundException("containerId is null"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index de6d219deaa..fddeb042e5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -143,10 +144,7 @@ public class NMWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getNodeApp(@PathParam("appid") String appId) { init(); - ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("app with id " + appId + " not found"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); Application app = this.nmContext.getApplications().get(id); if (app == null) { throw new NotFoundException("app with id " + appId + " not found"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index e274abb2aac..3cd8faf0389 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -565,11 +565,14 @@ public class TestNMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"foo\"", message); + "java.lang.IllegalArgumentException: Invalid ApplicationId prefix: " + + "app_foo_0000. The valid ApplicationId should start with prefix" + + " application", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 0ed8a4ed00f..8036af42f05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -662,14 +662,7 @@ public class RMWebServices extends WebServices { public AppInfo getApp(@Context HttpServletRequest hsr, @PathParam("appid") String appId) { init(); - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("appId is null"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); @@ -684,14 +677,7 @@ public class RMWebServices extends WebServices { @PathParam("appid") String appId) { init(); - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - id = ConverterUtils.toApplicationId(recordFactory, appId); - if (id == null) { - throw new NotFoundException("appId is null"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); @@ -1303,19 +1289,7 @@ public class RMWebServices extends WebServices { } private RMApp getRMAppForAppId(String appId) { - - if (appId == null || appId.isEmpty()) { - throw new NotFoundException("appId, " + appId + ", is empty or null"); - } - ApplicationId id; - try { - id = ConverterUtils.toApplicationId(recordFactory, appId); - } catch (NumberFormatException e) { - throw new NotFoundException("appId is invalid"); - } - if (id == null) { - throw new NotFoundException("appId is invalid"); - } + ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 0328a7a291b..72cccf66420 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1194,11 +1194,13 @@ public class TestRMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"invalid\"", message); + "java.lang.IllegalArgumentException: Invalid ApplicationId:" + + " application_invalid_12", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); @@ -1500,15 +1502,17 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void testInvalidAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(CONTAINER_MB); + RMApp app = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); try { r.path("ws").path("v1").path("cluster").path("apps") - .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) + .path(app.getApplicationId().toString()).path("appattempts") + .path("appattempt_invalid_12_000001") + .accept(MediaType.APPLICATION_JSON) .get(JSONObject.class); - fail("should have thrown exception on invalid appid"); + fail("should have thrown exception on invalid appAttempt"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1521,11 +1525,13 @@ public class TestRMWebServicesApps extends JerseyTestBase { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"invalid\"", message); + "java.lang.IllegalArgumentException: Invalid AppAttemptId:" + + " appattempt_invalid_12_000001", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 9abcf9c8d5b..38b32e96e17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -568,17 +568,21 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); amNodeManager.nodeHeartbeat(true); String[] testAppIds = { "application_1391705042196_0001", "random_string" }; - for (String testAppId : testAppIds) { + for (int i = 0; i < testAppIds.length; i++) { AppState info = new AppState("KILLED"); ClientResponse response = - this.constructWebResource("apps", testAppId, "state") + this.constructWebResource("apps", testAppIds[i], "state") .accept(MediaType.APPLICATION_XML) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); continue; } - assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + if (i == 0) { + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + } else { + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + } } rm.stop(); } From 89c93475116ee475645cf81cc80f155f830e61de Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 4 Apr 2016 17:00:35 +0100 Subject: [PATCH 44/75] HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse. - omitted new S3A subclass --- .../s3a/TestS3AContractGetFileStatus.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java new file mode 100644 index 00000000000..d7b8fe33fdc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; + +public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + +} From 7280550a8f668df8aa32e4630db4ead49e9b8b6d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 4 Apr 2016 10:50:11 -0700 Subject: [PATCH 45/75] HADOOP-11212. NetUtils.wrapException to handle SocketException explicitly. (Contributed by Steve Loughran) --- .../java/org/apache/hadoop/net/NetUtils.java | 15 ++++-- .../org/apache/hadoop/net/TestNetUtils.java | 47 ++++++++++--------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index 2c3661aaf2c..40501073fdc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -782,12 +782,21 @@ public class NetUtils { + ": " + exception + ";" + see("EOFException")); + } else if (exception instanceof SocketException) { + // Many of the predecessor exceptions are subclasses of SocketException, + // so must be handled before this + return wrapWithMessage(exception, + "Call From " + + localHost + " to " + destHost + ":" + destPort + + " failed on socket exception: " + exception + + ";" + + see("SocketException")); } else { return (IOException) new IOException("Failed on local exception: " - + exception - + "; Host Details : " - + getHostDetailsAsString(destHost, destPort, localHost)) + + exception + + "; Host Details : " + + getHostDetailsAsString(destHost, destPort, localHost)) .initCause(exception); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index c93ede8bf69..e59ac779828 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -72,7 +72,7 @@ public class TestNetUtils { * This is a regression test for HADOOP-6722. */ @Test - public void testAvoidLoopbackTcpSockets() throws Exception { + public void testAvoidLoopbackTcpSockets() throws Throwable { Configuration conf = new Configuration(); Socket socket = NetUtils.getDefaultSocketFactory(conf) @@ -88,11 +88,11 @@ public class TestNetUtils { fail("Should not have connected"); } catch (ConnectException ce) { System.err.println("Got exception: " + ce); - assertTrue(ce.getMessage().contains("resulted in a loopback")); + assertInException(ce, "resulted in a loopback"); } catch (SocketException se) { // Some TCP stacks will actually throw their own Invalid argument exception // here. This is also OK. - assertTrue(se.getMessage().contains("Invalid argument")); + assertInException(se, "Invalid argument"); } } @@ -188,15 +188,11 @@ public class TestNetUtils { } @Test - public void testVerifyHostnamesNoException() { + public void testVerifyHostnamesNoException() throws UnknownHostException { String[] names = {"valid.host.com", "1.com"}; - try { - NetUtils.verifyHostnames(names); - } catch (UnknownHostException e) { - fail("NetUtils.verifyHostnames threw unexpected UnknownHostException"); - } + NetUtils.verifyHostnames(names); } - + /** * Test for {@link NetUtils#isLocalAddress(java.net.InetAddress)} */ @@ -267,7 +263,18 @@ public class TestNetUtils { assertRemoteDetailsIncluded(wrapped); assertInException(wrapped, "/EOFException"); } - + + @Test + public void testWrapSocketException() throws Throwable { + IOException wrapped = verifyExceptionClass(new SocketException("failed"), + SocketException.class); + assertInException(wrapped, "failed"); + assertWikified(wrapped); + assertInException(wrapped, "localhost"); + assertRemoteDetailsIncluded(wrapped); + assertInException(wrapped, "/SocketException"); + } + @Test public void testGetConnectAddress() throws IOException { NetUtils.addStaticResolution("host", "127.0.0.1"); @@ -322,8 +329,8 @@ public class TestNetUtils { String message = extractExceptionMessage(e); if (!(message.contains(text))) { throw new AssertionFailedError("Wrong text in message " - + "\"" + message + "\"" - + " expected \"" + text + "\"") + + "\"" + message + "\"" + + " expected \"" + text + "\"") .initCause(e); } } @@ -343,8 +350,8 @@ public class TestNetUtils { String message = extractExceptionMessage(e); if (message.contains(text)) { throw new AssertionFailedError("Wrong text in message " - + "\"" + message + "\"" - + " did not expect \"" + text + "\"") + + "\"" + message + "\"" + + " did not expect \"" + text + "\"") .initCause(e); } } @@ -353,15 +360,13 @@ public class TestNetUtils { Class expectedClass) throws Throwable { assertNotNull("Null Exception", e); - IOException wrapped = - NetUtils.wrapException("desthost", DEST_PORT, - "localhost", LOCAL_PORT, - e); + IOException wrapped = NetUtils.wrapException("desthost", DEST_PORT, + "localhost", LOCAL_PORT, e); LOG.info(wrapped.toString(), wrapped); if(!(wrapped.getClass().equals(expectedClass))) { throw new AssertionFailedError("Wrong exception class; expected " - + expectedClass - + " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped); + + expectedClass + + " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped); } return wrapped; } From 154d2532cf015e9ab9141864bd3ab0d6100ef597 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Tue, 5 Apr 2016 03:19:48 +0900 Subject: [PATCH 46/75] HDFS-9599. TestDecommissioningStatus.testDecommissionStatus occasionally fails (Lin Yiqun via iwasakims) --- .../server/namenode/TestDecommissioningStatus.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java index 789ee6f4ed5..1e7312ae669 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java @@ -57,8 +57,8 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; /** @@ -78,8 +78,8 @@ public class TestDecommissioningStatus { final ArrayList decommissionedNodes = new ArrayList(numDatanodes); - @BeforeClass - public static void setUp() throws Exception { + @Before + public void setUp() throws Exception { conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); @@ -113,8 +113,8 @@ public class TestDecommissioningStatus { Logger.getLogger(DecommissionManager.class).setLevel(Level.DEBUG); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { if (localFileSys != null ) cleanupFile(localFileSys, dir); if(fileSys != null) fileSys.close(); if(cluster != null) cluster.shutdown(); From a7d1fb0cd2fdbf830602eb4dbbd9bbe62f4d5584 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Mon, 4 Apr 2016 16:39:23 -0500 Subject: [PATCH 47/75] HDFS-10178. Permanent write failures can happen if pipeline recoveries occur for the first packet. Contributed by Kihwal Lee. --- .../hdfs/server/datanode/BlockReceiver.java | 2 + .../hdfs/server/datanode/BlockSender.java | 6 ++- .../datanode/DataNodeFaultInjector.java | 2 + ...TestClientProtocolForPipelineRecovery.java | 53 +++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 2e4ee028473..fb0c1c5b1c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -574,6 +574,8 @@ class BlockReceiver implements Closeable { if (mirrorOut != null && !mirrorError) { try { long begin = Time.monotonicNow(); + // For testing. Normally no-op. + DataNodeFaultInjector.get().stopSendingPacketDownstream(); packetReceiver.mirrorPacketTo(mirrorOut); mirrorOut.flush(); long now = Time.monotonicNow(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 773a64c9a67..398935de451 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -300,11 +300,15 @@ class BlockSender implements java.io.Closeable { // The meta file will contain only the header if the NULL checksum // type was used, or if the replica was written to transient storage. + // Also, when only header portion of a data packet was transferred + // and then pipeline breaks, the meta file can contain only the + // header and 0 byte in the block data file. // Checksum verification is not performed for replicas on transient // storage. The header is important for determining the checksum // type later when lazy persistence copies the block to non-transient // storage and computes the checksum. - if (metaIn.getLength() > BlockMetadataHeader.getHeaderSize()) { + if (!replica.isOnTransientStorage() && + metaIn.getLength() >= BlockMetadataHeader.getHeaderSize()) { checksumIn = new DataInputStream(new BufferedInputStream( metaIn, IO_FILE_BUFFER_SIZE)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java index 0e3869429dd..732742098a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java @@ -50,5 +50,7 @@ public class DataNodeFaultInjector { return false; } + public void stopSendingPacketDownstream() throws IOException {} + public void noRegistration() throws IOException { } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java index d7aa79af3aa..0eeb3b78edf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java @@ -375,4 +375,57 @@ public class TestClientProtocolForPipelineRecovery { } } } + + /** + * Test to make sure the checksum is set correctly after pipeline + * recovery transfers 0 byte partial block. If fails the test case + * will say "java.io.IOException: Failed to replace a bad datanode + * on the existing pipeline due to no more good datanodes being + * available to try." This indicates there was a real failure + * after the staged failure. + */ + @Test + public void testZeroByteBlockRecovery() throws Exception { + // Make the first datanode fail once. With 3 nodes and a block being + // created with 2 replicas, anything more than this planned failure + // will cause a test failure. + DataNodeFaultInjector dnFaultInjector = new DataNodeFaultInjector() { + int tries = 1; + @Override + public void stopSendingPacketDownstream() throws IOException { + if (tries > 0) { + tries--; + try { + Thread.sleep(60000); + } catch (InterruptedException ie) { + throw new IOException("Interrupted while sleeping. Bailing out."); + } + } + } + }; + DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.get(); + DataNodeFaultInjector.set(dnFaultInjector); + + Configuration conf = new HdfsConfiguration(); + conf.set(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, "1000"); + conf.set(HdfsClientConfigKeys. + BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY, "ALWAYS"); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster.waitActive(); + + FileSystem fs = cluster.getFileSystem(); + FSDataOutputStream out = fs.create(new Path("noheartbeat.dat"), (short)2); + out.write(0x31); + out.hflush(); + out.close(); + + } finally { + if (cluster != null) { + cluster.shutdown(); + } + DataNodeFaultInjector.set(oldDnInjector); + } + } } From f61de4173684aa1767cef20b3cb4d54df20273cd Mon Sep 17 00:00:00 2001 From: Li Lu Date: Mon, 4 Apr 2016 14:39:47 -0700 Subject: [PATCH 48/75] YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9) --- .../hadoop-yarn-site/src/site/markdown/TimelineServer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 9283e5829a2..f20bd2cb7ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -56,6 +56,7 @@ With the introduction of the timeline server, the Application History Server bec the Timeline Server. Generic information includes application level data such as + * queue-name, * user information and the like set in the `ApplicationSubmissionContext`, * a list of application-attempts that ran for an application @@ -192,6 +193,7 @@ selected if this policy is `HTTPS_ONLY`. #### UI Hosting Configuration The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured. + | Configuration Property | Description | |:---- |:---- | | `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. | From f65f5b18fd4647e868b8d2a2c035a3b64dc16aa8 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Mon, 4 Apr 2016 16:30:32 -0700 Subject: [PATCH 49/75] HADOOP-12959. Add additional github web site for ISA-L library (Li Bo via cmccabe) --- BUILDING.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BUILDING.txt b/BUILDING.txt index 408cae13055..c7a91da5d0d 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -75,6 +75,7 @@ Optional packages: $ sudo apt-get install snappy libsnappy-dev * Intel ISA-L library for erasure coding Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version + (OR https://github.com/01org/isa-l) * Bzip2 $ sudo apt-get install bzip2 libbz2-dev * Jansson (C Library for JSON) @@ -188,11 +189,12 @@ Maven build goals: Intel ISA-L build options: - Intel ISA-L is a erasure coding library that can be utilized by the native code. + Intel ISA-L is an erasure coding library that can be utilized by the native code. It is currently an optional component, meaning that Hadoop can be built with or without this dependency. Note the library is used via dynamic module. Please reference the official site for the library details. https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version + (OR https://github.com/01org/isa-l) * Use -Drequire.isal to fail the build if libisal.so is not found. If this option is not specified and the isal library is missing, From f6b1a818124cc42688c4c5acaf537d96cf00e43b Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Mon, 4 Apr 2016 18:00:26 -0700 Subject: [PATCH 50/75] HDFS-8496. Calling stopWriter() with FSDatasetImpl lock held may block other threads (cmccabe) --- .../server/datanode/ReplicaInPipeline.java | 54 ++++--- .../fsdataset/impl/FsDatasetImpl.java | 145 ++++++++++++------ .../datanode/fsdataset/impl/ReplicaMap.java | 2 +- .../server/datanode/TestBlockRecovery.java | 137 +++++++++++++++-- 4 files changed, 257 insertions(+), 81 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java index 5caca15684c..732684692fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java @@ -22,6 +22,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.RandomAccessFile; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; @@ -44,7 +45,7 @@ public class ReplicaInPipeline extends ReplicaInfo private long bytesAcked; private long bytesOnDisk; private byte[] lastChecksum; - private Thread writer; + private AtomicReference writer = new AtomicReference(); /** * Bytes reserved for this replica on the containing volume. @@ -97,7 +98,7 @@ public class ReplicaInPipeline extends ReplicaInfo super( blockId, len, genStamp, vol, dir); this.bytesAcked = len; this.bytesOnDisk = len; - this.writer = writer; + this.writer.set(writer); this.bytesReserved = bytesToReserve; this.originalBytesReserved = bytesToReserve; } @@ -110,7 +111,7 @@ public class ReplicaInPipeline extends ReplicaInfo super(from); this.bytesAcked = from.getBytesAcked(); this.bytesOnDisk = from.getBytesOnDisk(); - this.writer = from.writer; + this.writer.set(from.writer.get()); this.bytesReserved = from.bytesReserved; this.originalBytesReserved = from.originalBytesReserved; } @@ -175,18 +176,11 @@ public class ReplicaInPipeline extends ReplicaInfo return new ChunkChecksum(getBytesOnDisk(), lastChecksum); } - /** - * Set the thread that is writing to this replica - * @param writer a thread writing to this replica - */ - public void setWriter(Thread writer) { - this.writer = writer; - } - public void interruptThread() { - if (writer != null && writer != Thread.currentThread() - && writer.isAlive()) { - this.writer.interrupt(); + Thread thread = writer.get(); + if (thread != null && thread != Thread.currentThread() + && thread.isAlive()) { + thread.interrupt(); } } @@ -195,18 +189,36 @@ public class ReplicaInPipeline extends ReplicaInfo return super.equals(o); } + /** + * Attempt to set the writer to a new value. + */ + public boolean attemptToSetWriter(Thread prevWriter, Thread newWriter) { + return writer.compareAndSet(prevWriter, newWriter); + } + /** * Interrupt the writing thread and wait until it dies * @throws IOException the waiting is interrupted */ public void stopWriter(long xceiverStopTimeout) throws IOException { - if (writer != null && writer != Thread.currentThread() && writer.isAlive()) { - writer.interrupt(); + while (true) { + Thread thread = writer.get(); + if ((thread == null) || (thread == Thread.currentThread()) || + (!thread.isAlive())) { + if (writer.compareAndSet(thread, null) == true) { + return; // Done + } + // The writer changed. Go back to the start of the loop and attempt to + // stop the new writer. + continue; + } + thread.interrupt(); try { - writer.join(xceiverStopTimeout); - if (writer.isAlive()) { - final String msg = "Join on writer thread " + writer + " timed out"; - DataNode.LOG.warn(msg + "\n" + StringUtils.getStackTrace(writer)); + thread.join(xceiverStopTimeout); + if (thread.isAlive()) { + // Our thread join timed out. + final String msg = "Join on writer thread " + thread + " timed out"; + DataNode.LOG.warn(msg + "\n" + StringUtils.getStackTrace(thread)); throw new IOException(msg); } } catch (InterruptedException e) { @@ -214,7 +226,7 @@ public class ReplicaInPipeline extends ReplicaInfo } } } - + @Override // Object public int hashCode() { return super.hashCode(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index d6a0df65cac..240345c654b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1207,8 +1207,20 @@ class FsDatasetImpl implements FsDatasetSpi { return newReplicaInfo; } + private static class MustStopExistingWriter extends Exception { + private final ReplicaInPipeline rip; + + MustStopExistingWriter(ReplicaInPipeline rip) { + this.rip = rip; + } + + ReplicaInPipeline getReplica() { + return rip; + } + } + private ReplicaInfo recoverCheck(ExtendedBlock b, long newGS, - long expectedBlockLen) throws IOException { + long expectedBlockLen) throws IOException, MustStopExistingWriter { ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId()); // check state @@ -1232,9 +1244,9 @@ class FsDatasetImpl implements FsDatasetSpi { long replicaLen = replicaInfo.getNumBytes(); if (replicaInfo.getState() == ReplicaState.RBW) { ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; - // kill the previous writer - rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); - rbw.setWriter(Thread.currentThread()); + if (!rbw.attemptToSetWriter(null, Thread.currentThread())) { + throw new MustStopExistingWriter(rbw); + } // check length: bytesRcvd, bytesOnDisk, and bytesAcked should be the same if (replicaLen != rbw.getBytesOnDisk() || replicaLen != rbw.getBytesAcked()) { @@ -1260,39 +1272,55 @@ class FsDatasetImpl implements FsDatasetSpi { ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed append to " + b); - ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); + while (true) { + try { + synchronized (this) { + ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); - FsVolumeReference ref = replicaInfo.getVolume().obtainReference(); - ReplicaBeingWritten replica; - try { - // change the replica's state/gs etc. - if (replicaInfo.getState() == ReplicaState.FINALIZED) { - replica = append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo, - newGS, b.getNumBytes()); - } else { //RBW - bumpReplicaGS(replicaInfo, newGS); - replica = (ReplicaBeingWritten) replicaInfo; + FsVolumeReference ref = replicaInfo.getVolume().obtainReference(); + ReplicaBeingWritten replica; + try { + // change the replica's state/gs etc. + if (replicaInfo.getState() == ReplicaState.FINALIZED) { + replica = append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo, + newGS, b.getNumBytes()); + } else { //RBW + bumpReplicaGS(replicaInfo, newGS); + replica = (ReplicaBeingWritten) replicaInfo; + } + } catch (IOException e) { + IOUtils.cleanup(null, ref); + throw e; + } + return new ReplicaHandler(replica, ref); + } + } catch (MustStopExistingWriter e) { + e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout()); } - } catch (IOException e) { - IOUtils.cleanup(null, ref); - throw e; } - return new ReplicaHandler(replica, ref); } @Override // FsDatasetSpi public synchronized Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed close " + b); - // check replica's state - ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); - // bump the replica's GS - bumpReplicaGS(replicaInfo, newGS); - // finalize the replica if RBW - if (replicaInfo.getState() == ReplicaState.RBW) { - finalizeReplica(b.getBlockPoolId(), replicaInfo); + while (true) { + try { + synchronized (this) { + // check replica's state + ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); + // bump the replica's GS + bumpReplicaGS(replicaInfo, newGS); + // finalize the replica if RBW + if (replicaInfo.getState() == ReplicaState.RBW) { + finalizeReplica(b.getBlockPoolId(), replicaInfo); + } + return replicaInfo; + } + } catch (MustStopExistingWriter e) { + e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout()); + } } - return replicaInfo; } /** @@ -1384,26 +1412,37 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized ReplicaHandler recoverRbw( + public ReplicaHandler recoverRbw( ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException { LOG.info("Recover RBW replica " + b); - ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId()); - - // check the replica's state - if (replicaInfo.getState() != ReplicaState.RBW) { - throw new ReplicaNotFoundException( - ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo); + while (true) { + try { + synchronized (this) { + ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId()); + + // check the replica's state + if (replicaInfo.getState() != ReplicaState.RBW) { + throw new ReplicaNotFoundException( + ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo); + } + ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; + if (!rbw.attemptToSetWriter(null, Thread.currentThread())) { + throw new MustStopExistingWriter(rbw); + } + LOG.info("Recovering " + rbw); + return recoverRbwImpl(rbw, b, newGS, minBytesRcvd, maxBytesRcvd); + } + } catch (MustStopExistingWriter e) { + e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout()); + } } - ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; - - LOG.info("Recovering " + rbw); - - // Stop the previous writer - rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout()); - rbw.setWriter(Thread.currentThread()); + } + private synchronized ReplicaHandler recoverRbwImpl(ReplicaBeingWritten rbw, + ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd) + throws IOException { // check generation stamp long replicaGenerationStamp = rbw.getGenerationStamp(); if (replicaGenerationStamp < b.getGenerationStamp() || @@ -1419,7 +1458,7 @@ class FsDatasetImpl implements FsDatasetSpi { long numBytes = rbw.getNumBytes(); if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){ throw new ReplicaNotFoundException("Unmatched length replica " + - replicaInfo + ": BytesAcked = " + bytesAcked + + rbw + ": BytesAcked = " + bytesAcked + " BytesRcvd = " + numBytes + " are not in the range of [" + minBytesRcvd + ", " + maxBytesRcvd + "]."); } @@ -2351,8 +2390,8 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized ReplicaRecoveryInfo initReplicaRecovery( - RecoveringBlock rBlock) throws IOException { + public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) + throws IOException { return initReplicaRecovery(rBlock.getBlock().getBlockPoolId(), volumeMap, rBlock.getBlock().getLocalBlock(), rBlock.getNewGenerationStamp(), datanode.getDnConf().getXceiverStopTimeout()); @@ -2361,6 +2400,20 @@ class FsDatasetImpl implements FsDatasetSpi { /** static version of {@link #initReplicaRecovery(RecoveringBlock)}. */ static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map, Block block, long recoveryId, long xceiverStopTimeout) throws IOException { + while (true) { + try { + synchronized (map.getMutex()) { + return initReplicaRecoveryImpl(bpid, map, block, recoveryId); + } + } catch (MustStopExistingWriter e) { + e.getReplica().stopWriter(xceiverStopTimeout); + } + } + } + + static ReplicaRecoveryInfo initReplicaRecoveryImpl(String bpid, ReplicaMap map, + Block block, long recoveryId) + throws IOException, MustStopExistingWriter { final ReplicaInfo replica = map.get(bpid, block.getBlockId()); LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId + ", replica=" + replica); @@ -2373,7 +2426,9 @@ class FsDatasetImpl implements FsDatasetSpi { //stop writer if there is any if (replica instanceof ReplicaInPipeline) { final ReplicaInPipeline rip = (ReplicaInPipeline)replica; - rip.stopWriter(xceiverStopTimeout); + if (!rip.attemptToSetWriter(null, Thread.currentThread())) { + throw new MustStopExistingWriter(rip); + } //check replica bytes on disk. if (rip.getBytesOnDisk() < rip.getVisibleLength()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java index 34c9f2e53b0..192702ed606 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java @@ -233,7 +233,7 @@ class ReplicaMap { * Give access to mutex used for synchronizing ReplicasMap * @return object used as lock */ - Object getMutext() { + Object getMutex() { return mutex; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 7e2bc0a5083..751089f2822 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -42,10 +42,12 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import com.google.common.collect.Iterators; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -92,7 +94,9 @@ import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -124,6 +128,9 @@ public class TestBlockRecovery { private final static ExtendedBlock block = new ExtendedBlock(POOL_ID, BLOCK_ID, BLOCK_LEN, GEN_STAMP); + @Rule + public TestName currentTestName = new TestName(); + private static final int CELL_SIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private static final int bytesPerChecksum = 512; @@ -153,6 +160,9 @@ public class TestBlockRecovery { GenericTestUtils.setLogLevel(LOG, Level.ALL); } + private final long + TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS = 1000000000L; + /** * Starts an instance of DataNode * @throws IOException @@ -165,6 +175,12 @@ public class TestBlockRecovery { conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0"); + if (currentTestName.getMethodName().equals( + "testInitReplicaRecoveryDoesNotHogLock")) { + // This test requires a very long value for the xceiver stop timeout. + conf.setLong(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, + TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS); + } conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); FileSystem.setDefaultUri(conf, "hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort()); @@ -297,7 +313,7 @@ public class TestBlockRecovery { * Two replicas are in Finalized state * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testFinalizedReplicas () throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -336,7 +352,7 @@ public class TestBlockRecovery { * One replica is Finalized and another is RBW. * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testFinalizedRbwReplicas() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -378,7 +394,7 @@ public class TestBlockRecovery { * One replica is Finalized and another is RWR. * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testFinalizedRwrReplicas() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -420,7 +436,7 @@ public class TestBlockRecovery { * Two replicas are RBW. * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testRBWReplicas() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -444,7 +460,7 @@ public class TestBlockRecovery { * One replica is RBW and another is RWR. * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testRBW_RWRReplicas() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -468,7 +484,7 @@ public class TestBlockRecovery { * Two replicas are RWR. * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testRWRReplicas() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -504,7 +520,7 @@ public class TestBlockRecovery { * @throws IOException * in case of an error */ - @Test + @Test(timeout=60000) public void testRecoveryInProgressException() throws IOException, InterruptedException { if(LOG.isDebugEnabled()) { @@ -529,7 +545,7 @@ public class TestBlockRecovery { * @throws IOException * in case of an error */ - @Test + @Test(timeout=60000) public void testErrorReplicas() throws IOException, InterruptedException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -556,7 +572,7 @@ public class TestBlockRecovery { * * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testZeroLenReplicas() throws IOException, InterruptedException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -596,7 +612,7 @@ public class TestBlockRecovery { * * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testFailedReplicaUpdate() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -618,7 +634,7 @@ public class TestBlockRecovery { * * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testNoReplicaUnderRecovery() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -643,7 +659,7 @@ public class TestBlockRecovery { * * @throws IOException in case of an error */ - @Test + @Test(timeout=60000) public void testNotMatchedReplicaID() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -744,7 +760,7 @@ public class TestBlockRecovery { * throw an exception. * @throws Exception */ - @Test + @Test(timeout=60000) public void testRURReplicas() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); @@ -775,7 +791,7 @@ public class TestBlockRecovery { } } - @Test + @Test(timeout=60000) public void testSafeLength() throws Exception { // hard coded policy to work with hard coded test suite ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager @@ -799,4 +815,97 @@ public class TestBlockRecovery { recoveryTask.getSafeLength(syncList)); } } + + /** + * Test that initReplicaRecovery does not hold the lock for an unreasonable + * amount of time if a writer is taking a long time to stop. + */ + @Test(timeout=60000) + public void testInitReplicaRecoveryDoesNotHogLock() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("Running " + GenericTestUtils.getMethodName()); + } + // We need a long value for the data xceiver stop timeout. + // Otherwise the timeout will trigger, and we will not have tested that + // thread join was done locklessly. + Assert.assertEquals( + TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS, + dn.getDnConf().getXceiverStopTimeout()); + final Semaphore progressParent = new Semaphore(0); + final Semaphore terminateSlowWorker = new Semaphore(0); + final AtomicBoolean failure = new AtomicBoolean(false); + Collection recoveringBlocks = + initRecoveringBlocks(); + final RecoveringBlock recoveringBlock = + Iterators.get(recoveringBlocks.iterator(), 0); + final ExtendedBlock block = recoveringBlock.getBlock(); + Thread slowWorker = new Thread(new Runnable() { + @Override + public void run() { + try { + // Register this thread as the writer for the recoveringBlock. + LOG.debug("slowWorker creating rbw"); + ReplicaHandler replicaHandler = + spyDN.data.createRbw(StorageType.DISK, block, false); + replicaHandler.close(); + LOG.debug("slowWorker created rbw"); + // Tell the parent thread to start progressing. + progressParent.release(); + while (true) { + try { + terminateSlowWorker.acquire(); + break; + } catch (InterruptedException e) { + // Ignore interrupted exceptions so that the waitingWorker thread + // will have to wait for us. + } + } + LOG.debug("slowWorker exiting"); + } catch (Throwable t) { + LOG.error("slowWorker got exception", t); + failure.set(true); + } + } + }); + // Start the slow worker thread and wait for it to take ownership of the + // ReplicaInPipeline + slowWorker.start(); + while (true) { + try { + progressParent.acquire(); + break; + } catch (InterruptedException e) { + // Ignore interrupted exceptions + } + } + + // Start a worker thread which will wait for the slow worker thread. + Thread waitingWorker = new Thread(new Runnable() { + @Override + public void run() { + try { + // Attempt to terminate the other worker thread and take ownership + // of the ReplicaInPipeline. + LOG.debug("waitingWorker initiating recovery"); + spyDN.initReplicaRecovery(recoveringBlock); + LOG.debug("waitingWorker initiated recovery"); + } catch (Throwable t) { + GenericTestUtils.assertExceptionContains("meta does not exist", t); + } + } + }); + waitingWorker.start(); + + // Do an operation that requires the lock. This should not be blocked + // by the replica recovery in progress. + spyDN.getFSDataset().getReplicaString( + recoveringBlock.getBlock().getBlockPoolId(), + recoveringBlock.getBlock().getBlockId()); + + // Wait for the two worker threads to exit normally. + terminateSlowWorker.release(); + slowWorker.join(); + waitingWorker.join(); + Assert.assertFalse("The slowWriter thread failed.", failure.get()); + } } From 818d6b799eead13a17a0214172df60a269b046fb Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Tue, 5 Apr 2016 09:49:39 +0800 Subject: [PATCH 51/75] HDFS-9917. IBR accumulate more objects when SNN was down for sometime. (Contributed by Brahma Reddy Battula) --- .../hdfs/server/datanode/BPServiceActor.java | 5 + .../IncrementalBlockReportManager.java | 9 ++ .../server/datanode/TestBPOfferService.java | 96 ++++++++++++++++++- 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 49f64c2e5d9..39f821972a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -798,6 +798,11 @@ class BPServiceActor implements Runnable { // and re-register register(nsInfo); scheduler.scheduleHeartbeat(); + // HDFS-9917,Standby NN IBR can be very huge if standby namenode is down + // for sometime. + if (state == HAServiceState.STANDBY) { + ibrManager.clearIBRs(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java index b9b348a7de2..e95142db872 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java @@ -258,4 +258,13 @@ class IncrementalBlockReportManager { } } } + + void clearIBRs() { + pendingIBRs.clear(); + } + + @VisibleForTesting + int getPendingIBRSize() { + return pendingIBRs.size(); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 95a103e2e83..29db702cf2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -48,10 +49,12 @@ import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; @@ -92,7 +95,10 @@ public class TestBPOfferService { private DatanodeProtocolClientSideTranslatorPB mockNN1; private DatanodeProtocolClientSideTranslatorPB mockNN2; - private final NNHAStatusHeartbeat[] mockHaStatuses = new NNHAStatusHeartbeat[2]; + private final NNHAStatusHeartbeat[] mockHaStatuses = + new NNHAStatusHeartbeat[2]; + private final DatanodeCommand[][] datanodeCommands = + new DatanodeCommand[2][0]; private final int[] heartbeatCounts = new int[2]; private DataNode mockDn; private FsDatasetSpi mockFSDataset; @@ -147,6 +153,7 @@ public class TestBPOfferService { Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean()); mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0); + datanodeCommands[nnIdx] = new DatanodeCommand[0]; return mock; } @@ -165,9 +172,12 @@ public class TestBPOfferService { @Override public HeartbeatResponse answer(InvocationOnMock invocation) throws Throwable { heartbeatCounts[nnIdx]++; - return new HeartbeatResponse(new DatanodeCommand[0], - mockHaStatuses[nnIdx], null, + HeartbeatResponse heartbeatResponse = new HeartbeatResponse( + datanodeCommands[nnIdx], mockHaStatuses[nnIdx], null, ThreadLocalRandom.current().nextLong() | 1L); + //reset the command + datanodeCommands[nnIdx] = new DatanodeCommand[0]; + return heartbeatResponse; } } @@ -709,4 +719,84 @@ public class TestBPOfferService { bpos.join(); } } + + /* + * HDFS-9917 : Standby IBR accumulation when Standby was down. + */ + @Test + public void testIBRClearanceForStandbyOnReRegister() throws Exception { + final BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2); + bpos.start(); + try { + waitForInitialization(bpos); + // Should start with neither NN as active. + assertNull(bpos.getActiveNN()); + // Have NN1 claim active at txid 1 + mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1); + bpos.triggerHeartbeatForTests(); + // Now mockNN1 is acting like active namenode and mockNN2 as Standby + assertSame(mockNN1, bpos.getActiveNN()); + // Return nothing when active Active Namenode gets IBRs + Mockito.doNothing().when(mockNN1).blockReceivedAndDeleted( + Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito + .any(StorageReceivedDeletedBlocks[].class)); + + final IOException re = new IOException( + "Standby NN is currently not able to process IBR"); + + final AtomicBoolean ibrReported = new AtomicBoolean(false); + // throw exception for standby when first IBR is receieved + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + ibrReported.set(true); + throw re; + } + }).when(mockNN2).blockReceivedAndDeleted( + Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito + .any(StorageReceivedDeletedBlocks[].class)); + + DatanodeStorage storage = Mockito.mock(DatanodeStorage.class); + Mockito.doReturn(storage).when(mockFSDataset).getStorage("storage0"); + // Add IBRs + bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, null, "storage0", false); + // Send heartbeat so that the BpServiceActor can send IBR to + // namenode + bpos.triggerHeartbeatForTests(); + // Wait till first IBR is received at standbyNN. Just for confirmation. + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return ibrReported.get(); + } + }, 100, 1000); + + // Send register command back to Datanode to reRegister(). + // After reRegister IBRs should be cleared. + datanodeCommands[1] = new DatanodeCommand[] { new RegisterCommand() }; + assertEquals( + "IBR size before reRegister should be non-0", 1, getStandbyIBRSize( + bpos)); + bpos.triggerHeartbeatForTests(); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return getStandbyIBRSize(bpos) == 0; + } + }, 100, 1000); + } finally { + bpos.stop(); + bpos.join(); + } + } + + private int getStandbyIBRSize(BPOfferService bpos) { + List bpServiceActors = bpos.getBPServiceActors(); + for (BPServiceActor bpServiceActor : bpServiceActors) { + if (bpServiceActor.state == HAServiceState.STANDBY) { + return bpServiceActor.getIbrManager().getPendingIBRSize(); + } + } + return -1; + } } From 552237d4a34ab10fa5f9ec7aad7942f2a110993e Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 5 Apr 2016 14:25:32 +0530 Subject: [PATCH 52/75] YARN-4880. Running TestZKRMStateStorePerf with real zookeeper cluster throws NPE. Contributed by Sunil G --- .../resourcemanager/recovery/TestZKRMStateStorePerf.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java index 4b0b06a633e..bd25defc223 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java @@ -91,7 +91,9 @@ public class TestZKRMStateStorePerf extends RMStateStoreTestBase if (appTokenMgr != null) { appTokenMgr.stop(); } - curatorTestingServer.stop(); + if (curatorTestingServer != null) { + curatorTestingServer.stop(); + } } private void initStore(String hostPort) { @@ -99,8 +101,9 @@ public class TestZKRMStateStorePerf extends RMStateStoreTestBase RMContext rmContext = mock(RMContext.class); conf = new YarnConfiguration(); - conf.set(YarnConfiguration.RM_ZK_ADDRESS, - optHostPort.or(curatorTestingServer.getConnectString())); + conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort + .or((curatorTestingServer == null) ? "" : curatorTestingServer + .getConnectString())); conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode); store = new ZKRMStateStore(); From 776b549e2ac20a68a5513cbcaac0edc33233dc03 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 5 Apr 2016 14:47:25 +0530 Subject: [PATCH 53/75] YARN-4609. RM Nodes list page takes too much time to load. Contributed by Bibin A Chundatt --- .../resourcemanager/webapp/NodesPage.java | 53 ++++++++++++------- .../resourcemanager/webapp/TestNodesPage.java | 37 ++++--------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 960346827b5..706342196b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; @@ -101,6 +100,7 @@ class NodesPage extends RmView { LOG.debug("Unexpected state filter for inactive RM node"); } } + StringBuilder nodeTableData = new StringBuilder("[\n"); for (RMNode ni : rmNodes) { if (stateFilter != null) { NodeState state = ni.getState(); @@ -129,27 +129,40 @@ class NodesPage extends RmView { NodeInfo info = new NodeInfo(ni, sched); int usedMemory = (int) info.getUsedMemory(); int availableMemory = (int) info.getAvailableMemory(); - TR>> row = - tbody.tr().td(StringUtils.join(",", info.getNodeLabels())) - .td(info.getRack()).td(info.getState()).td(info.getNodeId()); + nodeTableData.append("[\"") + .append(StringUtils.join(",", info.getNodeLabels())).append("\",\"") + .append(info.getRack()).append("\",\"").append(info.getState()) + .append("\",\"").append(info.getNodeId()); if (isInactive) { - row.td()._("N/A")._(); + nodeTableData.append("\",\"").append("N/A").append("\",\""); } else { String httpAddress = info.getNodeHTTPAddress(); - row.td().a("//" + httpAddress, httpAddress)._(); + nodeTableData.append("\",\"").append(httpAddress).append("\",").append("\""); } - row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._() - ._(Times.format(info.getLastHealthUpdate()))._() - .td(info.getHealthReport()) - .td(String.valueOf(info.getNumContainers())).td().br() - .$title(String.valueOf(usedMemory))._() - ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br() - .$title(String.valueOf(availableMemory))._() - ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._() - .td(String.valueOf(info.getUsedVirtualCores())) - .td(String.valueOf(info.getAvailableVirtualCores())) - .td(ni.getNodeManagerVersion())._(); + nodeTableData.append("
      ") + .append(Times.format(info.getLastHealthUpdate())).append("\",\"") + .append(info.getHealthReport()).append("\",\"") + .append(String.valueOf(info.getNumContainers())).append("\",\"") + .append("
      ").append(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)) + .append("\",\"").append("
      ") + .append(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)) + .append("\",\"").append(String.valueOf(info.getUsedVirtualCores())) + .append("\",\"") + .append(String.valueOf(info.getAvailableVirtualCores())) + .append("\",\"").append(ni.getNodeManagerVersion()) + .append("\"],\n"); } + if (nodeTableData.charAt(nodeTableData.length() - 2) == ',') { + nodeTableData.delete(nodeTableData.length() - 2, + nodeTableData.length() - 1); + } + nodeTableData.append("]"); + html.script().$type("text/javascript") + ._("var nodeTableData=" + nodeTableData)._(); tbody._()._(); } } @@ -175,7 +188,9 @@ class NodesPage extends RmView { } private String nodesTableInit() { - StringBuilder b = tableInit().append(", aoColumnDefs: ["); + StringBuilder b = tableInit().append(", 'aaData': nodeTableData") + .append(", bDeferRender: true").append(", bProcessing: true") + .append(", aoColumnDefs: ["); b.append("{'bSearchable': false, 'aTargets': [ 7 ]}"); b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + "'aTargets': [ 8, 9 ] }"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 0699c0e0e89..71d077bc30b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -81,11 +81,8 @@ public class TestNodesPage { Mockito.verify(writer, Mockito.times(numberOfActualTableHeaders + numberOfThInMetricsTable)) .print(" Date: Tue, 5 Apr 2016 13:40:19 +0000 Subject: [PATCH 54/75] YARN-4311. Removing nodes from include and exclude lists will not remove them from decommissioned nodes list. Contributed by Kuhu Shukla --- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 + .../yarn/sls/scheduler/RMNodeWrapper.java | 9 + .../hadoop/yarn/conf/YarnConfiguration.java | 9 + .../src/main/resources/yarn-default.xml | 13 ++ .../resourcemanager/NodesListManager.java | 104 ++++++++- .../server/resourcemanager/RMServerUtils.java | 2 +- .../ResourceTrackerService.java | 8 +- .../server/resourcemanager/rmnode/RMNode.java | 4 + .../resourcemanager/rmnode/RMNodeImpl.java | 22 +- .../server/resourcemanager/MockNodes.java | 9 + .../TestResourceTrackerService.java | 216 ++++++++++++++++-- .../webapp/TestRMWebServicesNodes.java | 12 +- 12 files changed, 387 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 92d586bfa37..951f5a850df 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -199,6 +199,15 @@ public class NodeInfo { public ResourceUtilization getNodeUtilization() { return null; } + + @Override + public long getUntrackedTimeStamp() { + return 0; + } + + @Override + public void setUntrackedTimeStamp(long timeStamp) { + } } public static RMNode newNodeInfo(String rackName, String hostName, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index 2e9cccb2778..e5013c43d75 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -188,4 +188,13 @@ public class RMNodeWrapper implements RMNode { public ResourceUtilization getNodeUtilization() { return node.getNodeUtilization(); } + + @Override + public long getUntrackedTimeStamp() { + return 0; + } + + @Override + public void setUntrackedTimeStamp(long timeStamp) { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8acee579ff3..66b293f9594 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -647,6 +647,15 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION = "NONE"; + /** + * Timeout(msec) for an untracked node to remain in shutdown or decommissioned + * state. + */ + public static final String RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC = + RM_PREFIX + "node-removal-untracked.timeout-ms"; + public static final int + DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC = 60000; + /** * RM proxy users' prefix */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 506cf3d9fc7..9e8b5e9b208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2722,4 +2722,17 @@ yarn.timeline-service.webapp.rest-csrf.methods-to-ignore GET,OPTIONS,HEAD       + + + + The least amount of time(msec.) an inactive (decommissioned or shutdown) node can + stay in the nodes list of the resourcemanager after being declared untracked. + A node is marked untracked if and only if it is absent from both include and + exclude nodemanager lists on the RM. All inactive nodes are checked twice per + timeout interval or every 10 minutes, whichever is lesser, and marked appropriately. + The same is done when refreshNodes command (graceful or otherwise) is invoked. + + yarn.resourcemanager.node-removal-untracked.timeout-ms + 60000 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index ec2708ebb3c..65a9d9498f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.HostsFileReader; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -68,6 +69,8 @@ public class NodesListManager extends CompositeService implements private String excludesFile; private Resolver resolver; + private Timer removalTimer; + private int nodeRemovalCheckInterval; public NodesListManager(RMContext rmContext) { super(NodesListManager.class.getName()); @@ -105,9 +108,56 @@ public class NodesListManager extends CompositeService implements } catch (IOException ioe) { disableHostsFileReader(ioe); } + + final int nodeRemovalTimeout = + conf.getInt( + YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC, + YarnConfiguration. + DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC); + nodeRemovalCheckInterval = (Math.min(nodeRemovalTimeout/2, + 600000)); + removalTimer = new Timer("Node Removal Timer"); + + removalTimer.schedule(new TimerTask() { + @Override + public void run() { + long now = Time.monotonicNow(); + for (Map.Entry entry : + rmContext.getInactiveRMNodes().entrySet()) { + NodeId nodeId = entry.getKey(); + RMNode rmNode = entry.getValue(); + if (isUntrackedNode(rmNode.getHostName())) { + if (rmNode.getUntrackedTimeStamp() == 0) { + rmNode.setUntrackedTimeStamp(now); + } else if (now - rmNode.getUntrackedTimeStamp() > + nodeRemovalTimeout) { + RMNode result = rmContext.getInactiveRMNodes().remove(nodeId); + if (result != null) { + ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); + if (rmNode.getState() == NodeState.SHUTDOWN) { + clusterMetrics.decrNumShutdownNMs(); + } else { + clusterMetrics.decrDecommisionedNMs(); + } + LOG.info("Removed "+result.getHostName() + + " from inactive nodes list"); + } + } + } else { + rmNode.setUntrackedTimeStamp(0); + } + } + } + }, nodeRemovalCheckInterval, nodeRemovalCheckInterval); + super.serviceInit(conf); } + @Override + public void serviceStop() { + removalTimer.cancel(); + } + private void printConfiguredHosts() { if (!LOG.isDebugEnabled()) { return; @@ -131,10 +181,13 @@ public class NodesListManager extends CompositeService implements for (NodeId nodeId: rmContext.getRMNodes().keySet()) { if (!isValidNode(nodeId.getHost())) { + RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ? + RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION; this.rmContext.getDispatcher().getEventHandler().handle( - new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); + new RMNodeEvent(nodeId, nodeEventType)); } } + updateInactiveNodes(); } private void refreshHostsReader(Configuration yarnConf) throws IOException, @@ -171,6 +224,16 @@ public class NodesListManager extends CompositeService implements } } + @VisibleForTesting + public int getNodeRemovalCheckInterval() { + return nodeRemovalCheckInterval; + } + + @VisibleForTesting + public void setNodeRemovalCheckInterval(int interval) { + this.nodeRemovalCheckInterval = interval; + } + @VisibleForTesting public Resolver getResolver() { return resolver; @@ -374,6 +437,33 @@ public class NodesListManager extends CompositeService implements return hostsReader; } + private void updateInactiveNodes() { + long now = Time.monotonicNow(); + for(Entry entry : + rmContext.getInactiveRMNodes().entrySet()) { + NodeId nodeId = entry.getKey(); + RMNode rmNode = entry.getValue(); + if (isUntrackedNode(nodeId.getHost()) && + rmNode.getUntrackedTimeStamp() == 0) { + rmNode.setUntrackedTimeStamp(now); + } + } + } + + public boolean isUntrackedNode(String hostName) { + boolean untracked; + String ip = resolver.resolve(hostName); + + synchronized (hostsReader) { + Set hostsList = hostsReader.getHosts(); + Set excludeList = hostsReader.getExcludedHosts(); + untracked = !hostsList.isEmpty() && + !hostsList.contains(hostName) && !hostsList.contains(ip) && + !excludeList.contains(hostName) && !excludeList.contains(ip); + } + return untracked; + } + /** * Refresh the nodes gracefully * @@ -384,11 +474,13 @@ public class NodesListManager extends CompositeService implements public void refreshNodesGracefully(Configuration conf) throws IOException, YarnException { refreshHostsReader(conf); - for (Entry entry:rmContext.getRMNodes().entrySet()) { + for (Entry entry : rmContext.getRMNodes().entrySet()) { NodeId nodeId = entry.getKey(); if (!isValidNode(nodeId.getHost())) { + RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ? + RMNodeEventType.SHUTDOWN : RMNodeEventType.GRACEFUL_DECOMMISSION; this.rmContext.getDispatcher().getEventHandler().handle( - new RMNodeEvent(nodeId, RMNodeEventType.GRACEFUL_DECOMMISSION)); + new RMNodeEvent(nodeId, nodeEventType)); } else { // Recommissioning the nodes if (entry.getValue().getState() == NodeState.DECOMMISSIONING) { @@ -397,6 +489,7 @@ public class NodesListManager extends CompositeService implements } } } + updateInactiveNodes(); } /** @@ -420,8 +513,11 @@ public class NodesListManager extends CompositeService implements public void refreshNodesForcefully() { for (Entry entry : rmContext.getRMNodes().entrySet()) { if (entry.getValue().getState() == NodeState.DECOMMISSIONING) { + RMNodeEventType nodeEventType = + isUntrackedNode(entry.getKey().getHost()) ? + RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION; this.rmContext.getDispatcher().getEventHandler().handle( - new RMNodeEvent(entry.getKey(), RMNodeEventType.DECOMMISSION)); + new RMNodeEvent(entry.getKey(), nodeEventType)); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index e19d55ee81d..1318d5814be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -87,7 +87,7 @@ public class RMServerUtils { acceptedStates.contains(NodeState.LOST) || acceptedStates.contains(NodeState.REBOOTED)) { for (RMNode rmNode : context.getInactiveRMNodes().values()) { - if (acceptedStates.contains(rmNode.getState())) { + if ((rmNode != null) && acceptedStates.contains(rmNode.getState())) { results.add(rmNode); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index b0bc565e6c3..238e5bcf1ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -320,7 +320,8 @@ public class ResourceTrackerService extends AbstractService implements } // Check if this node is a 'valid' node - if (!this.nodesListManager.isValidNode(host)) { + if (!this.nodesListManager.isValidNode(host) || + this.nodesListManager.isUntrackedNode(host)) { String message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager."; @@ -451,8 +452,9 @@ public class ResourceTrackerService extends AbstractService implements // 1. Check if it's a valid (i.e. not excluded) node, if not, see if it is // in decommissioning. - if (!this.nodesListManager.isValidNode(nodeId.getHost()) - && !isNodeInDecommissioning(nodeId)) { + if ((!this.nodesListManager.isValidNode(nodeId.getHost()) && + !isNodeInDecommissioning(nodeId)) || + this.nodesListManager.isUntrackedNode(nodeId.getHost())) { String message = "Disallowed NodeManager nodeId: " + nodeId + " hostname: " + nodeId.getHost(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index d8df9f16ef8..e599576592a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -168,4 +168,8 @@ public interface RMNode { NodeHeartbeatResponse response); public List pullNewlyIncreasedContainers(); + + long getUntrackedTimeStamp(); + + void setUntrackedTimeStamp(long timer); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 5f8317e890a..42608613ba3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -39,6 +39,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.net.Node; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -120,6 +121,7 @@ public class RMNodeImpl implements RMNode, EventHandler { private long lastHealthReportTime; private String nodeManagerVersion; + private long timeStamp; /* Aggregated resource utilization for the containers. */ private ResourceUtilization containersUtilization; /* Resource utilization for the node. */ @@ -259,6 +261,9 @@ public class RMNodeImpl implements RMNode, EventHandler { .addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING, RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition()) + .addTransition(NodeState.DECOMMISSIONING, NodeState.SHUTDOWN, + RMNodeEventType.SHUTDOWN, + new DeactivateNodeTransition(NodeState.SHUTDOWN)) // TODO (in YARN-3223) update resource when container finished. .addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING, @@ -346,6 +351,7 @@ public class RMNodeImpl implements RMNode, EventHandler { this.healthReport = "Healthy"; this.lastHealthReportTime = System.currentTimeMillis(); this.nodeManagerVersion = nodeManagerVersion; + this.timeStamp = 0; this.latestNodeHeartBeatResponse.setResponseId(0); @@ -1011,7 +1017,7 @@ public class RMNodeImpl implements RMNode, EventHandler { } /** - * Put a node in deactivated (decommissioned) status. + * Put a node in deactivated (decommissioned or shutdown) status. * @param rmNode * @param finalState */ @@ -1028,6 +1034,10 @@ public class RMNodeImpl implements RMNode, EventHandler { LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + finalState); rmNode.context.getInactiveRMNodes().put(rmNode.nodeId, rmNode); + if (finalState == NodeState.SHUTDOWN && + rmNode.context.getNodesListManager().isUntrackedNode(rmNode.hostName)) { + rmNode.setUntrackedTimeStamp(Time.monotonicNow()); + } } /** @@ -1383,4 +1393,14 @@ public class RMNodeImpl implements RMNode, EventHandler { public Resource getOriginalTotalCapability() { return this.originalTotalCapability; } + + @Override + public long getUntrackedTimeStamp() { + return this.timeStamp; + } + + @Override + public void setUntrackedTimeStamp(long ts) { + this.timeStamp = ts; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 89aff29b834..921b18eee93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -260,6 +260,15 @@ public class MockNodes { public ResourceUtilization getNodeUtilization() { return this.nodeUtilization; } + + @Override + public long getUntrackedTimeStamp() { + return 0; + } + + @Override + public void setUntrackedTimeStamp(long timeStamp) { + } }; private static RMNode buildRMNode(int rack, final Resource perNode, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 9ed79a31c23..dd37b67fdfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -31,6 +31,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; @@ -48,8 +50,6 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; @@ -141,12 +141,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.getNodesListManager().refreshNodes(conf); - checkDecommissionedNMCount(rm, ++metricCount); + checkShutdownNMCount(rm, ++metricCount); nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); Assert - .assertEquals(1, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); + .assertEquals(1, ClusterMetrics.getMetrics().getNumShutdownNMs()); nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue("Node is not decommisioned.", NodeAction.SHUTDOWN @@ -155,7 +155,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase { nodeHeartbeat = nm3.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); Assert.assertEquals(metricCount, ClusterMetrics.getMetrics() - .getNumDecommisionedNMs()); + .getNumShutdownNMs()); + rm.stop(); } /** @@ -228,7 +229,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { MockNM nm2 = rm.registerNode("host2:5678", 10240); ClusterMetrics metrics = ClusterMetrics.getMetrics(); assert(metrics != null); - int initialMetricCount = metrics.getNumDecommisionedNMs(); + int initialMetricCount = metrics.getNumShutdownNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals( NodeAction.NORMAL, @@ -241,16 +242,16 @@ public class TestResourceTrackerService extends NodeLabelTestBase { conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile .getAbsolutePath()); rm.getNodesListManager().refreshNodes(conf); - checkDecommissionedNMCount(rm, ++initialMetricCount); + checkShutdownNMCount(rm, ++initialMetricCount); nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals( - "Node should not have been decomissioned.", + "Node should not have been shutdown.", NodeAction.NORMAL, nodeHeartbeat.getNodeAction()); - nodeHeartbeat = nm2.nodeHeartbeat(true); - Assert.assertEquals("Node should have been decomissioned but is in state" + - nodeHeartbeat.getNodeAction(), - NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction()); + NodeState nodeState = + rm.getRMContext().getInactiveRMNodes().get(nm2.getNodeId()).getState(); + Assert.assertEquals("Node should have been shutdown but is in state" + + nodeState, NodeState.SHUTDOWN, nodeState); } /** @@ -1123,8 +1124,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); ResourceTrackerService resourceTrackerService = rm .getResourceTrackerService(); - int shutdownNMsCount = ClusterMetrics.getMetrics() - .getNumShutdownNMs(); int decommisionedNMsCount = ClusterMetrics.getMetrics() .getNumDecommisionedNMs(); @@ -1149,10 +1148,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.getNodesListManager().refreshNodes(conf); NodeHeartbeatResponse heartbeatResponse = nm1.nodeHeartbeat(true); Assert.assertEquals(NodeAction.SHUTDOWN, heartbeatResponse.getNodeAction()); + int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs(); checkShutdownNMCount(rm, shutdownNMsCount); - checkDecommissionedNMCount(rm, ++decommisionedNMsCount); + checkDecommissionedNMCount(rm, decommisionedNMsCount); request.setNodeId(nm1.getNodeId()); resourceTrackerService.unRegisterNodeManager(request); + shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs(); checkShutdownNMCount(rm, shutdownNMsCount); checkDecommissionedNMCount(rm, decommisionedNMsCount); @@ -1168,8 +1169,9 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.getNodesListManager().refreshNodes(conf); request.setNodeId(nm2.getNodeId()); resourceTrackerService.unRegisterNodeManager(request); - checkShutdownNMCount(rm, shutdownNMsCount); - checkDecommissionedNMCount(rm, ++decommisionedNMsCount); + checkShutdownNMCount(rm, ++shutdownNMsCount); + checkDecommissionedNMCount(rm, decommisionedNMsCount); + rm.stop(); } @Test(timeout = 30000) @@ -1304,6 +1306,186 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.stop(); } + /** + * Remove a node from all lists and check if its forgotten + */ + @Test + public void testNodeRemovalNormally() throws Exception { + testNodeRemovalUtil(false); + } + + @Test + public void testNodeRemovalGracefully() throws Exception { + testNodeRemovalUtil(true); + } + + public void refreshNodesOption(boolean doGraceful, Configuration conf) + throws Exception { + if (doGraceful) { + rm.getNodesListManager().refreshNodesGracefully(conf); + } else { + rm.getNodesListManager().refreshNodes(conf); + } + } + + public void testNodeRemovalUtil(boolean doGraceful) throws Exception { + Configuration conf = new Configuration(); + int timeoutValue = 500; + File excludeHostFile = new File(TEMP_DIR + File.separator + + "excludeHostFile.txt"); + conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, ""); + conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, ""); + conf.setInt(YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC, + timeoutValue); + CountDownLatch latch = new CountDownLatch(1); + rm = new MockRM(conf); + rm.init(conf); + rm.start(); + RMContext rmContext = rm.getRMContext(); + refreshNodesOption(doGraceful, conf); + MockNM nm1 = rm.registerNode("host1:1234", 5120); + MockNM nm2 = rm.registerNode("host2:5678", 10240); + MockNM nm3 = rm.registerNode("localhost:4433", 1024); + ClusterMetrics metrics = ClusterMetrics.getMetrics(); + assert (metrics != null); + + //check all 3 nodes joined in as NORMAL + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + nodeHeartbeat = nm2.nodeHeartbeat(true); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + nodeHeartbeat = nm3.nodeHeartbeat(true); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + rm.drainEvents(); + Assert.assertEquals("All 3 nodes should be active", + metrics.getNumActiveNMs(), 3); + + //Remove nm2 from include list, should now be shutdown with timer test + String ip = NetUtils.normalizeHostName("localhost"); + writeToHostsFile("host1", ip); + conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile + .getAbsolutePath()); + refreshNodesOption(doGraceful, conf); + nm1.nodeHeartbeat(true); + rm.drainEvents(); + Assert.assertTrue("Node should not be in active node list", + !rmContext.getRMNodes().containsKey(nm2.getNodeId())); + + RMNode rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertEquals("Node should be in inactive node list", + rmNode.getState(), NodeState.SHUTDOWN); + Assert.assertEquals("Active nodes should be 2", + metrics.getNumActiveNMs(), 2); + Assert.assertEquals("Shutdown nodes should be 1", + metrics.getNumShutdownNMs(), 1); + + int nodeRemovalTimeout = + conf.getInt( + YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC, + YarnConfiguration. + DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC); + int nodeRemovalInterval = + rmContext.getNodesListManager().getNodeRemovalCheckInterval(); + long maxThreadSleeptime = nodeRemovalInterval + nodeRemovalTimeout; + latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS); + + rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertEquals("Node should have been forgotten!", + rmNode, null); + Assert.assertEquals("Shutdown nodes should be 0 now", + metrics.getNumShutdownNMs(), 0); + + //Check node removal and re-addition before timer expires + writeToHostsFile("host1", ip, "host2"); + refreshNodesOption(doGraceful, conf); + nm2 = rm.registerNode("host2:5678", 10240); + rm.drainEvents(); + writeToHostsFile("host1", ip); + refreshNodesOption(doGraceful, conf); + rm.drainEvents(); + rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertEquals("Node should be shutdown", + rmNode.getState(), NodeState.SHUTDOWN); + Assert.assertEquals("Active nodes should be 2", + metrics.getNumActiveNMs(), 2); + Assert.assertEquals("Shutdown nodes should be 1", + metrics.getNumShutdownNMs(), 1); + + //add back the node before timer expires + latch.await(maxThreadSleeptime - 2000, TimeUnit.MILLISECONDS); + writeToHostsFile("host1", ip, "host2"); + refreshNodesOption(doGraceful, conf); + nm2 = rm.registerNode("host2:5678", 10240); + nodeHeartbeat = nm2.nodeHeartbeat(true); + rm.drainEvents(); + Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); + Assert.assertEquals("Shutdown nodes should be 0 now", + metrics.getNumShutdownNMs(), 0); + Assert.assertEquals("All 3 nodes should be active", + metrics.getNumActiveNMs(), 3); + + //Decommission this node, check timer doesn't remove it + writeToHostsFile("host1", "host2", ip); + writeToHostsFile(excludeHostFile, "host2"); + conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeHostFile + .getAbsolutePath()); + refreshNodesOption(doGraceful, conf); + rm.drainEvents(); + rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) : + rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING", + (rmNode.getState() == NodeState.DECOMMISSIONED) || + (rmNode.getState() == NodeState.DECOMMISSIONING)); + if (rmNode.getState() == NodeState.DECOMMISSIONED) { + Assert.assertEquals("Decommissioned/ing nodes should be 1 now", + metrics.getNumDecommisionedNMs(), 1); + } + latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS); + + rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) : + rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING", + (rmNode.getState() == NodeState.DECOMMISSIONED) || + (rmNode.getState() == NodeState.DECOMMISSIONING)); + if (rmNode.getState() == NodeState.DECOMMISSIONED) { + Assert.assertEquals("Decommissioned/ing nodes should be 1 now", + metrics.getNumDecommisionedNMs(), 1); + } + + //Test decommed/ing node that transitions to untracked,timer should remove + writeToHostsFile("host1", ip, "host2"); + writeToHostsFile(excludeHostFile, "host2"); + refreshNodesOption(doGraceful, conf); + nm1.nodeHeartbeat(true); + //nm2.nodeHeartbeat(true); + nm3.nodeHeartbeat(true); + latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS); + rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) : + rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertNotEquals("Timer for this node was not canceled!", + rmNode, null); + Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING", + (rmNode.getState() == NodeState.DECOMMISSIONED) || + (rmNode.getState() == NodeState.DECOMMISSIONING)); + + writeToHostsFile("host1", ip); + writeToHostsFile(excludeHostFile, ""); + refreshNodesOption(doGraceful, conf); + latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS); + rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) : + rmContext.getInactiveRMNodes().get(nm2.getNodeId()); + Assert.assertEquals("Node should have been forgotten!", + rmNode, null); + Assert.assertEquals("Shutdown nodes should be 0 now", + metrics.getNumDecommisionedNMs(), 0); + Assert.assertEquals("Shutdown nodes should be 0 now", + metrics.getNumShutdownNMs(), 0); + Assert.assertEquals("Active nodes should be 2", + metrics.getNumActiveNMs(), 2); + + rm.stop(); + } + private void writeToHostsFile(String... hosts) throws IOException { writeToHostsFile(hostFile, hosts); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 3fd1fd5f6f9..4b6ca1244e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -272,8 +272,10 @@ public class TestRMWebServicesNodes extends JerseyTestBase { RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nodeId); WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", info.getString("nodeHTTPAddress")); - WebServicesTestUtils.checkStringMatch("state", rmNode.getState() - .toString(), info.getString("state")); + if (rmNode != null) { + WebServicesTestUtils.checkStringMatch("state", + rmNode.getState().toString(), info.getString("state")); + } } } @@ -304,8 +306,10 @@ public class TestRMWebServicesNodes extends JerseyTestBase { RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nodeId); WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "", info.getString("nodeHTTPAddress")); - WebServicesTestUtils.checkStringMatch("state", - rmNode.getState().toString(), info.getString("state")); + if (rmNode != null) { + WebServicesTestUtils.checkStringMatch("state", + rmNode.getState().toString(), info.getString("state")); + } } @Test From 6be28bcc461292b24589dae17a235b3eaadc07ed Mon Sep 17 00:00:00 2001 From: Junping Du Date: Tue, 5 Apr 2016 06:57:26 -0700 Subject: [PATCH 55/75] YARN-4893. Fix some intermittent test failures in TestRMAdminService. Contributed by Brahma Reddy Battula. --- .../apache/hadoop/yarn/server/resourcemanager/MockRM.java | 7 +++++-- .../yarn/server/resourcemanager/TestRMAdminService.java | 3 --- .../hadoop/yarn/server/resourcemanager/TestRMRestart.java | 2 -- .../server/resourcemanager/TestResourceTrackerService.java | 6 ------ .../server/resourcemanager/rmapp/TestNodesListManager.java | 5 ++--- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index d5b64c134c5..25c558f80e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -56,12 +56,12 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -603,6 +603,7 @@ public class MockRM extends ResourceManager { public MockNM registerNode(String nodeIdStr, int memory) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService()); nm.registerNode(); + drainEvents(); return nm; } @@ -611,6 +612,7 @@ public class MockRM extends ResourceManager { MockNM nm = new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService()); nm.registerNode(); + drainEvents(); return nm; } @@ -620,6 +622,7 @@ public class MockRM extends ResourceManager { new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(), YarnVersionInfo.getVersion()); nm.registerNode(runningApplications); + drainEvents(); return nm; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 4513cbb337a..5c69411b981 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -27,9 +27,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -49,7 +47,6 @@ import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 96066877236..3b9952b287f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -81,8 +81,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index dd37b67fdfe..68ba7ecd424 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -176,7 +176,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { MockNM nm2 = rm.registerNode("host2:5678", 10240); MockNM nm3 = rm.registerNode("localhost:4433", 1024); - rm.drainEvents(); int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); @@ -208,7 +207,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.getNodesListManager().refreshNodes(conf); nm3 = rm.registerNode("localhost:4433", 1024); - rm.drainEvents(); nodeHeartbeat = nm3.nodeHeartbeat(true); rm.drainEvents(); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); @@ -1027,7 +1025,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { // unhealthy node changed back to healthy nm2 = rm.registerNode("host2:5678", 5120); - rm.drainEvents(); response = nm2.nodeHeartbeat(true); response = nm2.nodeHeartbeat(true); rm.drainEvents(); @@ -1035,7 +1032,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { // reconnect of node with changed capability nm1 = rm.registerNode("host2:5678", 10240); - rm.drainEvents(); response = nm1.nodeHeartbeat(true); rm.drainEvents(); Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction())); @@ -1045,7 +1041,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { List runningApps = new ArrayList(); runningApps.add(ApplicationId.newInstance(1, 0)); nm1 = rm.registerNode("host2:5678", 15360, 2, runningApps); - rm.drainEvents(); response = nm1.nodeHeartbeat(true); rm.drainEvents(); Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction())); @@ -1055,7 +1050,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase { nm1 = new MockNM("host1:1234", 5120, rm.getResourceTrackerService()); nm1.setHttpPort(3); nm1.registerNode(); - rm.drainEvents(); response = nm1.nodeHeartbeat(true); response = nm1.nodeHeartbeat(true); rm.drainEvents(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java index e747df40753..507165d3676 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java @@ -29,8 +29,8 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AbstractEvent; -import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.util.ControlledClock; -import org.apache.hadoop.yarn.util.SystemClock; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -236,7 +235,7 @@ public class TestNodesListManager { * Create dispatcher object */ private Dispatcher getDispatcher() { - Dispatcher dispatcher = new AsyncDispatcher() { + Dispatcher dispatcher = new DrainDispatcher() { @SuppressWarnings({ "rawtypes", "unchecked" }) @Override public EventHandler getEventHandler() { From 917464505c0e930ebeb4c775d829e51c56a48686 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Tue, 5 Apr 2016 09:07:24 -0500 Subject: [PATCH 56/75] HDFS-10239. Fsshell mv fails if port usage doesn't match in src and destination paths. Contributed by Kuhu Shukla. --- .../apache/hadoop/fs/shell/MoveCommands.java | 6 +++- .../org/apache/hadoop/hdfs/TestDFSShell.java | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java index 02a3b251bf0..d35928228a7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java @@ -100,7 +100,11 @@ class MoveCommands { @Override protected void processPath(PathData src, PathData target) throws IOException { - if (!src.fs.getUri().equals(target.fs.getUri())) { + String srcUri = src.fs.getUri().getScheme() + "://" + + src.fs.getUri().getHost(); + String dstUri = target.fs.getUri().getScheme() + "://" + + target.fs.getUri().getHost(); + if (!srcUri.equals(dstUri)) { throw new PathIOException(src.toString(), "Does not match target filesystem"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 41cd5c0c675..b75ac116975 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -559,6 +559,37 @@ public class TestDFSShell { } } + @Test + public void testMoveWithTargetPortEmpty() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .format(true) + .numDataNodes(2) + .nameNodePort(8020) + .waitSafeMode(true) + .build(); + FileSystem srcFs = cluster.getFileSystem(); + FsShell shell = new FsShell(); + shell.setConf(conf); + String[] argv = new String[2]; + argv[0] = "-mkdir"; + argv[1] = "/testfile"; + ToolRunner.run(shell, argv); + argv = new String[3]; + argv[0] = "-mv"; + argv[1] = srcFs.getUri() + "/testfile"; + argv[2] = "hdfs://localhost/testfile2"; + int ret = ToolRunner.run(shell, argv); + assertEquals("mv should have succeeded", 0, ret); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + @Test (timeout = 30000) public void testURIPaths() throws Exception { Configuration srcConf = new HdfsConfiguration(); From 00058167431475c6e63c80207424f1d365569e3a Mon Sep 17 00:00:00 2001 From: Junping Du Date: Tue, 5 Apr 2016 09:01:08 -0700 Subject: [PATCH 57/75] YARN-4916. TestNMProxy.tesNMProxyRPCRetry fails. Contributed by Tibor Kiss. --- .../yarn/server/nodemanager/containermanager/TestNMProxy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java index 7ce15c5441f..46b32deadc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java @@ -188,8 +188,7 @@ public class TestNMProxy extends BaseContainerManagerTest { Assert.fail("should get socket exception"); } catch (IOException e) { // socket exception should be thrown immediately, without RPC retries. - Assert.assertTrue(e.toString(). - contains("Failed on local exception: java.net.SocketException")); + Assert.assertTrue(e instanceof java.net.SocketException); } } From 85ec5573eb9fd746a9295ecc6fe1ae683073aaf5 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Wed, 6 Apr 2016 03:22:48 +0900 Subject: [PATCH 58/75] HADOOP-12672. RPC timeout should not override IPC ping interval (iwasakims) --- .../java/org/apache/hadoop/ipc/Client.java | 57 ++++++++++++---- .../src/main/resources/core-default.xml | 9 ++- .../java/org/apache/hadoop/ipc/TestRPC.java | 67 +++++++++++++++++-- .../hdfs/client/impl/DfsClientConf.java | 2 +- 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 7e6c7e3be26..fb11cb7a70b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -239,14 +239,33 @@ public class Client { * * @param conf Configuration * @return the timeout period in milliseconds. -1 if no timeout value is set + * @deprecated use {@link #getRpcTimeout(Configuration)} instead */ + @Deprecated final public static int getTimeout(Configuration conf) { + int timeout = getRpcTimeout(conf); + if (timeout > 0) { + return timeout; + } if (!conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT)) { return getPingInterval(conf); } return -1; } + + /** + * The time after which a RPC will timeout. + * + * @param conf Configuration + * @return the timeout period in milliseconds. + */ + public static final int getRpcTimeout(Configuration conf) { + int timeout = + conf.getInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, + CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_DEFAULT); + return (timeout < 0) ? 0 : timeout; + } /** * set the connection timeout value in configuration * @@ -386,7 +405,7 @@ public class Client { private Socket socket = null; // connected socket private DataInputStream in; private DataOutputStream out; - private int rpcTimeout; + private final int rpcTimeout; private int maxIdleTime; //connections will be culled if it was idle for //maxIdleTime msecs private final RetryPolicy connectionRetryPolicy; @@ -394,8 +413,9 @@ public class Client { private int maxRetriesOnSocketTimeouts; private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm private final boolean tcpLowLatency; // if T then use low-delay QoS - private boolean doPing; //do we need to send ping message - private int pingInterval; // how often sends ping to the server in msecs + private final boolean doPing; //do we need to send ping message + private final int pingInterval; // how often sends ping to the server + private final int soTimeout; // used by ipc ping and rpc timeout private ByteArrayOutputStream pingRequest; // ping message // currently active calls @@ -434,6 +454,14 @@ public class Client { pingHeader.writeDelimitedTo(pingRequest); } this.pingInterval = remoteId.getPingInterval(); + if (rpcTimeout > 0) { + // effective rpc timeout is rounded up to multiple of pingInterval + // if pingInterval < rpcTimeout. + this.soTimeout = (doPing && pingInterval < rpcTimeout) ? + pingInterval : rpcTimeout; + } else { + this.soTimeout = pingInterval; + } this.serviceClass = serviceClass; if (LOG.isDebugEnabled()) { LOG.debug("The ping interval is " + this.pingInterval + " ms."); @@ -484,12 +512,12 @@ public class Client { /* Process timeout exception * if the connection is not going to be closed or - * is not configured to have a RPC timeout, send a ping. - * (if rpcTimeout is not set to be 0, then RPC should timeout. - * otherwise, throw the timeout exception. + * the RPC is not timed out yet, send a ping. */ - private void handleTimeout(SocketTimeoutException e) throws IOException { - if (shouldCloseConnection.get() || !running.get() || rpcTimeout > 0) { + private void handleTimeout(SocketTimeoutException e, int waiting) + throws IOException { + if (shouldCloseConnection.get() || !running.get() || + (0 < rpcTimeout && rpcTimeout <= waiting)) { throw e; } else { sendPing(); @@ -503,11 +531,13 @@ public class Client { */ @Override public int read() throws IOException { + int waiting = 0; do { try { return super.read(); } catch (SocketTimeoutException e) { - handleTimeout(e); + waiting += soTimeout; + handleTimeout(e, waiting); } } while (true); } @@ -520,11 +550,13 @@ public class Client { */ @Override public int read(byte[] buf, int off, int len) throws IOException { + int waiting = 0; do { try { return super.read(buf, off, len); } catch (SocketTimeoutException e) { - handleTimeout(e); + waiting += soTimeout; + handleTimeout(e, waiting); } } while (true); } @@ -632,10 +664,7 @@ public class Client { } NetUtils.connect(this.socket, server, connectionTimeout); - if (rpcTimeout > 0) { - pingInterval = rpcTimeout; // rpcTimeout overwrites pingInterval - } - this.socket.setSoTimeout(pingInterval); + this.socket.setSoTimeout(soTimeout); return; } catch (ConnectTimeoutException toe) { /* Check for an address change and update the local reference. diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index d6e2bcd8ca0..b3436da6702 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1054,7 +1054,7 @@ true Send a ping to the server when timeout on reading the response, if set to true. If no failure is detected, the client retries until at least - a byte is read. + a byte is read or the time given by ipc.client.rpc-timeout.ms is passed. @@ -1071,10 +1071,9 @@ ipc.client.rpc-timeout.ms 0 Timeout on waiting response from server, in milliseconds. - Currently this timeout works only when ipc.client.ping is set to true - because it uses the same facilities with IPC ping. - The timeout overrides the ipc.ping.interval and client will throw exception - instead of sending ping when the interval is passed. + If ipc.client.ping is set to true and this rpc-timeout is greater than + the value of ipc.ping.interval, the effective value of the rpc-timeout is + rounded up to multiple of ipc.ping.interval. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 2ebd1c5e4e1..4eb3beac25e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -1118,14 +1118,67 @@ public class TestRPC extends TestRpcBase { .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000); try { - proxy = getClient(addr, conf); - proxy.sleep(null, newSleepRequest(3000)); - fail("RPC should time out."); - } catch (ServiceException e) { - assertTrue(e.getCause() instanceof SocketTimeoutException); - LOG.info("got expected timeout.", e); + // Test RPC timeout with default ipc.client.ping. + try { + Configuration c = new Configuration(conf); + c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000); + proxy = getClient(addr, c); + proxy.sleep(null, newSleepRequest(3000)); + fail("RPC should time out."); + } catch (ServiceException e) { + assertTrue(e.getCause() instanceof SocketTimeoutException); + LOG.info("got expected timeout.", e); + } + + // Test RPC timeout when ipc.client.ping is false. + try { + Configuration c = new Configuration(conf); + c.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, false); + c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000); + proxy = getClient(addr, c); + proxy.sleep(null, newSleepRequest(3000)); + fail("RPC should time out."); + } catch (ServiceException e) { + assertTrue(e.getCause() instanceof SocketTimeoutException); + LOG.info("got expected timeout.", e); + } + + // Test negative timeout value. + try { + Configuration c = new Configuration(conf); + c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, -1); + proxy = getClient(addr, c); + proxy.sleep(null, newSleepRequest(2000)); + } catch (ServiceException e) { + LOG.info("got unexpected exception.", e); + fail("RPC should not time out."); + } + + // Test RPC timeout greater than ipc.ping.interval. + try { + Configuration c = new Configuration(conf); + c.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true); + c.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, 800); + c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000); + proxy = getClient(addr, c); + + try { + // should not time out because effective rpc-timeout is + // multiple of ping interval: 1600 (= 800 * (1000 / 800 + 1)) + proxy.sleep(null, newSleepRequest(1300)); + } catch (ServiceException e) { + LOG.info("got unexpected exception.", e); + fail("RPC should not time out."); + } + + proxy.sleep(null, newSleepRequest(2000)); + fail("RPC should time out."); + } catch (ServiceException e) { + assertTrue(e.getCause() instanceof SocketTimeoutException); + LOG.info("got expected timeout.", e); + } + } finally { stop(server, proxy); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index eae3c8ee48d..883a5bbf092 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -141,7 +141,7 @@ public class DfsClientConf { public DfsClientConf(Configuration conf) { // The hdfsTimeout is currently the same as the ipc timeout - hdfsTimeout = Client.getTimeout(conf); + hdfsTimeout = Client.getRpcTimeout(conf); maxRetryAttempts = conf.getInt( Retry.MAX_ATTEMPTS_KEY, From 30206346cf13fe1b7267f86e7c210b77c86b88c9 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Wed, 6 Apr 2016 03:47:22 +0900 Subject: [PATCH 59/75] YARN-4915. Fix typo in YARN Secure Containers documentation (Takashi Ohnishi via iwasakims) --- .../hadoop-yarn-site/src/site/markdown/SecureContainer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md index cd4f913ec58..f7706c7426b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md @@ -114,7 +114,7 @@ min.user.id=1000#Prevent other super-users `yarn.nodemanager.windows-secure-container-executor.impersonate.allowed` should contain the users that are allowed to create containers in the cluster. These users will be allowed to be impersonated by hadoopwinutilsvc. - `yarn.nodemanager.windows-secure-container-executor.impersonate.denied` should contain users that are explictly forbiden from creating containers. hadoopwinutilsvc will refuse to impersonate these users. + `yarn.nodemanager.windows-secure-container-executor.impersonate.denied` should contain users that are explicitly forbidden from creating containers. hadoopwinutilsvc will refuse to impersonate these users. `yarn.nodemanager.windows-secure-container-executor.local-dirs` should contain the nodemanager local dirs. hadoopwinutilsvc will allow only file operations under these directories. This should contain the same values as `$yarn.nodemanager.local-dirs, $yarn.nodemanager.log-dirs` but note that hadoopwinutilsvc XML configuration processing does not do substitutions so the value must be the final value. All paths must be absolute and no environment variable substitution will be performed. The paths are compared LOCAL\_INVARIANT case insensitive string comparison, the file path validated must start with one of the paths listed in local-dirs configuration. Use comma as path separator:`,` From 500e5a5952f8f34bf0e1e2653fa01b357d68cc8f Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Wed, 6 Apr 2016 04:00:31 +0900 Subject: [PATCH 60/75] YARN-4917. Fix typos in documentation of Capacity Scheduler. (Takashi Ohnishi via iwasakims) --- .../src/site/markdown/CapacityScheduler.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index e86c4f9065f..8c0b8c88824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -55,11 +55,11 @@ The `CapacityScheduler` supports the following features: * **Hierarchical Queues** - Hierarchy of queues is supported to ensure resources are shared among the sub-queues of an organization before other queues are allowed to use free resources, there-by providing more control and predictability. -* **Capacity Guarantees** - Queues are allocated a fraction of the capacity of the grid in the sense that a certain capacity of resources will be at their disposal. All applications submitted to a queue will have access to the capacity allocated to the queue. Adminstrators can configure soft limits and optional hard limits on the capacity allocated to each queue. +* **Capacity Guarantees** - Queues are allocated a fraction of the capacity of the grid in the sense that a certain capacity of resources will be at their disposal. All applications submitted to a queue will have access to the capacity allocated to the queue. Administrators can configure soft limits and optional hard limits on the capacity allocated to each queue. * **Security** - Each queue has strict ACLs which controls which users can submit applications to individual queues. Also, there are safe-guards to ensure that users cannot view and/or modify applications from other users. Also, per-queue and system administrator roles are supported. -* **Elasticity** - Free resources can be allocated to any queue beyond its capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is also supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artifical silos of resources in the cluster which helps utilization. +* **Elasticity** - Free resources can be allocated to any queue beyond its capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is also supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artificial silos of resources in the cluster which helps utilization. * **Multi-tenancy** - Comprehensive set of limits are provided to prevent a single application, user and queue from monopolizing resources of the queue or the cluster as a whole to ensure that the cluster isn't overwhelmed. @@ -67,9 +67,9 @@ The `CapacityScheduler` supports the following features: * Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. - * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queueus*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. + * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. -* **Resource-based Scheduling** - Support for resource-intensive applications, where-in a application can optionally specify higher resource-requirements than the default, there-by accomodating applications with differing resource requirements. Currently, *memory* is the resource requirement supported. +* **Resource-based Scheduling** - Support for resource-intensive applications, where-in a application can optionally specify higher resource-requirements than the default, there-by accommodating applications with differing resource requirements. Currently, *memory* is the resource requirement supported. * **Queue Mapping based on User or Group** - This feature allows users to map a job to a specific queue based on the user or group. @@ -163,7 +163,7 @@ Configuration | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to aspecific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.| +| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.| | `yarn.scheduler.capacity.queue-mappings-override.enable` | This function is used to specify whether the user specified queues can be overridden. This is a Boolean value and the default value is *false*. | Example: From 0cd320a8463efe19a6228f9fe14693aa37ac8a10 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Tue, 5 Apr 2016 13:41:19 -0700 Subject: [PATCH 61/75] HDFS-10235. Last contact for Live Nodes should be relative time. Contributed by Brahma Reddy Battula. --- .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 3d9ca4270e0..a9c33041b1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -310,7 +310,7 @@ {#LiveNodes} {name} ({xferaddr}) - {#helper_relative_time value="{lastContact}"/} + {lastContact}s
      {capacity|fmt_bytes}
      From 9ba1e5af06070ba01dcf46e1a4c66713a1d43352 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Tue, 5 Apr 2016 16:26:18 -0500 Subject: [PATCH 62/75] HDFS-10261. TestBookKeeperHACheckpoints doesn't handle ephemeral HTTP ports. Contributed by Eric Badger. --- .../TestBookKeeperHACheckpoints.java | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java index ed53512ef1b..b8fc30d0e20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.contrib.bkjournal; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -27,6 +29,9 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import java.net.BindException; +import java.util.Random; + /** * Runs the same tests as TestStandbyCheckpoints, but * using a bookkeeper journal manager as the shared directory @@ -39,6 +44,9 @@ public class TestBookKeeperHACheckpoints extends TestStandbyCheckpoints { private static BKJMUtil bkutil = null; static int numBookies = 3; static int journalCount = 0; + private final Random random = new Random(); + + private static final Log LOG = LogFactory.getLog(TestStandbyCheckpoints.class); @SuppressWarnings("rawtypes") @Override @@ -49,22 +57,34 @@ public class TestBookKeeperHACheckpoints extends TestStandbyCheckpoints { BKJMUtil.createJournalURI("/checkpointing" + journalCount++) .toString()); BKJMUtil.addJournalManagerDefinition(conf); - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(1) - .manageNameDfsSharedDirs(false) - .build(); - cluster.waitActive(); + int retryCount = 0; + while (true) { + try { + int basePort = 10060 + random.nextInt(100) * 2; + MiniDFSNNTopology topology = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1))); - setNNs(); - fs = HATestUtil.configureFailoverFs(cluster, conf); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(1) + .manageNameDfsSharedDirs(false) + .build(); + cluster.waitActive(); - cluster.transitionToActive(0); + setNNs(); + fs = HATestUtil.configureFailoverFs(cluster, conf); + + cluster.transitionToActive(0); + ++retryCount; + break; + } catch (BindException e) { + LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry " + + retryCount + " times"); + } + } } @BeforeClass From 21eb4284487d6f8e4beedb8a0c3168e952f224fc Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 5 Apr 2016 16:24:11 -0700 Subject: [PATCH 63/75] YARN-4699. Scheduler UI and REST o/p is not in sync when -replaceLabelsOnNode is used to change label of a node. (Sunil G via wangda) --- .../scheduler/capacity/AbstractCSQueue.java | 6 +++ .../scheduler/capacity/CSQueueUtils.java | 2 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 40 ++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 6e715fb225d..c7d6d028983 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -591,6 +591,9 @@ public abstract class AbstractCSQueue implements CSQueue { } // ResourceUsage has its own lock, no addition lock needs here. queueUsage.incUsed(nodeLabel, resourceToInc); + CSQueueUtils.updateUsedCapacity(resourceCalculator, + labelManager.getResourceByLabel(nodeLabel, Resources.none()), + minimumAllocation, queueUsage, queueCapacities, nodeLabel); if (null != parent) { parent.incUsedResource(nodeLabel, resourceToInc, null); } @@ -604,6 +607,9 @@ public abstract class AbstractCSQueue implements CSQueue { } // ResourceUsage has its own lock, no addition lock needs here. queueUsage.decUsed(nodeLabel, resourceToDec); + CSQueueUtils.updateUsedCapacity(resourceCalculator, + labelManager.getResourceByLabel(nodeLabel, Resources.none()), + minimumAllocation, queueUsage, queueCapacities, nodeLabel); if (null != parent) { parent.decUsedResource(nodeLabel, resourceToDec, null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 9cdcb727899..0166d833eea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -180,7 +180,7 @@ class CSQueueUtils { * Update partitioned resource usage, if nodePartition == null, will update * used resource for all partitions of this queue. */ - private static void updateUsedCapacity(final ResourceCalculator rc, + public static void updateUsedCapacity(final ResourceCalculator rc, final Resource totalPartitionResource, final Resource minimumAllocation, ResourceUsage queueResourceUsage, QueueCapacities queueCapacities, String nodePartition) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index fe24b2d736a..cff79cd5c1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -100,6 +100,12 @@ public class TestCapacitySchedulerNodeLabelUpdate { checkAMUsedResource(rm, queueName, memory, RMNodeLabelsManager.NO_LABEL); } + private void checkUsedCapacity(MockRM rm, String queueName, int capacity, + int total) { + checkUsedCapacity(rm, queueName, capacity, total, + RMNodeLabelsManager.NO_LABEL); + } + private void checkUsedResource(MockRM rm, String queueName, int memory, String label) { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); @@ -108,6 +114,15 @@ public class TestCapacitySchedulerNodeLabelUpdate { .getMemory()); } + private void checkUsedCapacity(MockRM rm, String queueName, int capacity, + int total, String label) { + float epsillon = 0.0001f; + CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); + CSQueue queue = scheduler.getQueue(queueName); + Assert.assertEquals((float)capacity/total, + queue.getQueueCapacities().getUsedCapacity(label), epsillon); + } + private void checkAMUsedResource(MockRM rm, String queueName, int memory, String label) { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); @@ -188,7 +203,7 @@ public class TestCapacitySchedulerNodeLabelUpdate { rm.stop(); } - @Test (timeout = 60000) + @Test public void testResourceUsageWhenNodeUpdatesPartition() throws Exception { // set node -> label @@ -233,16 +248,23 @@ public class TestCapacitySchedulerNodeLabelUpdate { // queue-a used x=1G, ""=1G checkUsedResource(rm, "a", 1024, "x"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 1024, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 8000); CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId()); // change h1's label to z + mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("z"))); cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("z")))); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 1024, "z"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 8000, "z"); + checkUsedCapacity(rm, "a", 1024, 8000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 1024, "z"); checkUsedResource(rm, "root", 1024); @@ -254,12 +276,18 @@ public class TestCapacitySchedulerNodeLabelUpdate { app.getAppAttemptResourceUsage().getUsed("z").getMemory()); // change h1's label to y + mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("y"))); cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("y")))); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 1024, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 16000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 1024, 8000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 1024, "y"); checkUsedResource(rm, "root", 0, "z"); @@ -278,11 +306,17 @@ public class TestCapacitySchedulerNodeLabelUpdate { Set emptyLabels = new HashSet<>(); Map> map = ImmutableMap.of(nm1.getNodeId(), emptyLabels); + mgr.replaceLabelsOnNode(map); cs.handle(new NodeLabelsUpdateSchedulerEvent(map)); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 0, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 2048); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 0, 8000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 2048, 16000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 0, "y"); checkUsedResource(rm, "root", 0, "z"); @@ -314,6 +348,10 @@ public class TestCapacitySchedulerNodeLabelUpdate { checkUsedResource(rm, "a", 0, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 0); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 0, 8000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 0, 16000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 0, "y"); checkUsedResource(rm, "root", 0, "z"); From b41e65e5bc9459b4d950a2c53860a223f1a0d2ec Mon Sep 17 00:00:00 2001 From: Varun Vasudev Date: Wed, 6 Apr 2016 13:41:33 +0530 Subject: [PATCH 64/75] YARN-4906. Capture container start/finish time in container metrics. Contributed by Jian He. --- .../container/ContainerImpl.java | 22 +++++++++++++++++++ .../monitor/ContainerMetrics.java | 18 +++++++++++++++ .../containermanager/TestAuxServices.java | 2 +- .../container/TestContainer.java | 11 ++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index da8a3a67152..a43a0058805 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerMetrics; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent; import org.apache.hadoop.yarn.server.nodemanager.Context; @@ -100,6 +101,7 @@ public class ContainerImpl implements Container { private boolean wasLaunched; private long containerLocalizationStartTime; private long containerLaunchStartTime; + private ContainerMetrics containerMetrics; private static Clock clock = SystemClock.getInstance(); /** The NM-wide configuration - not specific to this container */ @@ -147,6 +149,21 @@ public class ContainerImpl implements Container { this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); this.context = context; + boolean containerMetricsEnabled = + conf.getBoolean(YarnConfiguration.NM_CONTAINER_METRICS_ENABLE, + YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_ENABLE); + + if (containerMetricsEnabled) { + long flushPeriod = + conf.getLong(YarnConfiguration.NM_CONTAINER_METRICS_PERIOD_MS, + YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS); + long unregisterDelay = conf.getLong( + YarnConfiguration.NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS, + YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS); + containerMetrics = ContainerMetrics + .forContainer(containerId, flushPeriod, unregisterDelay); + containerMetrics.recordStartTime(clock.getTime()); + } stateMachine = stateMachineFactory.make(this); } @@ -989,6 +1006,11 @@ public class ContainerImpl implements Container { @SuppressWarnings("unchecked") public void transition(ContainerImpl container, ContainerEvent event) { container.metrics.releaseContainer(container.resource); + if (container.containerMetrics != null) { + container.containerMetrics + .recordFinishTimeAndExitCode(clock.getTime(), container.exitCode); + container.containerMetrics.finished(); + } container.sendFinishedEvents(); //if the current state is NEW it means the CONTAINER_INIT was never // sent for the event, thus no need to send the CONTAINER_STOP diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java index 9d17db04889..f85431ee0bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java @@ -100,6 +100,15 @@ public class ContainerMetrics implements MetricsSource { @Metric public MutableGaugeLong localizationDurationMs; + @Metric + public MutableGaugeLong startTime; + + @Metric + public MutableGaugeLong finishTime; + + @Metric + public MutableGaugeInt exitCode; + static final MetricsInfo RECORD_INFO = info("ContainerResource", "Resource limit and usage by container"); @@ -277,6 +286,15 @@ public class ContainerMetrics implements MetricsSource { this.localizationDurationMs.set(localizationDuration); } + public void recordStartTime(long startTime) { + this.startTime.set(startTime); + } + + public void recordFinishTimeAndExitCode(long finishTime, int exitCode) { + this.finishTime.set(finishTime); + this.exitCode.set(exitCode); + } + private synchronized void scheduleTimerTaskIfRequired() { if (flushPeriodMs > 0) { // Lazily initialize timer diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 91466e8f0ce..9d0d0c037dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -195,7 +195,7 @@ public class TestAuxServices { ContainerId.newContainerId(attemptId, 1), "", "", Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0); Context context = mock(Context.class); - Container container = new ContainerImpl(null, null, null, null, + Container container = new ContainerImpl(new YarnConfiguration(), null, null, null, null, cti, context); ContainerId containerId = container.getContainerId(); Resource resource = container.getResource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 3e062367e97..cc98bdc54d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerMetrics; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; @@ -333,6 +334,7 @@ public class TestContainer { @Test public void testKillOnNew() throws Exception { WrappedContainer wc = null; + try { wc = new WrappedContainer(13, 314159265358979L, 4344, "yak"); assertEquals(ContainerState.NEW, wc.c.getContainerState()); @@ -345,6 +347,15 @@ public class TestContainer { assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics() .contains("KillRequest")); assertEquals(killed + 1, metrics.getKilledContainers()); + // check container metrics is generated. + ContainerMetrics containerMetrics = + ContainerMetrics.forContainer(wc.cId, 1, 5000); + Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, + containerMetrics.exitCode.value()); + Assert.assertTrue(containerMetrics.startTime.value() > 0); + Assert.assertTrue( + containerMetrics.finishTime.value() > containerMetrics.startTime + .value()); } finally { if (wc != null) { wc.finished(); From de96d7c88a42cd54bd88ce2de63122998e967efa Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 6 Apr 2016 08:32:35 -0700 Subject: [PATCH 65/75] =?UTF-8?q?MAPREDUCE-6670.=20TestJobListCache#testEv?= =?UTF-8?q?iction=20sometimes=20fails=20on=20Windows=20with=20timeout.=20C?= =?UTF-8?q?ontributed=20by=20Gergely=20Nov=C3=A1k.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java index 6ebbb7c139a..3ccc222b977 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobListCache.java @@ -45,7 +45,7 @@ public class TestJobListCache { cache.values().size()); } - @Test (timeout = 1000) + @Test (timeout = 5000) public void testEviction() throws InterruptedException { int maxSize = 2; JobListCache cache = new JobListCache(maxSize, 1000); From 221b3a8722f84f8e9ad0a98eea38a12cc4ad2f24 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 6 Apr 2016 10:42:59 -0700 Subject: [PATCH 66/75] HDFS-10192. Namenode safemode not coming out during failover. Contributed by Brahma Reddy Battula. --- .../server/blockmanagement/BlockManager.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 1 + .../TestBlockManagerSafeMode.java | 14 +++++++- .../server/namenode/ha/TestHASafeMode.java | 35 +++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 66ab78984bb..104d72379a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1974,7 +1974,7 @@ public class BlockManager implements BlockStatsMXBean { return bmSafeMode.leaveSafeMode(force); } - void checkSafeMode() { + public void checkSafeMode() { bmSafeMode.checkSafeMode(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 9ff4be637d1..681fc965a4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1154,6 +1154,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } } finally { startingActiveService = false; + blockManager.checkSafeMode(); writeUnlock(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java index cb749c7e2d1..a3476698a77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java @@ -66,6 +66,7 @@ public class TestBlockManagerSafeMode { private static final long BLOCK_THRESHOLD = (long)(BLOCK_TOTAL * THRESHOLD); private static final int EXTENSION = 1000; // 1 second + private FSNamesystem fsn; private BlockManager bm; private DatanodeManager dn; private BlockManagerSafeMode bmSafeMode; @@ -90,7 +91,7 @@ public class TestBlockManagerSafeMode { conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, DATANODE_NUM); - FSNamesystem fsn = mock(FSNamesystem.class); + fsn = mock(FSNamesystem.class); doReturn(true).when(fsn).hasWriteLock(); doReturn(true).when(fsn).hasReadLock(); doReturn(true).when(fsn).isRunning(); @@ -163,6 +164,17 @@ public class TestBlockManagerSafeMode { setBlockSafe(BLOCK_THRESHOLD); bmSafeMode.checkSafeMode(); assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus()); + + // should stay in PENDING_THRESHOLD during transitionToActive + doReturn(true).when(fsn).inTransitionToActive(); + Whitebox.setInternalState(bmSafeMode, "extension", 0); + setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD); + setBlockSafe(BLOCK_THRESHOLD); + bmSafeMode.checkSafeMode(); + assertEquals(BMSafeModeStatus.PENDING_THRESHOLD, getSafeModeStatus()); + doReturn(false).when(fsn).inTransitionToActive(); + bmSafeMode.checkSafeMode(); + assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 4b1d27d5b90..8b8343c7c7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -851,4 +851,39 @@ public class TestHASafeMode { cluster.shutdown(); } } + + @Test(timeout = 60000) + public void testSafeModeExitAfterTransition() throws Exception { + DFSTestUtil.createFile(fs, new Path("/test"), 5 * BLOCK_SIZE, (short) 3, + 1L); + banner("Stopping standby"); + cluster.shutdownNameNode(1); + DFSTestUtil.createFile(fs, new Path("/test2"), 3 * BLOCK_SIZE, (short) 3, + 1L); + // Roll edit logs to be read by standby + nn0.getRpcServer().rollEditLog(); + fs.delete(new Path("/test"), true); + // Wait till the blocks are deleted from all DNs + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return cluster.getNamesystem(0).getBlockManager() + .getPendingDeletionBlocksCount() == 0; + } + }, 1000, 10000); + restartStandby(); + // Wait till all the datanodes are registered. + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return cluster.getNamesystem(1).getNumLiveDataNodes() == 3; + } + }, 1000, 10000); + cluster.triggerBlockReports(); + NameNodeAdapter.abortEditLogs(nn0); + cluster.shutdownNameNode(0); + banner(nn1.getNamesystem().getSafemode()); + cluster.transitionToActive(1); + assertSafeMode(nn1, 3, 3, 3, 0); + } } From 188f65287d5b2f26a8862c88198f83ac59035016 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Wed, 6 Apr 2016 11:28:34 -0700 Subject: [PATCH 67/75] HDFS-6520. hdfs fsck passes invalid length value when creating BlockReader (Xiao Chen via cmccabe) --- .../hadoop/hdfs/BlockReaderFactory.java | 4 +- .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../TestBlockTokenWithDFS.java | 2 +- .../datanode/TestDataNodeVolumeFailure.java | 2 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 122 +++++++++++++++++- 5 files changed, 126 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java index 8a0050ff540..7af46097e79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java @@ -150,7 +150,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { private ClientContext clientContext; /** - * Number of bytes to read. -1 indicates no limit. + * Number of bytes to read. Must be set to a non-negative value. */ private long length = -1; @@ -341,6 +341,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { */ public BlockReader build() throws IOException { Preconditions.checkNotNull(configuration); + Preconditions + .checkState(length >= 0, "Length must be set to a non-negative value"); BlockReader reader = tryToCreateExternalBlockReader(); if (reader != null) { return reader; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 291ba5673bc..80f510cfa19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -931,7 +931,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { setBlock(block). setBlockToken(lblock.getBlockToken()). setStartOffset(0). - setLength(-1). + setLength(block.getNumBytes()). setVerifyChecksum(true). setClientName("fsck"). setDatanodeInfo(chosenNode). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java index 8e65ff68a86..aa46de2542f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java @@ -158,7 +158,7 @@ public class TestBlockTokenWithDFS { setBlockToken(lblock.getBlockToken()). setInetSocketAddress(targetAddr). setStartOffset(0). - setLength(-1). + setLength(0). setVerifyChecksum(true). setClientName("TestBlockTokenWithDFS"). setDatanodeInfo(nodes[0]). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 5da5cd96f29..73f02bd8860 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -523,7 +523,7 @@ public class TestDataNodeVolumeFailure { "test-blockpoolid", block.getBlockId())). setBlockToken(lblock.getBlockToken()). setStartOffset(0). - setLength(-1). + setLength(0). setVerifyChecksum(true). setClientName("TestDataNodeVolumeFailure"). setDatanodeInfo(datanode). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 1818db83202..7cb6edc35eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -35,6 +35,7 @@ import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.PrintStream; @@ -46,6 +47,8 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.channels.FileChannel; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,12 +57,15 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.base.Supplier; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; @@ -101,6 +107,7 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.apache.log4j.RollingFileAppender; +import org.junit.Assert; import org.junit.Test; import com.google.common.collect.Sets; @@ -147,7 +154,7 @@ public class TestFsck { assertEquals(expectedErrCode, errCode); } GenericTestUtils.setLogLevel(FSPermissionChecker.LOG, Level.INFO); - FSImage.LOG.error("OUTPUT = " + bStream.toString()); + FSImage.LOG.info("OUTPUT = " + bStream.toString()); return bStream.toString(); } @@ -475,7 +482,25 @@ public class TestFsck { } } } - + + public void corruptBlocks(MiniDFSCluster cluster) throws IOException { + for (int corruptIdx : blocksToCorrupt) { + // Corrupt a block by deleting it + ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(name, + blockSize * corruptIdx, Long.MAX_VALUE).get(0).getBlock(); + for (int i = 0; i < numDataNodes; i++) { + File blockFile = cluster.getBlockFile(i, block); + if(blockFile != null && blockFile.exists()) { + FileOutputStream blockFileStream = + new FileOutputStream(blockFile, false); + blockFileStream.write("corrupt".getBytes()); + blockFileStream.close(); + FSImage.LOG.info("Corrupted block file " + blockFile); + } + } + } + } + public void checkSalvagedRemains() throws IOException { int chainIdx = 0; HdfsFileStatus status = dfsClient.getFileInfo(name); @@ -1890,4 +1915,97 @@ public class TestFsck { if (cluster != null) {cluster.shutdown();} } } + + @Test (timeout = 300000) + public void testFsckMoveAfterCorruption() throws Exception { + final int DFS_BLOCK_SIZE = 512 * 1024; + final int NUM_DATANODES = 1; + final int REPLICATION = 1; + MiniDFSCluster cluster = null; + try { + final Configuration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); + conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION); + cluster = new MiniDFSCluster.Builder(conf).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + cluster.waitActive(); + + final String srcDir = "/srcdat"; + final DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck") + .setMinSize(DFS_BLOCK_SIZE * 2).setMaxSize(DFS_BLOCK_SIZE * 3) + .setNumFiles(1).build(); + util.createFiles(dfs, srcDir, (short) REPLICATION); + final String fileNames[] = util.getFileNames(srcDir); + FSImage.LOG.info("Created files: " + Arrays.toString(fileNames)); + + // Run fsck here. The output is automatically logged for easier debugging + String outStr = runFsck(conf, 0, true, "/", "-files", "-blocks"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + + // Corrupt the first block + final DFSClient dfsClient = new DFSClient( + new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); + final String blockFileToCorrupt = fileNames[0]; + final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt, + Sets.newHashSet(0), dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE); + ctf.corruptBlocks(cluster); + + // Wait for fsck to discover all the missing blocks + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + final String str = runFsck(conf, 1, false, "/"); + String numCorrupt = null; + for (String line : str.split(LINE_SEPARATOR)) { + Matcher m = numCorruptBlocksPattern.matcher(line); + if (m.matches()) { + numCorrupt = m.group(1); + break; + } + } + if (numCorrupt == null) { + Assert.fail("Cannot find corrupt blocks count in fsck output."); + } + if (Integer.parseInt(numCorrupt) == ctf.getTotalMissingBlocks()) { + assertTrue(str.contains(NamenodeFsck.CORRUPT_STATUS)); + return true; + } + } catch (Exception e) { + FSImage.LOG.error("Exception caught", e); + Assert.fail("Caught unexpected exception."); + } + return false; + } + }, 1000, 60000); + + runFsck(conf, 1, true, "/", "-files", "-blocks", "-racks"); + FSImage.LOG.info("Moving blocks to lost+found"); + // Fsck will return error since we corrupted a block + runFsck(conf, 1, false, "/", "-move"); + + final List retVal = new ArrayList<>(); + final RemoteIterator iter = + dfs.listFiles(new Path("/lost+found"), true); + while (iter.hasNext()) { + retVal.add(iter.next()); + } + FSImage.LOG.info("Items in lost+found: " + retVal); + + // Expect all good blocks moved, only corrupted block skipped. + long totalLength = 0; + for (LocatedFileStatus lfs: retVal) { + totalLength += lfs.getLen(); + } + Assert.assertTrue("Nothing is moved to lost+found!", totalLength > 0); + util.cleanup(dfs, srcDir); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + } From aede8c10ecad4f2a8802a834e4bd0b8286cebade Mon Sep 17 00:00:00 2001 From: Eric Payne Date: Wed, 6 Apr 2016 20:20:14 +0000 Subject: [PATCH 68/75] HDFS-9945. Datanode command for evicting writers. Contributed by Kihwal Lee --- .../hdfs/protocol/ClientDatanodeProtocol.java | 7 +++ .../ClientDatanodeProtocolTranslatorPB.java | 12 +++++ .../main/proto/ClientDatanodeProtocol.proto | 10 ++++ ...atanodeProtocolServerSideTranslatorPB.java | 15 ++++++ .../hdfs/server/datanode/BlockReceiver.java | 3 ++ .../hadoop/hdfs/server/datanode/DataNode.java | 7 +++ .../hdfs/server/datanode/DataXceiver.java | 48 +++++++++++++++--- .../server/datanode/DataXceiverServer.java | 6 +++ .../apache/hadoop/hdfs/tools/DFSAdmin.java | 21 ++++++++ ...TestClientProtocolForPipelineRecovery.java | 49 +++++++++++++++++++ 10 files changed, 170 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java index 08547c1dc1b..e5413884535 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java @@ -121,6 +121,13 @@ public interface ClientDatanodeProtocol { */ void shutdownDatanode(boolean forUpgrade) throws IOException; + /** + * Evict clients that are writing to a datanode. + * + * @throws IOException + */ + void evictWriters() throws IOException; + /** * Obtains datanode info * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java index 2fffffd9344..6aaa0257d86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.EvictWritersRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBalancerBandwidthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBalancerBandwidthResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; @@ -97,6 +98,8 @@ public class ClientDatanodeProtocolTranslatorPB implements private static final GetBalancerBandwidthRequestProto VOID_GET_BALANCER_BANDWIDTH = GetBalancerBandwidthRequestProto.newBuilder().build(); + private final static EvictWritersRequestProto VOID_EVICT_WRITERS = + EvictWritersRequestProto.newBuilder().build(); public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid, Configuration conf, int socketTimeout, boolean connectToDnViaHostname, @@ -243,6 +246,15 @@ public class ClientDatanodeProtocolTranslatorPB implements } } + @Override + public void evictWriters() throws IOException { + try { + rpcProxy.evictWriters(NULL_CONTROLLER, VOID_EVICT_WRITERS); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public DatanodeLocalInfo getDatanodeInfo() throws IOException { GetDatanodeInfoResponseProto response; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto index 954fedc5e36..e135df84fb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto @@ -114,6 +114,13 @@ message ShutdownDatanodeRequestProto { message ShutdownDatanodeResponseProto { } +/** Tell datanode to evict active clients that are writing */ +message EvictWritersRequestProto { +} + +message EvictWritersResponseProto { +} + /** * Ping datanode for liveness and quick info */ @@ -176,6 +183,9 @@ service ClientDatanodeProtocolService { rpc shutdownDatanode(ShutdownDatanodeRequestProto) returns(ShutdownDatanodeResponseProto); + rpc evictWriters(EvictWritersRequestProto) + returns(EvictWritersResponseProto); + rpc getDatanodeInfo(GetDatanodeInfoRequestProto) returns(GetDatanodeInfoResponseProto); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java index 0feecc142de..e0401f739f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.EvictWritersRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.EvictWritersResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBalancerBandwidthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBalancerBandwidthResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; @@ -67,6 +69,8 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements StartReconfigurationResponseProto.newBuilder().build(); private final static TriggerBlockReportResponseProto TRIGGER_BLOCK_REPORT_RESP = TriggerBlockReportResponseProto.newBuilder().build(); + private final static EvictWritersResponseProto EVICT_WRITERS_RESP = + EvictWritersResponseProto.newBuilder().build(); private final ClientDatanodeProtocol impl; @@ -142,6 +146,17 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements return SHUTDOWN_DATANODE_RESP; } + @Override + public EvictWritersResponseProto evictWriters(RpcController unused, + EvictWritersRequestProto request) throws ServiceException { + try { + impl.evictWriters(); + } catch (IOException e) { + throw new ServiceException(e); + } + return EVICT_WRITERS_RESP; + } + public GetDatanodeInfoResponseProto getDatanodeInfo(RpcController unused, GetDatanodeInfoRequestProto request) throws ServiceException { GetDatanodeInfoResponseProto res; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index fb0c1c5b1c3..8f9138ca6bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -889,6 +889,9 @@ class BlockReceiver implements Closeable { } public void sendOOB() throws IOException, InterruptedException { + if (isDatanode) { + return; + } ((PacketResponder) responder.getRunnable()).sendOOBResponse(PipelineAck .getRestartOOBStatus()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 989afbed1da..625eb3f3b6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -2973,6 +2973,13 @@ public class DataNode extends ReconfigurableBase shutdownThread.start(); } + @Override //ClientDatanodeProtocol + public void evictWriters() throws IOException { + checkSuperuserPrivilege(); + LOG.info("Evicting all writers."); + xserver.stopWriters(); + } + @Override //ClientDatanodeProtocol public DatanodeLocalInfo getDatanodeInfo() { long uptime = ManagementFactory.getRuntimeMXBean().getUptime()/1000; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 63bf5ae5363..d5dc3281f10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -116,6 +116,7 @@ class DataXceiver extends Receiver implements Runnable { private BlockReceiver blockReceiver = null; private final int ioFileBufferSize; private final int smallBufferSize; + private Thread xceiver = null; /** * Client Name used in previous operation. Not available on first request @@ -178,9 +179,38 @@ class DataXceiver extends Receiver implements Runnable { } public void sendOOB() throws IOException, InterruptedException { + BlockReceiver br = getCurrentBlockReceiver(); + if (br == null) { + return; + } + // This doesn't need to be in a critical section. Althogh the client + // can resue the connection to issue a different request, trying sending + // an OOB through the recently closed block receiver is harmless. LOG.info("Sending OOB to peer: " + peer); - if(blockReceiver!=null) - blockReceiver.sendOOB(); + br.sendOOB(); + } + + public void stopWriter() { + // We want to interrupt the xceiver only when it is serving writes. + synchronized(this) { + if (getCurrentBlockReceiver() == null) { + return; + } + xceiver.interrupt(); + } + LOG.info("Stopped the writer: " + peer); + } + + /** + * blockReceiver is updated at multiple places. Use the synchronized setter + * and getter. + */ + private synchronized void setCurrentBlockReceiver(BlockReceiver br) { + blockReceiver = br; + } + + private synchronized BlockReceiver getCurrentBlockReceiver() { + return blockReceiver; } /** @@ -192,6 +222,9 @@ class DataXceiver extends Receiver implements Runnable { Op op = null; try { + synchronized(this) { + xceiver = Thread.currentThread(); + } dataXceiverServer.addPeer(peer, Thread.currentThread(), this); peer.setWriteTimeout(datanode.getDnConf().socketWriteTimeout); InputStream input = socketIn; @@ -679,12 +712,12 @@ class DataXceiver extends Receiver implements Runnable { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { // open a block receiver - blockReceiver = getBlockReceiver(block, storageType, in, + setCurrentBlockReceiver(getBlockReceiver(block, storageType, in, peer.getRemoteAddressString(), peer.getLocalAddressString(), stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum, - cachingStrategy, allowLazyPersist, pinning); + cachingStrategy, allowLazyPersist, pinning)); replica = blockReceiver.getReplica(); } else { replica = datanode.data.recoverClose( @@ -853,7 +886,7 @@ class DataXceiver extends Receiver implements Runnable { IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); - blockReceiver = null; + setCurrentBlockReceiver(null); } //update metrics @@ -1060,7 +1093,6 @@ class DataXceiver extends Receiver implements Runnable { DataOutputStream proxyOut = null; Status opStatus = SUCCESS; String errMsg = null; - BlockReceiver blockReceiver = null; DataInputStream proxyReply = null; boolean IoeDuringCopyBlockOperation = false; try { @@ -1119,11 +1151,11 @@ class DataXceiver extends Receiver implements Runnable { DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto( checksumInfo.getChecksum()); // open a block receiver and check if the block does not exist - blockReceiver = getBlockReceiver(block, storageType, + setCurrentBlockReceiver(getBlockReceiver(block, storageType, proxyReply, proxySock.getRemoteSocketAddress().toString(), proxySock.getLocalSocketAddress().toString(), null, 0, 0, 0, "", null, datanode, remoteChecksum, - CachingStrategy.newDropBehind(), false, false); + CachingStrategy.newDropBehind(), false, false)); // receive a block blockReceiver.receiveBlock(null, null, replyOut, null, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 10945e71682..126d5b10d42 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -256,6 +256,12 @@ class DataXceiverServer implements Runnable { } } } + + public synchronized void stopWriters() { + for (Peer p : peers.keySet()) { + peersXceiver.get(p).stopWriter(); + } + } // Notify all peers of the shutdown and restart. // datanode.shouldRun should still be true and datanode.restarting should diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index d84d66449df..a35246f38b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -1090,6 +1090,10 @@ public class DFSAdmin extends FsShell { + "\tclients will timeout and ignore the datanode. In such case, the\n" + "\tfast start-up mode will also be disabled.\n"; + String evictWriters = "-evictWriters \n" + + "\tMake the datanode evict all clients that are writing a block.\n" + + "\tThis is useful if decommissioning is hung due to slow writers.\n"; + String getDatanodeInfo = "-getDatanodeInfo \n" + "\tGet the information about the given datanode. This command can\n" + "\tbe used for checking if a datanode is alive.\n"; @@ -1159,6 +1163,8 @@ public class DFSAdmin extends FsShell { System.out.println(disallowSnapshot); } else if ("shutdownDatanode".equalsIgnoreCase(cmd)) { System.out.println(shutdownDatanode); + } else if ("evictWriters".equalsIgnoreCase(cmd)) { + System.out.println(evictWriters); } else if ("getDatanodeInfo".equalsIgnoreCase(cmd)) { System.out.println(getDatanodeInfo); } else if ("help".equals(cmd)) { @@ -1193,6 +1199,7 @@ public class DFSAdmin extends FsShell { System.out.println(allowSnapshot); System.out.println(disallowSnapshot); System.out.println(shutdownDatanode); + System.out.println(evictWriters); System.out.println(getDatanodeInfo); System.out.println(triggerBlockReport); System.out.println(help); @@ -2047,6 +2054,8 @@ public class DFSAdmin extends FsShell { exitCode = fetchImage(argv, i); } else if ("-shutdownDatanode".equals(cmd)) { exitCode = shutdownDatanode(argv, i); + } else if ("-evictWriters".equals(cmd)) { + exitCode = evictWriters(argv, i); } else if ("-getDatanodeInfo".equals(cmd)) { exitCode = getDatanodeInfo(argv, i); } else if ("-reconfig".equals(cmd)) { @@ -2171,6 +2180,18 @@ public class DFSAdmin extends FsShell { return 0; } + private int evictWriters(String[] argv, int i) throws IOException { + final String dn = argv[i]; + ClientDatanodeProtocol dnProxy = getDataNodeProxy(dn); + try { + dnProxy.evictWriters(); + System.out.println("Requested writer eviction to datanode " + dn); + } catch (IOException ioe) { + return -1; + } + return 0; + } + private int getDatanodeInfo(String[] argv, int i) throws IOException { ClientDatanodeProtocol dnProxy = getDataNodeProxy(argv[i]); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java index 0eeb3b78edf..5e320fa29a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java @@ -271,6 +271,55 @@ public class TestClientProtocolForPipelineRecovery { } } + /** + * Test that the writer is kicked out of a node. + */ + @Test + public void testEvictWriter() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes((int)3) + .build(); + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + Path file = new Path("testEvictWriter.dat"); + FSDataOutputStream out = fs.create(file, (short)2); + out.write(0x31); + out.hflush(); + + // get nodes in the pipeline + DFSOutputStream dfsOut = (DFSOutputStream)out.getWrappedStream(); + DatanodeInfo[] nodes = dfsOut.getPipeline(); + Assert.assertEquals(2, nodes.length); + String dnAddr = nodes[1].getIpcAddr(false); + + // evict the writer from the second datanode and wait until + // the pipeline is rebuilt. + DFSAdmin dfsadmin = new DFSAdmin(conf); + final String[] args1 = {"-evictWriters", dnAddr }; + Assert.assertEquals(0, dfsadmin.run(args1)); + out.write(0x31); + out.hflush(); + + // get the new pipline and check the node is not in there. + nodes = dfsOut.getPipeline(); + try { + Assert.assertTrue(nodes.length > 0 ); + for (int i = 0; i < nodes.length; i++) { + Assert.assertFalse(dnAddr.equals(nodes[i].getIpcAddr(false))); + } + } finally { + out.close(); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** Test restart timeout */ @Test public void testPipelineRecoveryOnRestartFailure() throws Exception { From 93bacda08bc546612f9278b31f5c38107867630a Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 6 Apr 2016 16:13:47 -0700 Subject: [PATCH 69/75] YARN-4769. Add support for CSRF header in the dump capacity scheduler logs and kill app buttons in RM web UI. Contributed by Varun Vasudev --- .../http/RestCsrfPreventionFilter.java | 2 +- .../hadoop/yarn/server/webapp/AppBlock.java | 20 +++++++++++++++++++ .../webapp/CapacitySchedulerPage.java | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java index c0f7e39abd5..33579b42d80 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java @@ -62,7 +62,7 @@ public class RestCsrfPreventionFilter implements Filter { public static final String CUSTOM_METHODS_TO_IGNORE_PARAM = "methods-to-ignore"; static final String BROWSER_USER_AGENTS_DEFAULT = "^Mozilla.*,^Opera.*"; - static final String HEADER_DEFAULT = "X-XSRF-HEADER"; + public static final String HEADER_DEFAULT = "X-XSRF-HEADER"; static final String METHODS_TO_IGNORE_DEFAULT = "GET,OPTIONS,HEAD,TRACE"; private String headerName = HEADER_DEFAULT; private Set methodsToIgnore = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 44ed22345da..69beef27ae6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -24,12 +24,14 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE; import java.security.PrivilegedExceptionAction; import java.util.Collection; +import java.util.Map; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.http.RestCsrfPreventionFilter; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -143,6 +145,7 @@ public class AppBlock extends HtmlBlock { .append(" type: 'PUT',") .append(" url: '/ws/v1/cluster/apps/").append(aid).append("/state',") .append(" contentType: 'application/json',") + .append(getCSRFHeaderString(conf)) .append(" data: '{\"state\":\"KILLED\"}',") .append(" dataType: 'json'") .append(" }).done(function(data){") @@ -369,4 +372,21 @@ public class AppBlock extends HtmlBlock { protected LogAggregationStatus getLogAggregationStatus() { return null; } + + public static String getCSRFHeaderString(Configuration conf) { + String ret = ""; + if (conf.getBoolean(YarnConfiguration.RM_CSRF_ENABLED, false)) { + ret = " headers : { '"; + Map filterParams = RestCsrfPreventionFilter + .getFilterParams(conf, YarnConfiguration.RM_CSRF_PREFIX); + if (filterParams + .containsKey(RestCsrfPreventionFilter.CUSTOM_HEADER_PARAM)) { + ret += filterParams.get(RestCsrfPreventionFilter.CUSTOM_HEADER_PARAM); + } else { + ret += RestCsrfPreventionFilter.HEADER_DEFAULT; + } + ret += "' : 'null' },"; + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 5abc2503d14..bfa081faac9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionQueueCa import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionResourcesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.webapp.AppBlock; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.ResponseInfo; @@ -357,6 +358,7 @@ class CapacitySchedulerPage extends RmView { .append(" type: 'POST',") .append(" url: '/ws/v1/cluster/scheduler/logs',") .append(" contentType: 'text/plain',") + .append(AppBlock.getCSRFHeaderString(rm.getConfig())) .append(" data: 'time=' + timePeriod,") .append(" dataType: 'text'") .append(" }).done(function(data){") From 3be1ab485f557c8a3c6a5066453f24d8d61f30be Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Wed, 6 Apr 2016 17:15:43 -0700 Subject: [PATCH 70/75] MAPREDUCE-6647. MR usage counters use the resources requested instead of the resources allocated (haibochen via rkanter) --- .../v2/app/job/impl/TaskAttemptImpl.java | 41 +++++++++++-------- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 10 ++++- .../hadoop/mapreduce/v2/app/TestRecovery.java | 29 +++++++------ .../v2/app/job/impl/TestTaskAttempt.java | 37 +++++++++++------ 4 files changed, 74 insertions(+), 43 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 8fff7de5a19..5f0a622ec44 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1406,29 +1406,36 @@ public abstract class TaskAttemptImpl implements private static void updateMillisCounters(JobCounterUpdateEvent jce, TaskAttemptImpl taskAttempt) { - TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); + // if container/resource if not allocated, do not update + if (null == taskAttempt.container || + null == taskAttempt.container.getResource()) { + return; + } long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime()); - int mbRequired = - taskAttempt.getMemoryRequired(taskAttempt.conf, taskType); - int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType); - + Resource allocatedResource = taskAttempt.container.getResource(); + int mbAllocated = allocatedResource.getMemory(); + int vcoresAllocated = allocatedResource.getVirtualCores(); int minSlotMemSize = taskAttempt.conf.getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - - int simSlotsRequired = - minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired - / minSlotMemSize); + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int simSlotsAllocated = minSlotMemSize == 0 ? 0 : + (int) Math.ceil((float) mbAllocated / minSlotMemSize); + TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); if (taskType == TaskType.MAP) { - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration); - jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired); - jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, + simSlotsAllocated * duration); + jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbAllocated); + jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, + duration * vcoresAllocated); jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration); } else { - jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration); - jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired); - jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, + simSlotsAllocated * duration); + jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, + duration * mbAllocated); + jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, + duration * vcoresAllocated); jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index fd0e8e2b9b6..b43a7b43f00 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -111,6 +111,10 @@ import org.junit.Assert; public class MRApp extends MRAppMaster { private static final Log LOG = LogFactory.getLog(MRApp.class); + /** + * The available resource of each container allocated. + */ + private Resource resource; int maps; int reduces; @@ -250,6 +254,7 @@ public class MRApp extends MRAppMaster { // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.set(unregistered); this.assignedQueue = assignedQueue; + this.resource = Resource.newInstance(1234, 2); } @Override @@ -589,7 +594,6 @@ public class MRApp extends MRAppMaster { ContainerId.newContainerId(getContext().getApplicationAttemptId(), containerCount++); NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT); - Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), "user", resource, System.currentTimeMillis() + 10000, 42, 42, @@ -712,6 +716,10 @@ public class MRApp extends MRAppMaster { } } + public void setAllocatedContainerResource(Resource resource) { + this.resource = resource; + } + class TestJob extends JobImpl { //override the init transition private final TestInitTransition initTransition = new TestInitTransition( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 504a5f7c71d..0f112381810 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -1744,19 +1744,24 @@ public class TestRecovery { expectedJobHistoryEvents.remove(0); } else if (current instanceof JobCounterUpdateEvent) { JobCounterUpdateEvent jcue = (JobCounterUpdateEvent) current; - - LOG.info("JobCounterUpdateEvent " - + jcue.getCounterUpdates().get(0).getCounterKey() - + " " + jcue.getCounterUpdates().get(0).getIncrementValue()); - if (jcue.getCounterUpdates().get(0).getCounterKey() == - JobCounter.NUM_FAILED_MAPS) { - totalFailedMaps += jcue.getCounterUpdates().get(0) - .getIncrementValue(); - } else if (jcue.getCounterUpdates().get(0).getCounterKey() == - JobCounter.TOTAL_LAUNCHED_MAPS) { - totalLaunchedMaps += jcue.getCounterUpdates().get(0) - .getIncrementValue(); + boolean containsUpdates = jcue.getCounterUpdates().size() > 0; + // there is no updates in a JobCounterUpdateEvent emitted on + // TaskAttempt recovery. Check that first. + if(containsUpdates) { + LOG.info("JobCounterUpdateEvent " + + jcue.getCounterUpdates().get(0).getCounterKey() + + " " + jcue.getCounterUpdates().get(0).getIncrementValue()); + if (jcue.getCounterUpdates().get(0).getCounterKey() == + JobCounter.NUM_FAILED_MAPS) { + totalFailedMaps += jcue.getCounterUpdates().get(0) + .getIncrementValue(); + } else if (jcue.getCounterUpdates().get(0).getCounterKey() == + JobCounter.TOTAL_LAUNCHED_MAPS) { + totalLaunchedMaps += jcue.getCounterUpdates().get(0) + .getIncrementValue(); + } } + } else if (current instanceof JobTaskEvent) { JobTaskEvent jte = (JobTaskEvent) current; assertEquals(jte.getState(), finalState); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 5a660617661..509f6af6129 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -33,6 +33,12 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.apache.hadoop.mapreduce.jobhistory.NormalizedResourceEvent; +import org.apache.hadoop.mapreduce.v2.app.client.ClientService; +import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; +import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -41,9 +47,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapTaskAttemptImpl; +import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobCounter; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletion; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; @@ -82,6 +90,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Event; @@ -250,22 +259,21 @@ public class TestTaskAttempt{ @Test public void testMillisCountersUpdate() throws Exception { - verifyMillisCounters(2048, 2048, 1024); - verifyMillisCounters(2048, 1024, 1024); - verifyMillisCounters(10240, 1024, 2048); + verifyMillisCounters(Resource.newInstance(1024, 1), 512); + verifyMillisCounters(Resource.newInstance(2048, 4), 1024); + verifyMillisCounters(Resource.newInstance(10240, 8), 2048); } - public void verifyMillisCounters(int mapMemMb, int reduceMemMb, + public void verifyMillisCounters(Resource containerResource, int minContainerSize) throws Exception { Clock actualClock = SystemClock.getInstance(); ControlledClock clock = new ControlledClock(actualClock); clock.setTime(10); MRApp app = new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock); + app.setAllocatedContainerResource(containerResource); Configuration conf = new Configuration(); - conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb); - conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minContainerSize); app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1))); @@ -300,21 +308,24 @@ public class TestTaskAttempt{ Assert.assertEquals(rta.getFinishTime(), 11); Assert.assertEquals(rta.getLaunchTime(), 10); Counters counters = job.getAllCounters(); - Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize), + + int memoryMb = containerResource.getMemory(); + int vcores = containerResource.getVirtualCores(); + Assert.assertEquals((int) Math.ceil((float) memoryMb / minContainerSize), counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue()); - Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize), + Assert.assertEquals((int) Math.ceil((float) memoryMb / minContainerSize), counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue()); Assert.assertEquals(1, counters.findCounter(JobCounter.MILLIS_MAPS).getValue()); Assert.assertEquals(1, counters.findCounter(JobCounter.MILLIS_REDUCES).getValue()); - Assert.assertEquals(mapMemMb, + Assert.assertEquals(memoryMb, counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue()); - Assert.assertEquals(reduceMemMb, + Assert.assertEquals(memoryMb, counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue()); - Assert.assertEquals(1, + Assert.assertEquals(vcores, counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue()); - Assert.assertEquals(1, + Assert.assertEquals(vcores, counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue()); } From 4bd7cbc29d142fc56324156333b9a8a7d7b68042 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Wed, 6 Apr 2016 12:36:54 -0700 Subject: [PATCH 71/75] HDFS-10267. Extra "synchronized" on FsDatasetImpl#recoverAppend and FsDatasetImpl#recoverClose --- .../fsdataset/impl/FsDatasetImpl.java | 4 +- .../server/datanode/TestBlockRecovery.java | 236 +++++++++++++----- 2 files changed, 181 insertions(+), 59 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 240345c654b..7e4e8eb3b77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1268,7 +1268,7 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized ReplicaHandler recoverAppend( + public ReplicaHandler recoverAppend( ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed append to " + b); @@ -1301,7 +1301,7 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized Replica recoverClose(ExtendedBlock b, long newGS, + public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed close " + b); while (true) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 751089f2822..42e80fa8f68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -44,8 +44,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.Iterators; import org.apache.commons.logging.Log; @@ -90,6 +92,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; @@ -161,7 +164,7 @@ public class TestBlockRecovery { } private final long - TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS = 1000000000L; + TEST_STOP_WORKER_XCEIVER_STOP_TIMEOUT_MILLIS = 1000000000L; /** * Starts an instance of DataNode @@ -175,11 +178,10 @@ public class TestBlockRecovery { conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0"); - if (currentTestName.getMethodName().equals( - "testInitReplicaRecoveryDoesNotHogLock")) { + if (currentTestName.getMethodName().contains("DoesNotHoldLock")) { // This test requires a very long value for the xceiver stop timeout. conf.setLong(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, - TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS); + TEST_STOP_WORKER_XCEIVER_STOP_TIMEOUT_MILLIS); } conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); FileSystem.setDefaultUri(conf, @@ -816,96 +818,216 @@ public class TestBlockRecovery { } } - /** - * Test that initReplicaRecovery does not hold the lock for an unreasonable - * amount of time if a writer is taking a long time to stop. - */ - @Test(timeout=60000) - public void testInitReplicaRecoveryDoesNotHogLock() throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("Running " + GenericTestUtils.getMethodName()); + private static class TestStopWorkerSemaphore { + final Semaphore sem; + + final AtomicBoolean gotInterruption = new AtomicBoolean(false); + + TestStopWorkerSemaphore() { + this.sem = new Semaphore(0); } + + /** + * Attempt to acquire a sempahore within a given timeout. + * + * This is useful for unit tests where we need to ignore InterruptedException + * when attempting to take a semaphore, but still want to honor the overall + * test timeout. + * + * @param timeoutMs The timeout in miliseconds. + */ + private void uninterruptiblyAcquire(long timeoutMs) throws Exception { + long startTimeMs = Time.monotonicNow(); + while (true) { + long remTime = startTimeMs + timeoutMs - Time.monotonicNow(); + if (remTime < 0) { + throw new RuntimeException("Failed to acquire the semaphore within " + + timeoutMs + " milliseconds."); + } + try { + if (sem.tryAcquire(1, remTime, TimeUnit.MILLISECONDS)) { + return; + } + } catch (InterruptedException e) { + gotInterruption.set(true); + } + } + } + } + + private interface TestStopWorkerRunnable { + /** + * Return the name of the operation that this runnable performs. + */ + String opName(); + + /** + * Perform the operation. + */ + void run(RecoveringBlock recoveringBlock) throws Exception; + } + + @Test(timeout=90000) + public void testInitReplicaRecoveryDoesNotHoldLock() throws Exception { + testStopWorker(new TestStopWorkerRunnable() { + @Override + public String opName() { + return "initReplicaRecovery"; + } + + @Override + public void run(RecoveringBlock recoveringBlock) throws Exception { + try { + spyDN.initReplicaRecovery(recoveringBlock); + } catch (Exception e) { + if (!e.getMessage().contains("meta does not exist")) { + throw e; + } + } + } + }); + } + + @Test(timeout=90000) + public void testRecoverAppendDoesNotHoldLock() throws Exception { + testStopWorker(new TestStopWorkerRunnable() { + @Override + public String opName() { + return "recoverAppend"; + } + + @Override + public void run(RecoveringBlock recoveringBlock) throws Exception { + try { + ExtendedBlock extBlock = recoveringBlock.getBlock(); + spyDN.getFSDataset().recoverAppend(extBlock, + extBlock.getGenerationStamp() + 1, extBlock.getNumBytes()); + } catch (Exception e) { + if (!e.getMessage().contains( + "Corrupted replica ReplicaBeingWritten")) { + throw e; + } + } + } + }); + } + + @Test(timeout=90000) + public void testRecoverCloseDoesNotHoldLock() throws Exception { + testStopWorker(new TestStopWorkerRunnable() { + @Override + public String opName() { + return "recoverClose"; + } + + @Override + public void run(RecoveringBlock recoveringBlock) throws Exception { + try { + ExtendedBlock extBlock = recoveringBlock.getBlock(); + spyDN.getFSDataset().recoverClose(extBlock, + extBlock.getGenerationStamp() + 1, extBlock.getNumBytes()); + } catch (Exception e) { + if (!e.getMessage().contains( + "Corrupted replica ReplicaBeingWritten")) { + throw e; + } + } + } + }); + } + + /** + * Test that an FsDatasetImpl operation does not hold the lock for an + * unreasonable amount of time if a writer is taking a long time to stop. + */ + private void testStopWorker(final TestStopWorkerRunnable tswr) + throws Exception { + LOG.debug("Running " + currentTestName.getMethodName()); // We need a long value for the data xceiver stop timeout. // Otherwise the timeout will trigger, and we will not have tested that // thread join was done locklessly. Assert.assertEquals( - TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS, + TEST_STOP_WORKER_XCEIVER_STOP_TIMEOUT_MILLIS, dn.getDnConf().getXceiverStopTimeout()); - final Semaphore progressParent = new Semaphore(0); - final Semaphore terminateSlowWorker = new Semaphore(0); - final AtomicBoolean failure = new AtomicBoolean(false); + final TestStopWorkerSemaphore progressParent = + new TestStopWorkerSemaphore(); + final TestStopWorkerSemaphore terminateSlowWriter = + new TestStopWorkerSemaphore(); + final AtomicReference failure = + new AtomicReference(null); Collection recoveringBlocks = initRecoveringBlocks(); final RecoveringBlock recoveringBlock = Iterators.get(recoveringBlocks.iterator(), 0); final ExtendedBlock block = recoveringBlock.getBlock(); - Thread slowWorker = new Thread(new Runnable() { + Thread slowWriterThread = new Thread(new Runnable() { @Override public void run() { try { // Register this thread as the writer for the recoveringBlock. - LOG.debug("slowWorker creating rbw"); + LOG.debug("slowWriter creating rbw"); ReplicaHandler replicaHandler = spyDN.data.createRbw(StorageType.DISK, block, false); replicaHandler.close(); - LOG.debug("slowWorker created rbw"); + LOG.debug("slowWriter created rbw"); // Tell the parent thread to start progressing. - progressParent.release(); - while (true) { - try { - terminateSlowWorker.acquire(); - break; - } catch (InterruptedException e) { - // Ignore interrupted exceptions so that the waitingWorker thread - // will have to wait for us. - } - } - LOG.debug("slowWorker exiting"); + progressParent.sem.release(); + terminateSlowWriter.uninterruptiblyAcquire(60000); + LOG.debug("slowWriter exiting"); } catch (Throwable t) { - LOG.error("slowWorker got exception", t); - failure.set(true); + LOG.error("slowWriter got exception", t); + failure.compareAndSet(null, "slowWriter got exception " + + t.getMessage()); } } }); // Start the slow worker thread and wait for it to take ownership of the // ReplicaInPipeline - slowWorker.start(); - while (true) { - try { - progressParent.acquire(); - break; - } catch (InterruptedException e) { - // Ignore interrupted exceptions - } - } + slowWriterThread.start(); + progressParent.uninterruptiblyAcquire(60000); - // Start a worker thread which will wait for the slow worker thread. - Thread waitingWorker = new Thread(new Runnable() { + // Start a worker thread which will attempt to stop the writer. + Thread stopWriterThread = new Thread(new Runnable() { @Override public void run() { try { - // Attempt to terminate the other worker thread and take ownership - // of the ReplicaInPipeline. - LOG.debug("waitingWorker initiating recovery"); - spyDN.initReplicaRecovery(recoveringBlock); - LOG.debug("waitingWorker initiated recovery"); + LOG.debug("initiating " + tswr.opName()); + tswr.run(recoveringBlock); + LOG.debug("finished " + tswr.opName()); } catch (Throwable t) { - GenericTestUtils.assertExceptionContains("meta does not exist", t); + LOG.error("stopWriterThread got unexpected exception for " + + tswr.opName(), t); + failure.compareAndSet(null, "stopWriterThread got unexpected " + + "exception for " + tswr.opName() + ": " + t.getMessage()); } } }); - waitingWorker.start(); + stopWriterThread.start(); - // Do an operation that requires the lock. This should not be blocked - // by the replica recovery in progress. + while (!terminateSlowWriter.gotInterruption.get()) { + // Wait until stopWriterThread attempts to stop our slow writer by sending + // it an InterruptedException. + Thread.sleep(1); + } + + // We know that stopWriterThread is in the process of joining our slow + // writer. It must not hold the lock during this operation. + // In order to test that it does not, we attempt to do an operation that + // requires the lock-- getReplicaString. spyDN.getFSDataset().getReplicaString( recoveringBlock.getBlock().getBlockPoolId(), recoveringBlock.getBlock().getBlockId()); - // Wait for the two worker threads to exit normally. - terminateSlowWorker.release(); - slowWorker.join(); - waitingWorker.join(); - Assert.assertFalse("The slowWriter thread failed.", failure.get()); + // Tell the slow writer to exit, and then wait for all threads to join. + terminateSlowWriter.sem.release(); + slowWriterThread.join(); + stopWriterThread.join(); + + // Check that our worker threads exited cleanly. This is not checked by the + // unit test framework, so we have to do it manually here. + String failureReason = failure.get(); + if (failureReason != null) { + Assert.fail("Thread failure: " + failureReason); + } } } From 654cd1d0c0427c23e73804fc9d87208f76bbf6aa Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 7 Apr 2016 12:36:29 +0800 Subject: [PATCH 72/75] HDFS-10186. DirectoryScanner: Improve logs by adding full path of both actual and expected block directories. Contributed by Rakesh R --- .../hdfs/server/datanode/DirectoryScanner.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 083ca31da30..0e51cecf2b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -902,8 +902,7 @@ public class DirectoryScanner implements Runnable { break; } } - verifyFileLocation(blockFile.getParentFile(), bpFinalizedDir, - blockId); + verifyFileLocation(blockFile, bpFinalizedDir, blockId); report.add(new ScanInfo(blockId, blockFile, metaFile, vol)); } return report; @@ -913,12 +912,15 @@ public class DirectoryScanner implements Runnable { * Verify whether the actual directory location of block file has the * expected directory path computed using its block ID. */ - private void verifyFileLocation(File actualBlockDir, + private void verifyFileLocation(File actualBlockFile, File bpFinalizedDir, long blockId) { File blockDir = DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId); - if (actualBlockDir.compareTo(blockDir) != 0) { + if (actualBlockFile.getParentFile().compareTo(blockDir) != 0) { + File expBlockFile = new File(blockDir, actualBlockFile.getName()); LOG.warn("Block: " + blockId - + " has to be upgraded to block ID-based layout"); + + " has to be upgraded to block ID-based layout. " + + "Actual block file path: " + actualBlockFile + + ", expected block file path: " + expBlockFile); } } From 8d29e2451f5ca60f864c7ece16722c0abdd1c657 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Thu, 7 Apr 2016 10:12:00 +0530 Subject: [PATCH 73/75] HADOOP-12984. Add GenericTestUtils.getTestDir method and use it for emporary directory in tests (Contributed by Steve Loughran and Vinayakumar B This closes #89 --- .../apache/hadoop/conf/TestConfiguration.java | 8 +- .../crypto/TestCryptoStreamsForLocalFS.java | 5 +- .../hadoop/crypto/key/TestKeyShell.java | 5 +- .../org/apache/hadoop/fs/FSTestWrapper.java | 3 +- .../fs/FileContextMainOperationsBaseTest.java | 4 +- .../hadoop/fs/FileContextTestHelper.java | 6 +- .../apache/hadoop/fs/FileContextURIBase.java | 6 +- .../hadoop/fs/FileSystemTestHelper.java | 4 +- .../org/apache/hadoop/fs/TestAvroFSInput.java | 9 +- .../hadoop/fs/TestChecksumFileSystem.java | 5 +- .../apache/hadoop/fs/TestDFVariations.java | 2 +- .../java/org/apache/hadoop/fs/TestDU.java | 4 +- .../hadoop/fs/TestFileContextResolveAfs.java | 8 +- .../org/apache/hadoop/fs/TestFileUtil.java | 9 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 6 +- .../hadoop/fs/TestFsShellReturnCode.java | 8 +- .../apache/hadoop/fs/TestFsShellTouch.java | 6 +- .../hadoop/fs/TestGetFileBlockLocations.java | 5 +- .../hadoop/fs/TestHarFileSystemBasics.java | 5 +- .../org/apache/hadoop/fs/TestHardLink.java | 5 +- .../org/apache/hadoop/fs/TestListFiles.java | 14 ++-- .../apache/hadoop/fs/TestLocalFileSystem.java | 7 +- .../fs/TestLocalFileSystemPermission.java | 18 ++-- .../java/org/apache/hadoop/fs/TestPath.java | 6 +- .../java/org/apache/hadoop/fs/TestTrash.java | 6 +- .../hadoop/fs/TestTruncatedInputBug.java | 4 +- .../hadoop/fs/sftp/TestSFTPFileSystem.java | 3 +- .../apache/hadoop/fs/shell/TestPathData.java | 5 +- .../hadoop/fs/shell/TestTextCommand.java | 4 +- .../fs/viewfs/TestViewfsFileStatus.java | 7 +- .../apache/hadoop/ha/ClientBaseWithFixes.java | 4 +- .../http/TestAuthenticationSessionCookie.java | 5 +- .../hadoop/http/TestHttpCookieFlag.java | 5 +- .../hadoop/http/TestHttpServerLifecycle.java | 5 +- .../apache/hadoop/http/TestSSLHttpServer.java | 5 +- .../org/apache/hadoop/io/TestArrayFile.java | 6 +- .../apache/hadoop/io/TestBloomMapFile.java | 6 +- .../org/apache/hadoop/io/TestMapFile.java | 6 +- .../apache/hadoop/io/TestSequenceFile.java | 48 +++++------ .../hadoop/io/TestSequenceFileAppend.java | 4 +- .../io/TestSequenceFileSerialization.java | 4 +- .../hadoop/io/TestSequenceFileSync.java | 5 +- .../org/apache/hadoop/io/TestSetFile.java | 5 +- .../apache/hadoop/io/compress/TestCodec.java | 23 +++-- .../hadoop/io/file/tfile/TestTFile.java | 4 +- .../io/file/tfile/TestTFileByteArrays.java | 4 +- .../io/file/tfile/TestTFileComparator2.java | 4 +- .../io/file/tfile/TestTFileComparators.java | 5 +- .../hadoop/io/file/tfile/TestTFileSeek.java | 4 +- .../tfile/TestTFileSeqFileComparison.java | 5 +- .../hadoop/io/file/tfile/TestTFileSplit.java | 4 +- .../io/file/tfile/TestTFileStreams.java | 4 +- .../tfile/TestTFileUnsortedByteArrays.java | 5 +- .../hadoop/io/file/tfile/TestVLong.java | 4 +- .../hadoop/io/nativeio/TestNativeIO.java | 10 +-- .../TestSharedFileDescriptorFactory.java | 4 +- .../sink/RollingFileSystemSinkTestBase.java | 6 +- .../hadoop/security/TestCredentials.java | 4 +- .../security/TestLdapGroupsMapping.java | 7 +- .../hadoop/security/alias/TestCredShell.java | 4 +- .../alias/TestCredentialProviderFactory.java | 5 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 4 +- .../ssl/TestReloadingX509TrustManager.java | 6 +- .../hadoop/security/ssl/TestSSLFactory.java | 6 +- .../apache/hadoop/test/GenericTestUtils.java | 84 ++++++++++++++++++- .../org/apache/hadoop/util/JarFinder.java | 4 +- .../util/TestApplicationClassLoader.java | 4 +- .../org/apache/hadoop/util/TestClasspath.java | 5 +- .../hadoop/util/TestGenericOptionsParser.java | 2 +- .../hadoop/util/TestHostsFileReader.java | 4 +- .../org/apache/hadoop/util/TestJarFinder.java | 9 +- .../org/apache/hadoop/util/TestRunJar.java | 8 +- .../org/apache/hadoop/util/TestShell.java | 4 +- .../apache/hadoop/util/TestSysInfoLinux.java | 5 +- .../org/apache/hadoop/util/TestWinUtils.java | 5 +- .../org/apache/hadoop/util/TestZKUtil.java | 5 +- 76 files changed, 314 insertions(+), 242 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 46b2e508a33..aa3b2398ecf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -49,6 +49,8 @@ import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.GenericTestUtils; + import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import org.codehaus.jackson.map.ObjectMapper; @@ -427,8 +429,7 @@ public class TestConfiguration extends TestCase { Configuration conf = new Configuration(); String[] dirs = new String[]{"a", "b", "c"}; for (int i = 0; i < dirs.length; i++) { - dirs[i] = new Path(System.getProperty("test.build.data"), dirs[i]) - .toString(); + dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString(); } conf.set("dirs", StringUtils.join(dirs, ",")); for (int i = 0; i < 1000; i++) { @@ -444,8 +445,7 @@ public class TestConfiguration extends TestCase { Configuration conf = new Configuration(); String[] dirs = new String[]{"a", "b", "c"}; for (int i = 0; i < dirs.length; i++) { - dirs[i] = new Path(System.getProperty("test.build.data"), dirs[i]) - .toString(); + dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString(); } conf.set("dirs", StringUtils.join(dirs, ",")); for (int i = 0; i < 1000; i++) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java index 0a65085e167..235e9c2dc9f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -38,8 +39,8 @@ import org.junit.Ignore; import org.junit.Test; public class TestCryptoStreamsForLocalFS extends CryptoStreamsTestBase { - private static final String TEST_ROOT_DIR - = System.getProperty("test.build.data","build/test/data") + "/work-dir/localfs"; + private static final String TEST_ROOT_DIR = + GenericTestUtils.getTempPath("work-dir/testcryptostreamsforlocalfs"); private final File base = new File(TEST_ROOT_DIR); private final Path file = new Path(TEST_ROOT_DIR, "test-file"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java index fe887182a23..9e4a9ad5052 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java @@ -25,6 +25,7 @@ import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -47,8 +48,8 @@ public class TestKeyShell { public void setup() throws Exception { outContent.reset(); errContent.reset(); - final File tmpDir = new File(System.getProperty("test.build.data", "target"), - UUID.randomUUID().toString()); + final File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID() + .toString()); if (!tmpDir.mkdirs()) { throw new IOException("Unable to create " + tmpDir); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java index c9e7e3cf197..da071050eb0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.fs.Options.CreateOpts; +import org.apache.hadoop.test.GenericTestUtils; /** * Abstraction of filesystem functionality with additional helper methods @@ -43,7 +44,7 @@ public abstract class FSTestWrapper implements FSWrapper { public FSTestWrapper(String testRootDir) { // Use default test dir if not provided if (testRootDir == null || testRootDir.isEmpty()) { - testRootDir = System.getProperty("test.build.data", "build/test/data"); + testRootDir = GenericTestUtils.getTestDir().getAbsolutePath(); } // salt test dir with some random digits for safe parallel runs this.testRootDir = testRootDir + "/" diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 0c736a6b0ed..5f201eb2674 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -29,6 +29,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import static org.junit.Assert.*; @@ -99,8 +100,7 @@ public abstract class FileContextMainOperationsBaseTest { @Before public void setUp() throws Exception { - File testBuildData = new File(System.getProperty("test.build.data", - "build/test/data"), RandomStringUtils.randomAlphanumeric(10)); + File testBuildData = GenericTestUtils.getRandomizedTestDir(); Path rootPath = new Path(testBuildData.getAbsolutePath(), "root-uri"); localFsRootPath = rootPath.makeQualified(LocalFileSystem.NAME, null); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java index 677aa037f03..5bf842cb671 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java @@ -26,6 +26,7 @@ import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.CreateOpts.BlockSize; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; /** @@ -39,11 +40,10 @@ public final class FileContextTestHelper { private String absTestRootDir = null; /** - * Create a context with test root relative to the /build/test/data + * Create a context with test root relative to the test directory */ public FileContextTestHelper() { - this(System.getProperty("test.build.data", "target/test/data") + "/" + - RandomStringUtils.randomAlphanumeric(10)); + this(GenericTestUtils.getRandomizedTestDir().getAbsolutePath()); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java index 4082737289e..0a6ba651213 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.ArrayList; import java.util.regex.Pattern; + +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.apache.hadoop.fs.permission.FsPermission; @@ -51,8 +53,8 @@ import static org.apache.hadoop.fs.FileContextTestHelper.*; *

      */ public abstract class FileContextURIBase { - private static final String basePath = System.getProperty("test.build.data", - "build/test/data") + "/testContextURI"; + private static final String basePath = + GenericTestUtils.getTempPath("testContextURI"); private static final Path BASE = new Path(basePath); // Matches anything containing <, >, :, ", |, ?, *, or anything that ends with diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index 4a88c51ccec..c9d17819caa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -22,9 +22,9 @@ import java.io.FileNotFoundException; import java.net.URI; import java.util.Random; -import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import static org.junit.Assert.*; @@ -45,7 +45,7 @@ public class FileSystemTestHelper { * Create helper with test root located at /build/test/data */ public FileSystemTestHelper() { - this(System.getProperty("test.build.data", "target/test/data") + "/" + RandomStringUtils.randomAlphanumeric(10)); + this(GenericTestUtils.getRandomizedTempPath()); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAvroFSInput.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAvroFSInput.java index 3e5970d228a..4009a601d73 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAvroFSInput.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAvroFSInput.java @@ -22,6 +22,7 @@ import java.io.BufferedWriter; import java.io.OutputStreamWriter; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import junit.framework.TestCase; @@ -30,15 +31,9 @@ public class TestAvroFSInput extends TestCase { private static final String INPUT_DIR = "AvroFSInput"; private Path getInputPath() { - String dataDir = System.getProperty("test.build.data"); - if (null == dataDir) { - return new Path(INPUT_DIR); - } else { - return new Path(new Path(dataDir), INPUT_DIR); - } + return new Path(GenericTestUtils.getTempPath(INPUT_DIR)); } - public void testAFSInput() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java index 923d21967c0..9b7175eb992 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java @@ -22,12 +22,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import static org.apache.hadoop.fs.FileSystemTestHelper.*; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.*; import static org.junit.Assert.*; public class TestChecksumFileSystem { - static final String TEST_ROOT_DIR - = System.getProperty("test.build.data","build/test/data/work-dir/localfs"); + static final String TEST_ROOT_DIR = + GenericTestUtils.getTempPath("work-dir/localfs"); static LocalFileSystem localFs; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java index 97dbe5e6069..3476f3eef43 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java @@ -37,7 +37,7 @@ import static org.junit.Assert.*; public class TestDFVariations { private static final String TEST_ROOT_DIR = - System.getProperty("test.build.data","build/test/data") + "/TestDFVariations"; + GenericTestUtils.getTestDir("testdfvariations").getAbsolutePath(); private static File test_root = null; @Before diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java index 3add2a70b16..dded9fbd120 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java @@ -26,11 +26,11 @@ import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.test.GenericTestUtils; /** This test makes sure that "DU" does not get to run on each call to getUsed */ public class TestDU extends TestCase { - final static private File DU_DIR = new File( - System.getProperty("test.build.data","/tmp"), "dutmp"); + final static private File DU_DIR = GenericTestUtils.getTestDir("dutmp"); @Override public void setUp() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java index 1fe82e958fd..96fac57518b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -34,9 +35,10 @@ public class TestFileContextResolveAfs { static { FileSystem.enableSymlinks(); } - private static String TEST_ROOT_DIR_LOCAL - = System.getProperty("test.build.data","/tmp"); - + + private static String TEST_ROOT_DIR_LOCAL = + GenericTestUtils.getTestDir().getAbsolutePath(); + private FileContext fc; private FileSystem localFs; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index a9ef5c04280..2116a4b7ca7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.*; import java.io.File; @@ -55,9 +56,7 @@ import static org.mockito.Mockito.when; public class TestFileUtil { private static final Log LOG = LogFactory.getLog(TestFileUtil.class); - private static final String TEST_ROOT_DIR = System.getProperty( - "test.build.data", "/tmp") + "/fu"; - private static final File TEST_DIR = new File(TEST_ROOT_DIR); + private static final File TEST_DIR = GenericTestUtils.getTestDir("fu"); private static final String FILE = "x"; private static final String LINK = "y"; private static final String DIR = "dir"; @@ -962,10 +961,10 @@ public class TestFileUtil { @Test (timeout = 30000) public void testUntar() throws IOException { String tarGzFileName = System.getProperty("test.cache.data", - "build/test/cache") + "/test-untar.tgz"; + "target/test/cache") + "/test-untar.tgz"; String tarFileName = System.getProperty("test.cache.data", "build/test/cache") + "/test-untar.tar"; - String dataDir = System.getProperty("test.build.data", "build/test/data"); + File dataDir = GenericTestUtils.getTestDir(); File untarDir = new File(dataDir, "untarDir"); doUntarAndVerify(new File(tarGzFileName), untarDir); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index c00ed66d6fa..9e199ca7f84 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -32,6 +32,7 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -50,9 +51,8 @@ public class TestFsShellCopy { conf = new Configuration(); shell = new FsShell(conf); lfs = FileSystem.getLocal(conf); - testRootDir = lfs.makeQualified(new Path( - System.getProperty("test.build.data","test/build/data"), - "testFsShellCopy")); + testRootDir = lfs.makeQualified(new Path(GenericTestUtils.getTempPath( + "testFsShellCopy"))); lfs.mkdirs(testRootDir); lfs.setWorkingDirectory(testRootDir); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index a2276ec37eb..38ad65b0447 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -41,6 +41,8 @@ import org.apache.hadoop.fs.shell.FsCommand; import org.apache.hadoop.fs.shell.PathData; import org.apache.hadoop.io.IOUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; + +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.BeforeClass; import org.junit.Test; @@ -63,9 +65,9 @@ public class TestFsShellReturnCode { fileSys = FileSystem.get(conf); fsShell = new FsShell(conf); } - - private static String TEST_ROOT_DIR = System.getProperty("test.build.data", - "build/test/data/testCHReturnCode"); + + private static String TEST_ROOT_DIR = + GenericTestUtils.getTempPath("testCHReturnCode"); static void writeFile(FileSystem fs, Path name) throws Exception { FSDataOutputStream stm = fs.create(name); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java index 988a57c2950..89c886ef62c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertThat; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -41,9 +42,8 @@ public class TestFsShellTouch { Configuration conf = new Configuration(); shell = new FsShell(conf); lfs = FileSystem.getLocal(conf); - testRootDir = lfs.makeQualified(new Path( - System.getProperty("test.build.data","test/build/data"), - "testFsShell")); + testRootDir = lfs.makeQualified( + new Path(GenericTestUtils.getTempPath("testFsShell"))); lfs.mkdirs(testRootDir); lfs.setWorkingDirectory(testRootDir); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetFileBlockLocations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetFileBlockLocations.java index ccf98ac44ed..87265f4f4be 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetFileBlockLocations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetFileBlockLocations.java @@ -25,13 +25,14 @@ import java.util.Random; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; /** * Testing the correctness of FileSystem.getFileBlockLocations. */ public class TestGetFileBlockLocations extends TestCase { - private static String TEST_ROOT_DIR = - System.getProperty("test.build.data", "/tmp/testGetFileBlockLocations"); + private static String TEST_ROOT_DIR = GenericTestUtils.getTempPath( + "testGetFileBlockLocations"); private static final int FileLength = 4 * 1024 * 1024; // 4MB private Configuration conf; private Path path; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java index 53507b932a1..c58e731b82b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Assert; @@ -45,8 +46,8 @@ import static org.junit.Assert.*; */ public class TestHarFileSystemBasics { - private static final String ROOT_PATH = System.getProperty("test.build.data", - "build/test/data"); + private static final String ROOT_PATH = + GenericTestUtils.getTempPath("testharfilesystembasics"); private static final Path rootPath; static { String root = new Path(new File(ROOT_PATH).getAbsolutePath(), "localfs") diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java index c68861c55cf..b32b95ea3e8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java @@ -24,6 +24,7 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import static org.junit.Assert.*; import org.junit.Before; @@ -57,9 +58,7 @@ import static org.apache.hadoop.fs.HardLink.*; */ public class TestHardLink { - public static final String TEST_ROOT_DIR = - System.getProperty("test.build.data", "build/test/data") + "/test"; - final static private File TEST_DIR = new File(TEST_ROOT_DIR, "hl"); + final static private File TEST_DIR = GenericTestUtils.getTestDir("test/hl"); private static String DIR = "dir_"; //define source and target directories private static File src = new File(TEST_DIR, DIR + "src"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java index df519c84e8c..010754fe50a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java @@ -22,11 +22,8 @@ import java.util.HashSet; import java.util.Random; import java.util.Set; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import static org.junit.Assert.*; @@ -37,8 +34,8 @@ import org.junit.BeforeClass; * This class tests the FileStatus API. */ public class TestListFiles { - { - ((Log4JLogger)FileSystem.LOG).getLogger().setLevel(Level.ALL); + static { + GenericTestUtils.setLogLevel(FileSystem.LOG, Level.ALL); } static final long seed = 0xDEADBEEFL; @@ -53,9 +50,8 @@ public class TestListFiles { private static Path FILE3; static { - setTestPaths(new Path( - System.getProperty("test.build.data", "build/test/data/work-dir/localfs"), - "main_")); + setTestPaths(new Path(GenericTestUtils.getTempPath("testlistfiles"), + "main_")); } protected static Path getTestDir() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 912c4f43e02..3aadd2fb546 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; @@ -45,10 +46,10 @@ import org.mockito.internal.util.reflection.Whitebox; * This class tests the local file system via the FileSystem abstraction. */ public class TestLocalFileSystem { - private static final String TEST_ROOT_DIR - = System.getProperty("test.build.data","build/test/data") + "/work-dir/localfs"; + private static final File base = + GenericTestUtils.getTestDir("work-dir/localfs"); - private final File base = new File(TEST_ROOT_DIR); + private static final String TEST_ROOT_DIR = base.getAbsolutePath(); private final Path TEST_PATH = new Path(TEST_ROOT_DIR, "test-file"); private Configuration conf; private LocalFileSystem fileSys; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java index 148cf3e1036..e37de1957bb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java @@ -19,7 +19,9 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.*; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; +import org.apache.log4j.Level; import org.apache.hadoop.util.Shell; import java.io.*; @@ -31,19 +33,11 @@ import junit.framework.*; * This class tests the local file system via the FileSystem abstraction. */ public class TestLocalFileSystemPermission extends TestCase { - static final String TEST_PATH_PREFIX = new Path(System.getProperty( - "test.build.data", "/tmp")).toString().replace(' ', '_') - + "/" + TestLocalFileSystemPermission.class.getSimpleName() + "_"; + static final String TEST_PATH_PREFIX = GenericTestUtils.getTempPath( + TestLocalFileSystemPermission.class.getSimpleName()); - { - try { - ((org.apache.commons.logging.impl.Log4JLogger)FileSystem.LOG).getLogger() - .setLevel(org.apache.log4j.Level.DEBUG); - } - catch(Exception e) { - System.out.println("Cannot change log level\n" - + StringUtils.stringifyException(e)); - } + static { + GenericTestUtils.setLogLevel(FileSystem.LOG, Level.DEBUG); } private Path writeFile(FileSystem fs, String name) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index 94908da7a38..e5b22f95a38 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -26,6 +26,7 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.AvroTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import com.google.common.base.Joiner; @@ -402,9 +403,8 @@ public class TestPath extends TestCase { // This test is not meaningful on Windows where * is disallowed in file name. if (Shell.WINDOWS) return; FileSystem lfs = FileSystem.getLocal(new Configuration()); - Path testRoot = lfs.makeQualified(new Path( - System.getProperty("test.build.data","test/build/data"), - "testPathGlob")); + Path testRoot = lfs.makeQualified( + new Path(GenericTestUtils.getTempPath("testPathGlob"))); lfs.delete(testRoot, true); lfs.mkdirs(testRoot); assertTrue(lfs.isDirectory(testRoot)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index 88194fd2887..338aff6e8d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -34,6 +34,7 @@ import java.util.Set; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; /** @@ -41,9 +42,8 @@ import org.apache.hadoop.util.Time; */ public class TestTrash extends TestCase { - private final static Path TEST_DIR = - new Path(new File(System.getProperty("test.build.data","/tmp") - ).toURI().toString().replace(' ', '+'), "testTrash"); + private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath( + "testTrash")); protected static Path mkdir(FileSystem fs, Path p) throws IOException { assertTrue(fs.mkdirs(p)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTruncatedInputBug.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTruncatedInputBug.java index e7dabf903cd..41c4d477512 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTruncatedInputBug.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTruncatedInputBug.java @@ -23,6 +23,7 @@ import java.io.IOException; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; /** * test for the input truncation bug when mark/reset is used. @@ -30,8 +31,7 @@ import org.apache.hadoop.conf.Configuration; */ public class TestTruncatedInputBug extends TestCase { private static String TEST_ROOT_DIR = - new Path(System.getProperty("test.build.data","/tmp")) - .toString().replace(' ', '+'); + GenericTestUtils.getTestDir().getAbsolutePath(); private void writeFile(FileSystem fileSys, Path name, int nBytesToWrite) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java index 06d9bf04257..36aaceed11c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.apache.sshd.SshServer; @@ -54,7 +55,7 @@ public class TestSFTPFileSystem { private static final String TEST_SFTP_DIR = "testsftp"; private static final String TEST_ROOT_DIR = - System.getProperty("test.build.data", "build/test/data"); + GenericTestUtils.getTestDir().getAbsolutePath(); @Rule public TestName name = new TestName(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java index ea9e9847fd3..e3e574afe45 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java @@ -29,14 +29,15 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Before; import org.junit.Test; public class TestPathData { - private static final String TEST_ROOT_DIR = - System.getProperty("test.build.data","build/test/data") + "/testPD"; + private static final String TEST_ROOT_DIR = + GenericTestUtils.getTestDir("testPD").getAbsolutePath(); protected Configuration conf; protected FileSystem fs; protected Path testDir; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java index 0e33d6ab647..7b848363720 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java @@ -33,6 +33,7 @@ import java.nio.file.Paths; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; /** @@ -41,8 +42,7 @@ import org.junit.Test; */ public class TestTextCommand { private static final File TEST_ROOT_DIR = - Paths.get(System.getProperty("test.build.data", "build/test/data"), - "testText").toFile(); + GenericTestUtils.getTestDir("testText"); private static final String AVRO_FILENAME = new File(TEST_ROOT_DIR, "weather.avro").toURI().getPath(); private static final String TEXT_FILENAME = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java index 9eec749336f..be1caec9d85 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java @@ -30,7 +30,7 @@ import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; - +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Test; import org.mockito.Mockito; @@ -44,9 +44,8 @@ import static org.junit.Assert.*; */ public class TestViewfsFileStatus { - private static final File TEST_DIR = - new File(System.getProperty("test.build.data", "/tmp"), - TestViewfsFileStatus.class.getSimpleName()); + private static final File TEST_DIR = GenericTestUtils.getTestDir( + TestViewfsFileStatus.class.getSimpleName()); @Test public void testFileStatusSerialziation() diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java index b1ce1d152db..7396694ce95 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.net.ServerSocketUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.zookeeper.TestableZooKeeper; import org.apache.zookeeper.WatchedEvent; @@ -62,8 +63,7 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { protected static final Logger LOG = LoggerFactory.getLogger(ClientBaseWithFixes.class); public static int CONNECTION_TIMEOUT = 30000; - static final File BASETEST = - new File(System.getProperty("test.build.data", "build")); + static final File BASETEST = GenericTestUtils.getTestDir(); protected final String hostPort = initHostPort(); protected int maxCnxns = 0; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestAuthenticationSessionCookie.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestAuthenticationSessionCookie.java index 058633a1eee..c51f1e84145 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestAuthenticationSessionCookie.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestAuthenticationSessionCookie.java @@ -19,6 +19,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; import org.mortbay.log.Log; @@ -35,8 +36,8 @@ import java.net.HttpCookie; import java.util.List; public class TestAuthenticationSessionCookie { - private static final String BASEDIR = System.getProperty("test.build.dir", - "target/test-dir") + "/" + TestHttpCookieFlag.class.getSimpleName(); + private static final String BASEDIR = + GenericTestUtils.getTempPath(TestHttpCookieFlag.class.getSimpleName()); private static boolean isCookiePersistent; private static final long TOKEN_VALIDITY_SEC = 1000; private static long expires; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java index f93f6632c69..8a722dff88d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java @@ -20,6 +20,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -37,8 +38,8 @@ import java.net.HttpCookie; import java.util.List; public class TestHttpCookieFlag { - private static final String BASEDIR = System.getProperty("test.build.dir", - "target/test-dir") + "/" + TestHttpCookieFlag.class.getSimpleName(); + private static final String BASEDIR = + GenericTestUtils.getTempPath(TestHttpCookieFlag.class.getSimpleName()); private static String keystoresDir; private static String sslConfDir; private static SSLFactory clientSslFactory; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java index edae3c25cb2..40f1b3df08c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.http; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -76,8 +77,8 @@ public class TestHttpServerLifecycle extends HttpServerFunctionalTest { public void testStartedServerWithRequestLog() throws Throwable { HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); requestLogAppender.setName("httprequestlog"); - requestLogAppender.setFilename(System.getProperty("test.build.data", "/tmp/") - + "jetty-name-yyyy_mm_dd.log"); + requestLogAppender.setFilename( + GenericTestUtils.getTempPath("jetty-name-yyyy_mm_dd.log")); Logger.getLogger(HttpServer2.class.getName() + ".test").addAppender(requestLogAppender); HttpServer2 server = null; server = createTestServer(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java index c5ecfa17905..49d73d65e13 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -51,8 +52,8 @@ import org.junit.Test; */ public class TestSSLHttpServer extends HttpServerFunctionalTest { - private static final String BASEDIR = System.getProperty("test.build.dir", - "target/test-dir") + "/" + TestSSLHttpServer.class.getSimpleName(); + private static final String BASEDIR = + GenericTestUtils.getTempPath(TestSSLHttpServer.class.getSimpleName()); private static final Log LOG = LogFactory.getLog(TestSSLHttpServer.class); private static Configuration conf; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestArrayFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestArrayFile.java index 8c8da4a1eb5..505aca736c2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestArrayFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestArrayFile.java @@ -24,6 +24,7 @@ import java.io.*; import org.apache.commons.logging.*; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.conf.*; import org.junit.Test; @@ -38,9 +39,8 @@ import static org.junit.Assert.fail; public class TestArrayFile { private static final Log LOG = LogFactory.getLog(TestArrayFile.class); - private static final Path TEST_DIR = new Path( - System.getProperty("test.build.data", "/tmp"), - TestMapFile.class.getSimpleName()); + private static final Path TEST_DIR = new Path(GenericTestUtils.getTempPath( + TestMapFile.class.getSimpleName())); private static String TEST_FILE = new Path(TEST_DIR, "test.array").toString(); @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java index 55a91884e19..740540d5d23 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java @@ -38,6 +38,7 @@ import org.apache.hadoop.io.compress.CompressionInputStream; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Progressable; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -49,9 +50,8 @@ import org.junit.Test; public class TestBloomMapFile { private static Configuration conf = new Configuration(); - private static final Path TEST_ROOT = new Path( - System.getProperty("test.build.data", "/tmp"), - TestMapFile.class.getSimpleName()); + private static final Path TEST_ROOT = new Path(GenericTestUtils.getTempPath( + TestMapFile.class.getSimpleName())); private static final Path TEST_DIR = new Path(TEST_ROOT, "testfile"); private static final Path TEST_FILE = new Path(TEST_ROOT, "testfile"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java index 3f14de09bc5..ff8df7cf063 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java @@ -37,6 +37,7 @@ import org.apache.hadoop.io.compress.CompressionInputStream; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Progressable; import org.junit.Assert; import org.junit.Before; @@ -48,9 +49,8 @@ import static org.mockito.Mockito.*; public class TestMapFile { - private static final Path TEST_DIR = new Path( - System.getProperty("test.build.data", "/tmp"), - TestMapFile.class.getSimpleName()); + private static final Path TEST_DIR = new Path(GenericTestUtils.getTempPath( + TestMapFile.class.getSimpleName())); private static Configuration conf = new Configuration(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java index 4cb4e130692..b76cff6663e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.SequenceFile.Metadata; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.conf.*; import org.junit.Test; @@ -58,11 +59,11 @@ public class TestSequenceFile { int count = 1024 * 10; int megabytes = 1; int factor = 5; - Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq"); - Path recordCompressedFile = - new Path(System.getProperty("test.build.data",".")+"/test.rc.seq"); - Path blockCompressedFile = - new Path(System.getProperty("test.build.data",".")+"/test.bc.seq"); + Path file = new Path(GenericTestUtils.getTempPath("test.seq")); + Path recordCompressedFile = new Path(GenericTestUtils.getTempPath( + "test.rc.seq")); + Path blockCompressedFile = new Path(GenericTestUtils.getTempPath( + "test.bc.seq")); int seed = new Random().nextInt(); LOG.info("Seed = " + seed); @@ -320,13 +321,13 @@ public class TestSequenceFile { LOG.info("Testing SequenceFile with metadata"); int count = 1024 * 10; CompressionCodec codec = new DefaultCodec(); - Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata"); - Path sortedFile = - new Path(System.getProperty("test.build.data",".")+"/test.sorted.seq.metadata"); - Path recordCompressedFile = - new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata"); - Path blockCompressedFile = - new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata"); + Path file = new Path(GenericTestUtils.getTempPath("test.seq.metadata")); + Path sortedFile = new Path(GenericTestUtils.getTempPath( + "test.sorted.seq.metadata")); + Path recordCompressedFile = new Path(GenericTestUtils.getTempPath( + "test.rc.seq.metadata")); + Path blockCompressedFile = new Path(GenericTestUtils.getTempPath( + "test.bc.seq.metadata")); FileSystem fs = FileSystem.getLocal(conf); SequenceFile.Metadata theMetadata = new SequenceFile.Metadata(); @@ -426,14 +427,14 @@ public class TestSequenceFile { LocalFileSystem fs = FileSystem.getLocal(conf); // create a sequence file 1 - Path path1 = new Path(System.getProperty("test.build.data",".")+"/test1.seq"); + Path path1 = new Path(GenericTestUtils.getTempPath("test1.seq")); SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path1, Text.class, NullWritable.class, CompressionType.BLOCK); writer.append(new Text("file1-1"), NullWritable.get()); writer.append(new Text("file1-2"), NullWritable.get()); writer.close(); - Path path2 = new Path(System.getProperty("test.build.data",".")+"/test2.seq"); + Path path2 = new Path(GenericTestUtils.getTempPath("test2.seq")); writer = SequenceFile.createWriter(fs, conf, path2, Text.class, NullWritable.class, CompressionType.BLOCK); writer.append(new Text("file2-1"), NullWritable.get()); @@ -482,7 +483,7 @@ public class TestSequenceFile { public void testCreateUsesFsArg() throws Exception { FileSystem fs = FileSystem.getLocal(conf); FileSystem spyFs = Mockito.spy(fs); - Path p = new Path(System.getProperty("test.build.data", ".")+"/testCreateUsesFSArg.seq"); + Path p = new Path(GenericTestUtils.getTempPath("testCreateUsesFSArg.seq")); SequenceFile.Writer writer = SequenceFile.createWriter( spyFs, conf, p, NullWritable.class, NullWritable.class); writer.close(); @@ -515,7 +516,7 @@ public class TestSequenceFile { LocalFileSystem fs = FileSystem.getLocal(conf); // create an empty file (which is not a valid sequence file) - Path path = new Path(System.getProperty("test.build.data",".")+"/broken.seq"); + Path path = new Path(GenericTestUtils.getTempPath("broken.seq")); fs.create(path).close(); // try to create SequenceFile.Reader @@ -547,8 +548,7 @@ public class TestSequenceFile { LocalFileSystem fs = FileSystem.getLocal(conf); // create an empty file (which is not a valid sequence file) - Path path = new Path(System.getProperty("test.build.data", ".") + - "/zerolength.seq"); + Path path = new Path(GenericTestUtils.getTempPath("zerolength.seq")); fs.create(path).close(); try { @@ -569,8 +569,8 @@ public class TestSequenceFile { public void testCreateWriterOnExistingFile() throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); - Path name = new Path(new Path(System.getProperty("test.build.data","."), - "createWriterOnExistingFile") , "file"); + Path name = new Path(new Path(GenericTestUtils.getTempPath( + "createWriterOnExistingFile")), "file"); fs.create(name); SequenceFile.createWriter(fs, conf, name, RandomDatum.class, @@ -582,8 +582,8 @@ public class TestSequenceFile { @Test public void testRecursiveSeqFileCreate() throws IOException { FileSystem fs = FileSystem.getLocal(conf); - Path name = new Path(new Path(System.getProperty("test.build.data","."), - "recursiveCreateDir") , "file"); + Path name = new Path(new Path(GenericTestUtils.getTempPath( + "recursiveCreateDir")), "file"); boolean createParent = false; try { @@ -605,8 +605,8 @@ public class TestSequenceFile { @Test public void testSerializationAvailability() throws IOException { Configuration conf = new Configuration(); - Path path = new Path(System.getProperty("test.build.data", "."), - "serializationAvailability"); + Path path = new Path(GenericTestUtils.getTempPath( + "serializationAvailability")); // Check if any serializers aren't found. try { SequenceFile.createWriter( diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java index be4ab921a25..9e49c9e86e4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java @@ -43,8 +43,8 @@ public class TestSequenceFileAppend { private static Configuration conf; private static FileSystem fs; - private static Path ROOT_PATH = new Path(System.getProperty( - "test.build.data", "build/test/data")); + private static Path ROOT_PATH = + new Path(GenericTestUtils.getTestDir().getAbsolutePath()); @BeforeClass public static void setUp() throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java index 3ca9187f895..b1c519a7085 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile.Reader; import org.apache.hadoop.io.SequenceFile.Writer; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,8 +51,7 @@ public class TestSequenceFileSerialization { @Test public void testJavaSerialization() throws Exception { - Path file = new Path(System.getProperty("test.build.data",".") + - "/testseqser.seq"); + Path file = new Path(GenericTestUtils.getTempPath("testseqser.seq")); fs.delete(file, true); Writer writer = SequenceFile.createWriter(fs, conf, file, Long.class, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java index fb094252699..bceb8aff2e1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSync.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; public class TestSequenceFileSync { @@ -52,8 +53,8 @@ public class TestSequenceFileSync { public void testLowSyncpoint() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf); - final Path path = new Path(System.getProperty("test.build.data", "/tmp"), - "sequencefile.sync.test"); + final Path path = new Path(GenericTestUtils.getTempPath( + "sequencefile.sync.test")); final IntWritable input = new IntWritable(); final Text val = new Text(); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSetFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSetFile.java index ff92b371ca6..1fcfab673c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSetFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSetFile.java @@ -20,7 +20,6 @@ package org.apache.hadoop.io; import java.io.*; import java.util.*; -import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.*; @@ -28,6 +27,7 @@ import org.apache.commons.logging.*; import org.apache.hadoop.fs.*; import org.apache.hadoop.conf.*; import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import static org.junit.Assert.assertTrue; @@ -38,8 +38,7 @@ import static org.junit.Assert.fail; /** Support for flat files of binary key/value pairs. */ public class TestSetFile { private static final Log LOG = LogFactory.getLog(TestSetFile.class); - private static String FILE = - System.getProperty("test.build.data",".") + "/test.set"; + private static String FILE = GenericTestUtils.getTempPath("test.set"); private static Configuration conf = new Configuration(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 1e3809ea0e7..5443ca768b4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -72,6 +72,7 @@ import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; import org.apache.hadoop.io.compress.zlib.ZlibCompressor; import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionLevel; import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionStrategy; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.util.LineReader; import org.apache.hadoop.util.NativeCodeLoader; @@ -338,9 +339,9 @@ public class TestCodec { private static Path writeSplitTestFile(FileSystem fs, Random rand, CompressionCodec codec, long infLen) throws IOException { final int REC_SIZE = 1024; - final Path wd = new Path(new Path( - System.getProperty("test.build.data", "/tmp")).makeQualified(fs), - codec.getClass().getSimpleName()); + final Path wd = new Path(GenericTestUtils.getTempPath( + codec.getClass().getSimpleName())).makeQualified( + fs.getUri(), fs.getWorkingDirectory()); final Path file = new Path(wd, "test" + codec.getDefaultExtension()); final byte[] b = new byte[REC_SIZE]; final Base64 b64 = new Base64(0, null); @@ -596,9 +597,8 @@ public class TestCodec { FileSystem fs = FileSystem.get(conf); LOG.info("Creating MapFiles with " + records + " records using codec " + clazz.getSimpleName()); - Path path = new Path(new Path( - System.getProperty("test.build.data", "/tmp")), - clazz.getSimpleName() + "-" + type + "-" + records); + Path path = new Path(GenericTestUtils.getTempPath( + clazz.getSimpleName() + "-" + type + "-" + records)); LOG.info("Writing " + path); createMapFile(conf, fs, path, clazz.newInstance(), type, records); @@ -750,8 +750,7 @@ public class TestCodec { CodecPool.returnDecompressor(zlibDecompressor); // Now create a GZip text file. - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz"); + Path f = new Path(GenericTestUtils.getTempPath("testGzipCodecRead.txt.gz")); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new GZIPOutputStream(new FileOutputStream(f.toString())))); final String msg = "This is the message in the file!"; @@ -802,8 +801,7 @@ public class TestCodec { CodecPool.returnDecompressor(zlibDecompressor); // Now create a GZip text file. - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz"); + Path f = new Path(GenericTestUtils.getTempPath("testGzipLongOverflow.bin.gz")); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new GZIPOutputStream(new FileOutputStream(f.toString())))); @@ -862,9 +860,8 @@ public class TestCodec { codec instanceof GzipCodec); final String msg = "This is the message we are going to compress."; - final String tmpDir = System.getProperty("test.build.data", "/tmp/"); - final String fileName = new Path(new Path(tmpDir), - "testGzipCodecWrite.txt.gz").toString(); + final String fileName = new Path(GenericTestUtils.getTempPath( + "testGzipCodecWrite.txt.gz")).toString(); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.getCompressor(codec); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFile.java index df3f48dec2d..80aeef2d636 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFile.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; import org.junit.After; import org.junit.Before; @@ -43,8 +44,7 @@ import static org.junit.Assert.assertFalse; * */ public class TestTFile { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTempPath("tfile-test"); private FileSystem fs; private Configuration conf; private static final int minBlockSize = 512; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java index 8040cdf8a98..7051f002134 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java @@ -35,6 +35,7 @@ import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; import org.apache.hadoop.io.file.tfile.TFile.Reader.Location; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -46,8 +47,7 @@ import org.junit.Test; * */ public class TestTFileByteArrays { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private final static int BLOCK_SIZE = 512; private final static int BUF_SIZE = 64; private final static int K = 1024; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparator2.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparator2.java index 9e13c71403c..5a8b5b30fd4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparator2.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparator2.java @@ -27,13 +27,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.file.tfile.TFile.Writer; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import static org.junit.Assert.*; public class TestTFileComparator2 { - private static final String ROOT = System.getProperty("test.build.data", - "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private static final String name = "test-tfile-comparator2"; private final static int BLOCK_SIZE = 512; private static final String VALUE = "value"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparators.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparators.java index 198000b22b1..e46006296f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparators.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileComparators.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.file.tfile.TFile.Writer; +import org.apache.hadoop.test.GenericTestUtils; /** * @@ -38,9 +39,7 @@ import org.apache.hadoop.io.file.tfile.TFile.Writer; * */ public class TestTFileComparators { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); - + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private final static int BLOCK_SIZE = 512; private FileSystem fs; private Configuration conf; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeek.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeek.java index b362c4204f2..f1eaec23000 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeek.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeek.java @@ -45,6 +45,7 @@ import org.apache.hadoop.io.file.tfile.RandomDistribution.DiscreteRNG; import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; +import org.apache.hadoop.test.GenericTestUtils; /** * test the performance for seek. @@ -246,8 +247,7 @@ public class TestTFileSeek { int fsOutputBufferSizeLzo = 1; int fsOutputBufferSizeGz = 1; - String rootDir = - System.getProperty("test.build.data", "/tmp/tfile-test"); + String rootDir = GenericTestUtils.getTestDir().getAbsolutePath(); String file = "TestTFileSeek"; String compress = "gz"; int minKeyLen = 10; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java index 5dd6d029a9e..48924203f34 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java @@ -45,6 +45,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner.Entry; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; public class TestTFileSeqFileComparison { @@ -515,9 +516,7 @@ public class TestTFileSeqFileComparison { } private static class MyOptions { - String rootDir = - System - .getProperty("test.build.data", "/tmp/tfile-test"); + String rootDir = GenericTestUtils.getTestDir().getAbsolutePath();; String compress = "gz"; String format = "tfile"; int dictSize = 1000; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSplit.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSplit.java index 08695d95e6d..613ae4fbcef 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSplit.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSplit.java @@ -32,10 +32,10 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; +import org.apache.hadoop.test.GenericTestUtils; public class TestTFileSplit { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private final static int BLOCK_SIZE = 64 * 1024; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java index 6524c374cd2..a108408f507 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java @@ -37,6 +37,7 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; +import org.apache.hadoop.test.GenericTestUtils; /** * @@ -46,8 +47,7 @@ import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; */ public class TestTFileStreams { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private final static int BLOCK_SIZE = 512; private final static int K = 1024; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileUnsortedByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileUnsortedByteArrays.java index 235e5e477ac..f243b2a9406 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileUnsortedByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileUnsortedByteArrays.java @@ -29,13 +29,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.file.tfile.TFile.Reader; import org.apache.hadoop.io.file.tfile.TFile.Writer; import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; public class TestTFileUnsortedByteArrays { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); - + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private final static int BLOCK_SIZE = 512; private final static int BUF_SIZE = 64; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java index 9efd2717d22..69e6eb87412 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java @@ -29,12 +29,12 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; public class TestVLong { - private static String ROOT = - System.getProperty("test.build.data", "/tmp/tfile-test"); + private static String ROOT = GenericTestUtils.getTestDir().getAbsolutePath(); private Configuration conf; private FileSystem fs; private Path path; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java index 13fdbc17a40..e6f25dc2eea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java @@ -61,8 +61,7 @@ import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat.*; public class TestNativeIO { static final Log LOG = LogFactory.getLog(TestNativeIO.class); - static final File TEST_DIR = new File( - System.getProperty("test.build.data"), "testnativeio"); + static final File TEST_DIR = GenericTestUtils.getTestDir("testnativeio"); @Before public void checkLoaded() { @@ -516,8 +515,7 @@ public class TestNativeIO { @Test (timeout = 30000) public void testRenameTo() throws Exception { - final File TEST_DIR = new File(new File( - System.getProperty("test.build.data","build/test/data")), "renameTest"); + final File TEST_DIR = GenericTestUtils.getTestDir("renameTest") ; assumeTrue(TEST_DIR.mkdirs()); File nonExistentFile = new File(TEST_DIR, "nonexistent"); File targetFile = new File(TEST_DIR, "target"); @@ -566,9 +564,7 @@ public class TestNativeIO { @Test(timeout=10000) public void testMlock() throws Exception { assumeTrue(NativeIO.isAvailable()); - final File TEST_FILE = new File(new File( - System.getProperty("test.build.data","build/test/data")), - "testMlockFile"); + final File TEST_FILE = GenericTestUtils.getTestDir("testMlockFile"); final int BUF_LEN = 12289; byte buf[] = new byte[BUF_LEN]; int bufSum = 0; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java index 899ba4b7b8e..64abecdbf31 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java @@ -31,12 +31,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; public class TestSharedFileDescriptorFactory { static final Log LOG = LogFactory.getLog(TestSharedFileDescriptorFactory.class); - private static final File TEST_BASE = - new File(System.getProperty("test.build.data", "/tmp")); + private static final File TEST_BASE = GenericTestUtils.getTestDir(); @Before public void setup() throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSinkTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSinkTestBase.java index 9914c5e30f3..b65a759686b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSinkTestBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSinkTestBase.java @@ -53,6 +53,7 @@ import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; import org.apache.hadoop.metrics2.impl.TestMetricsConfig; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Rule; import org.junit.rules.TestName; @@ -68,9 +69,8 @@ import static org.junit.Assert.assertTrue; public class RollingFileSystemSinkTestBase { protected static final String SINK_PRINCIPAL_KEY = "rfssink.principal"; protected static final String SINK_KEYTAB_FILE_KEY = "rfssink.keytab"; - protected static final File ROOT_TEST_DIR = - new File(System.getProperty("test.build.data", "target/test"), - "RollingFileSystemSinkTest"); + protected static final File ROOT_TEST_DIR = GenericTestUtils.getTestDir( + "RollingFileSystemSinkTest"); protected static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyyMMddHH"); protected static File methodDir; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java index cad0262a92d..e1e7d1d73fa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -47,8 +48,7 @@ import static org.junit.Assert.*; public class TestCredentials { private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; - private static final File tmpDir = - new File(System.getProperty("test.build.data", "/tmp"), "mapred"); + private static final File tmpDir = GenericTestUtils.getTestDir("mapred"); @Before public void setUp() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index 17a14d18894..da469702703 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -108,8 +109,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase { @Test public void testExtractPassword() throws IOException { - File testDir = new File(System.getProperty("test.build.data", - "target/test-dir")); + File testDir = GenericTestUtils.getTestDir(); testDir.mkdirs(); File secretFile = new File(testDir, "secret.txt"); Writer writer = new FileWriter(secretFile); @@ -123,8 +123,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase { @Test public void testConfGetPassword() throws Exception { - File testDir = new File(System.getProperty("test.build.data", - "target/test-dir")); + File testDir = GenericTestUtils.getTestDir(); Configuration conf = new Configuration(); final Path jksPath = new Path(testDir.toString(), "test.jks"); final String ourUrl = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java index 7551df67116..f4541fc195d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java @@ -30,14 +30,14 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; public class TestCredShell { private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); - private static final File tmpDir = - new File(System.getProperty("test.build.data", "/tmp"), "creds"); + private static final File tmpDir = GenericTestUtils.getTestDir("creds"); /* The default JCEKS provider - for testing purposes */ private String jceksProvider; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java index 73cf3f4da2a..567adbbbc07 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java @@ -34,7 +34,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.UserGroupInformation; - +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -61,8 +61,7 @@ public class TestCredentialProviderFactory { 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '2', '3', '4', '5', '6', '7', '8', '9',}; - private static final File tmpDir = - new File(System.getProperty("test.build.data", "/tmp"), "creds"); + private static final File tmpDir = GenericTestUtils.getTestDir("creds"); @Test public void testFactory() throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java index 00cd1cbe6e8..898c94ec33b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.test.GenericTestUtils; import java.io.File; import java.io.FileOutputStream; @@ -470,8 +471,7 @@ public class KeyStoreTestUtil { } public static void provisionPasswordsToCredentialProvider() throws Exception { - File testDir = new File(System.getProperty("test.build.data", - "target/test-dir")); + File testDir = GenericTestUtils.getTestDir(); Configuration conf = new Configuration(); final Path jksPath = new Path(testDir.toString(), "test.jks"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java index 75e5a8e57bd..9375da8db17 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509TrustManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security.ssl; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.BeforeClass; import org.junit.Test; @@ -37,9 +38,8 @@ import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.generateKeyPair; public class TestReloadingX509TrustManager { - private static final String BASEDIR = - System.getProperty("test.build.data", "target/test-dir") + "/" + - TestReloadingX509TrustManager.class.getSimpleName(); + private static final String BASEDIR = GenericTestUtils.getTempPath( + TestReloadingX509TrustManager.class.getSimpleName()); private X509Certificate cert1; private X509Certificate cert2; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java index b8a09ed92e7..f0c502ebe12 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -52,8 +52,7 @@ public class TestSSLFactory { private static final Logger LOG = LoggerFactory .getLogger(TestSSLFactory.class); private static final String BASEDIR = - System.getProperty("test.build.dir", "target/test-dir") + "/" + - TestSSLFactory.class.getSimpleName(); + GenericTestUtils.getTempPath(TestSSLFactory.class.getSimpleName()); private static final String KEYSTORES_DIR = new File(BASEDIR).getAbsolutePath(); private String sslConfsDir; @@ -433,8 +432,7 @@ public class TestSSLFactory { sslConf = KeyStoreTestUtil.createServerSSLConfig(keystore, confPassword, confKeyPassword, truststore); if (useCredProvider) { - File testDir = new File(System.getProperty("test.build.data", - "target/test-dir")); + File testDir = GenericTestUtils.getTestDir(); final Path jksPath = new Path(testDir.toString(), "test.jks"); final String ourUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 190709456ae..116a111cdbc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -20,7 +20,6 @@ package org.apache.hadoop.test; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.StringWriter; @@ -37,10 +36,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Layout; @@ -64,6 +63,22 @@ public abstract class GenericTestUtils { private static final AtomicInteger sequence = new AtomicInteger(); + /** + * system property for test data: {@value} + */ + public static final String SYSPROP_TEST_DATA_DIR = "test.build.data"; + + /** + * Default path for test data: {@value} + */ + public static final String DEFAULT_TEST_DATA_DIR = + "target" + File.separator + "test" + File.separator + "data"; + + /** + * The default path for using in Hadoop path references: {@value} + */ + public static final String DEFAULT_TEST_DATA_PATH = "target/test/data/"; + @SuppressWarnings("unchecked") public static void disableLog(Log log) { // We expect that commons-logging is a wrapper around Log4j. @@ -119,7 +134,70 @@ public abstract class GenericTestUtils { public static int uniqueSequenceId() { return sequence.incrementAndGet(); } - + + /** + * Get the (created) base directory for tests. + * @return the absolute directory + */ + public static File getTestDir() { + String prop = System.getProperty(SYSPROP_TEST_DATA_DIR, DEFAULT_TEST_DATA_DIR); + if (prop.isEmpty()) { + // corner case: property is there but empty + prop = DEFAULT_TEST_DATA_DIR; + } + File dir = new File(prop).getAbsoluteFile(); + dir.mkdirs(); + assertExists(dir); + return dir; + } + + /** + * Get an uncreated directory for tests. + * @return the absolute directory for tests. Caller is expected to create it. + */ + public static File getTestDir(String subdir) { + return new File(getTestDir(), subdir).getAbsoluteFile(); + } + + /** + * Get an uncreated directory for tests with a randomized alphanumeric + * name. This is likely to provide a unique path for tests run in parallel + * @return the absolute directory for tests. Caller is expected to create it. + */ + public static File getRandomizedTestDir() { + return new File(getRandomizedTempPath()).getAbsoluteFile(); + } + + /** + * Get a temp path. This may or may not be relative; it depends on what the + * {@link #SYSPROP_TEST_DATA_DIR} is set to. If unset, it returns a path + * under the relative path {@link #DEFAULT_TEST_DATA_PATH} + * @param subpath sub path, with no leading "/" character + * @return a string to use in paths + */ + public static String getTempPath(String subpath) { + String prop = System.getProperty(SYSPROP_TEST_DATA_DIR, DEFAULT_TEST_DATA_PATH); + if (prop.isEmpty()) { + // corner case: property is there but empty + prop = DEFAULT_TEST_DATA_PATH; + } + if (!prop.endsWith("/")) { + prop = prop + "/"; + } + return prop + subpath; + } + + /** + * Get a temp path. This may or may not be relative; it depends on what the + * {@link #SYSPROP_TEST_DATA_DIR} is set to. If unset, it returns a path + * under the relative path {@link #DEFAULT_TEST_DATA_PATH} + * @param subpath sub path, with no leading "/" character + * @return a string to use in paths + */ + public static String getRandomizedTempPath() { + return getTempPath(RandomStringUtils.randomAlphanumeric(10)); + } + /** * Assert that a given file exists. */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java index 98acd168ef4..33aa02570f9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java @@ -31,6 +31,8 @@ import java.util.jar.Manifest; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.apache.hadoop.test.GenericTestUtils; + /** * Finds the Jar for a class. If the class is in a directory in the * classpath, it creates a Jar on the fly with the contents of the directory @@ -152,7 +154,7 @@ public class JarFinder { klassName = klassName.replace(".", "/") + ".class"; path = path.substring(0, path.length() - klassName.length()); File baseDir = new File(path); - File testDir = new File(System.getProperty("test.build.dir", "target/test-dir")); + File testDir = GenericTestUtils.getTestDir(); testDir = testDir.getAbsoluteFile(); if (!testDir.exists()) { testDir.mkdirs(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java index be8e61ea23a..570e54214ad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java @@ -37,6 +37,7 @@ import java.util.zip.ZipEntry; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; @@ -45,8 +46,7 @@ import com.google.common.collect.Lists; public class TestApplicationClassLoader { - private static File testDir = new File(System.getProperty("test.build.data", - System.getProperty("java.io.tmpdir")), "appclassloader"); + private static File testDir = GenericTestUtils.getTestDir("appclassloader"); @Before public void setUp() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java index 9ffde9030e2..a38c3d764af 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java @@ -32,6 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -42,8 +43,8 @@ import org.junit.Test; public class TestClasspath { private static final Log LOG = LogFactory.getLog(TestClasspath.class); - private static final File TEST_DIR = new File( - System.getProperty("test.build.data", "/tmp"), "TestClasspath"); + private static final File TEST_DIR = GenericTestUtils.getTestDir( + "TestClasspath"); private static final Charset UTF8 = Charset.forName("UTF-8"); static { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java index 2bc19154f40..d5755869ca3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -199,7 +199,7 @@ public class TestGenericOptionsParser extends TestCase { super.setUp(); conf = new Configuration(); localFs = FileSystem.getLocal(conf); - testDir = new File(System.getProperty("test.build.data", "/tmp"), "generic"); + testDir = GenericTestUtils.getTestDir("generic"); if(testDir.exists()) localFs.delete(new Path(testDir.toString()), true); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java index 7de0be859f7..3000069ed22 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileWriter; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.*; import static org.junit.Assert.*; @@ -31,8 +32,7 @@ import static org.junit.Assert.*; public class TestHostsFileReader { // Using /test/build/data/tmp directory to store temprory files - final String HOSTS_TEST_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).getAbsolutePath(); + final String HOSTS_TEST_DIR = GenericTestUtils.getTestDir().getAbsolutePath(); File EXCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.exclude"); File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include"); String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java index 4997b7a8242..e58fb3bffde 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java @@ -19,6 +19,7 @@ package org.apache.hadoop.util; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -79,8 +80,8 @@ public class TestJarFinder { @Test public void testExistingManifest() throws Exception { - File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), - TestJarFinder.class.getName() + "-testExistingManifest"); + File dir = GenericTestUtils + .getTestDir(TestJarFinder.class.getName() + "-testExistingManifest"); delete(dir); dir.mkdirs(); @@ -107,8 +108,8 @@ public class TestJarFinder { @Test public void testNoManifest() throws Exception { - File dir = new File(System.getProperty("test.build.dir", "target/test-dir"), - TestJarFinder.class.getName() + "-testNoManifest"); + File dir = GenericTestUtils + .getTestDir(TestJarFinder.class.getName() + "-testNoManifest"); delete(dir); dir.mkdirs(); File propsFile = new File(dir, "props.properties"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java index f592d0400a4..5b751e22b3c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java @@ -32,6 +32,7 @@ import java.util.zip.ZipEntry; import junit.framework.TestCase; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -44,11 +45,8 @@ public class TestRunJar extends TestCase { @Override @Before - protected void setUp() - throws Exception { - TEST_ROOT_DIR = - new File(System.getProperty("test.build.data", "/tmp"), getClass() - .getSimpleName()); + protected void setUp() throws Exception { + TEST_ROOT_DIR = GenericTestUtils.getTestDir(getClass().getSimpleName()); if (!TEST_ROOT_DIR.exists()) { TEST_ROOT_DIR.mkdirs(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index a9f7f6ddd46..f20c140a818 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -31,6 +31,8 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; + import static org.apache.hadoop.util.Shell.*; import org.junit.Assume; import org.junit.Before; @@ -49,7 +51,7 @@ public class TestShell extends Assert { @Rule public TestName methodName = new TestName(); - private File rootTestDir = new File(System.getProperty("test.build.data", "target/")); + private File rootTestDir = GenericTestUtils.getTestDir(); /** * A filename generated uniquely for each test method. The file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java index 47fef3307d8..8a6df0c0511 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java @@ -25,6 +25,7 @@ import java.util.Random; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -63,8 +64,8 @@ public class TestSysInfoLinux { } } private static final FakeLinuxResourceCalculatorPlugin plugin; - private static String TEST_ROOT_DIR = new Path(System.getProperty( - "test.build.data", "/tmp")).toString().replace(' ', '+'); + private static String TEST_ROOT_DIR = GenericTestUtils.getTestDir() + .getAbsolutePath(); private static final String FAKE_MEMFILE; private static final String FAKE_CPUFILE; private static final String FAKE_STATFILE; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java index fde28227679..cfa97f4a099 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java @@ -32,6 +32,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -44,8 +45,8 @@ import static org.hamcrest.CoreMatchers.*; public class TestWinUtils { private static final Log LOG = LogFactory.getLog(TestWinUtils.class); - private static File TEST_DIR = new File(System.getProperty("test.build.data", - "target"+File.pathSeparator + "tmp"), TestWinUtils.class.getSimpleName()); + private static File TEST_DIR = GenericTestUtils.getTestDir( + TestWinUtils.class.getSimpleName()); String winutils; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java index 52d10ca2fca..0e39ca94dea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java @@ -24,6 +24,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ZKUtil.BadAclFormatException; import org.apache.hadoop.util.ZKUtil.ZKAuthInfo; import org.apache.zookeeper.ZooDefs.Perms; @@ -34,8 +35,8 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; public class TestZKUtil { - private static final String TEST_ROOT_DIR = System.getProperty( - "test.build.data", "/tmp") + "/TestZKUtil"; + private static final String TEST_ROOT_DIR = GenericTestUtils.getTempPath( + "TestZKUtil"); private static final File TEST_FILE = new File(TEST_ROOT_DIR, "test-file"); From 3c18a53cbd2efabb2ad108d63a0b0b558424115f Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Wed, 6 Apr 2016 22:50:24 -0700 Subject: [PATCH 74/75] HDFS-9719. Refactoring ErasureCodingWorker into smaller reusable constructs. Contributed by Kai Zheng. --- .../hadoop/hdfs/util/StripedBlockUtil.java | 22 +- .../erasurecode/ErasureCodingWorker.java | 1018 +---------------- .../erasurecode/StripedBlockReader.java | 202 ++++ .../erasurecode/StripedBlockWriter.java | 196 ++++ .../datanode/erasurecode/StripedReader.java | 466 ++++++++ .../erasurecode/StripedReconstructor.java | 273 +++++ .../datanode/erasurecode/StripedWriter.java | 313 +++++ .../datanode/erasurecode/package-info.java | 26 + .../hdfs/TestReconstructStripedFile.java | 11 +- 9 files changed, 1556 insertions(+), 971 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/package-info.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 0819376aac0..c8827d9b562 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -141,6 +141,12 @@ public class StripedBlockUtil { return locatedBlock; } + public static ExtendedBlock constructInternalBlock( + ExtendedBlock blockGroup, ErasureCodingPolicy ecPolicy, + int idxInBlockGroup) { + return constructInternalBlock(blockGroup, ecPolicy.getCellSize(), + ecPolicy.getNumDataUnits(), idxInBlockGroup); + } /** * This method creates an internal {@link ExtendedBlock} at the given index * of a block group. @@ -154,21 +160,28 @@ public class StripedBlockUtil { return block; } + public static long getInternalBlockLength(long dataSize, + ErasureCodingPolicy ecPolicy, + int idxInBlockGroup) { + return getInternalBlockLength(dataSize, ecPolicy.getCellSize(), + ecPolicy.getNumDataUnits(), idxInBlockGroup); + } + /** * Get the size of an internal block at the given index of a block group * * @param dataSize Size of the block group only counting data blocks * @param cellSize The size of a striping cell * @param numDataBlocks The number of data blocks - * @param i The logical index in the striped block group + * @param idxInBlockGroup The logical index in the striped block group * @return The size of the internal block at the specified index */ public static long getInternalBlockLength(long dataSize, - int cellSize, int numDataBlocks, int i) { + int cellSize, int numDataBlocks, int idxInBlockGroup) { Preconditions.checkArgument(dataSize >= 0); Preconditions.checkArgument(cellSize > 0); Preconditions.checkArgument(numDataBlocks > 0); - Preconditions.checkArgument(i >= 0); + Preconditions.checkArgument(idxInBlockGroup >= 0); // Size of each stripe (only counting data blocks) final int stripeSize = cellSize * numDataBlocks; // If block group ends at stripe boundary, each internal block has an equal @@ -180,7 +193,8 @@ public class StripedBlockUtil { final int numStripes = (int) ((dataSize - 1) / stripeSize + 1); return (numStripes - 1L)*cellSize - + lastCellSize(lastStripeDataLen, cellSize, numDataBlocks, i); + + lastCellSize(lastStripeDataLen, cellSize, + numDataBlocks, idxInBlockGroup); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 4bcb291b288..e7c5abc2bc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -17,169 +17,111 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import java.io.BufferedOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.BitSet; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; +import org.apache.hadoop.util.Daemon; +import org.slf4j.Logger; + import java.util.Collection; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.BlockReader; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSPacket; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks; -import org.apache.hadoop.hdfs.RemoteBlockReader2; -import org.apache.hadoop.hdfs.net.Peer; -import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; -import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; -import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; -import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; -import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory; -import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; -import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; -import org.apache.hadoop.hdfs.util.StripedBlockUtil; -import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.erasurecode.CodecUtil; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.Daemon; -import org.apache.hadoop.util.DataChecksum; - -import com.google.common.base.Preconditions; -import org.slf4j.Logger; - /** * ErasureCodingWorker handles the erasure coding reconstruction work commands. - * These commands would be issued from Namenode as part of Datanode's heart - * beat response. BPOfferService delegates the work to this class for handling - * EC commands. + * These commands would be issued from Namenode as part of Datanode's heart beat + * response. BPOfferService delegates the work to this class for handling EC + * commands. */ @InterfaceAudience.Private public final class ErasureCodingWorker { private static final Logger LOG = DataNode.LOG; - - private final DataNode datanode; + + private final DataNode datanode; private final Configuration conf; - private ThreadPoolExecutor EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL; - private ThreadPoolExecutor EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL; - private final int EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS; - private final int EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE; + private ThreadPoolExecutor stripedReconstructionPool; + private ThreadPoolExecutor stripedReadPool; public ErasureCodingWorker(Configuration conf, DataNode datanode) { this.datanode = datanode; this.conf = conf; - EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS = conf.getInt( - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_KEY, - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT); initializeStripedReadThreadPool(conf.getInt( DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY, DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT)); - EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE = conf.getInt( - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY, - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT); - initializeStripedBlkReconstructionThreadPool(conf.getInt( DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_BLK_THREADS_KEY, DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_BLK_THREADS_DEFAULT)); } - - private RawErasureDecoder newDecoder(int numDataUnits, int numParityUnits) { - return CodecUtil.createRSRawDecoder(conf, numDataUnits, numParityUnits); - } private void initializeStripedReadThreadPool(int num) { LOG.debug("Using striped reads; pool threads={}", num); - EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, - 60, TimeUnit.SECONDS, new SynchronousQueue(), + stripedReadPool = new ThreadPoolExecutor(1, num, 60, TimeUnit.SECONDS, + new SynchronousQueue(), new Daemon.DaemonFactory() { - private final AtomicInteger threadIndex = new AtomicInteger(0); + private final AtomicInteger threadIndex = new AtomicInteger(0); - @Override - public Thread newThread(Runnable r) { - Thread t = super.newThread(r); - t.setName("stripedRead-" + threadIndex.getAndIncrement()); - return t; - } - }, new ThreadPoolExecutor.CallerRunsPolicy() { - @Override - public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) { - LOG.info("Execution for striped reading rejected, " - + "Executing in current thread"); - // will run in the current thread - super.rejectedExecution(runnable, e); - } - }); - EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true); + @Override + public Thread newThread(Runnable r) { + Thread t = super.newThread(r); + t.setName("stripedRead-" + threadIndex.getAndIncrement()); + return t; + } + }, + new ThreadPoolExecutor.CallerRunsPolicy() { + @Override + public void rejectedExecution(Runnable runnable, + ThreadPoolExecutor e) { + LOG.info("Execution for striped reading rejected, " + + "Executing in current thread"); + // will run in the current thread + super.rejectedExecution(runnable, e); + } + }); + + stripedReadPool.allowCoreThreadTimeOut(true); } private void initializeStripedBlkReconstructionThreadPool(int num) { - LOG.debug("Using striped block reconstruction; pool threads={}" + num); - EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL = new ThreadPoolExecutor(2, num, - 60, TimeUnit.SECONDS, new LinkedBlockingQueue(), + LOG.debug("Using striped block reconstruction; pool threads={}", num); + stripedReconstructionPool = new ThreadPoolExecutor(2, num, 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue(), new Daemon.DaemonFactory() { private final AtomicInteger threadIdx = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { Thread t = super.newThread(r); - t.setName( - "stripedBlockReconstruction-" + threadIdx.getAndIncrement()); + t.setName("stripedBlockReconstruction-" + + threadIdx.getAndIncrement()); return t; } }); - EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL.allowCoreThreadTimeOut(true); + stripedReconstructionPool.allowCoreThreadTimeOut(true); } /** * Handles the Erasure Coding reconstruction work commands. * - * @param ecTasks - * BlockECReconstructionInfo + * @param ecTasks BlockECReconstructionInfo + * */ public void processErasureCodingTasks( Collection ecTasks) { for (BlockECReconstructionInfo reconstructionInfo : ecTasks) { try { - ReconstructAndTransferBlock task = - new ReconstructAndTransferBlock(reconstructionInfo); + final StripedReconstructor task = + new StripedReconstructor(this, reconstructionInfo); if (task.hasValidTargets()) { - EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL.submit(task); + stripedReconstructionPool.submit(task); } else { LOG.warn("No missing internal block. Skip reconstruction for task:{}", reconstructionInfo); @@ -191,863 +133,15 @@ public final class ErasureCodingWorker { } } - /** - * ReconstructAndTransferBlock reconstruct one or more missed striped block - * in the striped block group, the minimum number of live striped blocks - * should be no less than data block number. - * - * | <- Striped Block Group -> | - * blk_0 blk_1 blk_2(*) blk_3 ... <- A striped block group - * | | | | - * v v v v - * +------+ +------+ +------+ +------+ - * |cell_0| |cell_1| |cell_2| |cell_3| ... - * +------+ +------+ +------+ +------+ - * |cell_4| |cell_5| |cell_6| |cell_7| ... - * +------+ +------+ +------+ +------+ - * |cell_8| |cell_9| |cell10| |cell11| ... - * +------+ +------+ +------+ +------+ - * ... ... ... ... - * - * - * We use following steps to reconstruct striped block group, in each round, - * we reconstruct bufferSize data until finish, the - * bufferSize is configurable and may be less or larger than - * cell size: - * step1: read bufferSize data from minimum number of sources - * required by reconstruction. - * step2: decode data for targets. - * step3: transfer data to targets. - * - * In step1, try to read bufferSize data from minimum number - * of sources , if there is corrupt or stale sources, read from new source - * will be scheduled. The best sources are remembered for next round and - * may be updated in each round. - * - * In step2, typically if source blocks we read are all data blocks, we - * need to call encode, and if there is one parity block, we need to call - * decode. Notice we only read once and reconstruct all missed striped block - * if they are more than one. - * - * In step3, send the reconstructed data to targets by constructing packet - * and send them directly. Same as continuous block replication, we - * don't check the packet ack. Since the datanode doing the reconstruction - * work are one of the source datanodes, so the reconstructed data are sent - * remotely. - * - * There are some points we can do further improvements in next phase: - * 1. we can read the block file directly on the local datanode, - * currently we use remote block reader. (Notice short-circuit is not - * a good choice, see inline comments). - * 2. We need to check the packet ack for EC reconstruction? Since EC - * reconstruction is more expensive than continuous block replication, - * it needs to read from several other datanodes, should we make sure - * the reconstructed result received by targets? - */ - private class ReconstructAndTransferBlock implements Runnable { - private final int dataBlkNum; - private final int parityBlkNum; - private final int cellSize; - - private RawErasureDecoder decoder; - - // Striped read buffer size - private int bufferSize; - - private final ExtendedBlock blockGroup; - private final int minRequiredSources; - // position in striped internal block - private long positionInBlock; - - // sources - private final byte[] liveIndices; - private final DatanodeInfo[] sources; - - private final List stripedReaders; - - // The buffers and indices for striped blocks whose length is 0 - private ByteBuffer[] zeroStripeBuffers; - private short[] zeroStripeIndices; - - // targets - private final DatanodeInfo[] targets; - private final StorageType[] targetStorageTypes; - - private final short[] targetIndices; - private final ByteBuffer[] targetBuffers; - - private final Socket[] targetSockets; - private final DataOutputStream[] targetOutputStreams; - private final DataInputStream[] targetInputStreams; - - private final long[] blockOffset4Targets; - private final long[] seqNo4Targets; - - private final static int WRITE_PACKET_SIZE = 64 * 1024; - private DataChecksum checksum; - private int maxChunksPerPacket; - private byte[] packetBuf; - private byte[] checksumBuf; - private int bytesPerChecksum; - private int checksumSize; - - private final CachingStrategy cachingStrategy; - - private final Map, Integer> futures = new HashMap<>(); - private final CompletionService readService = - new ExecutorCompletionService<>( - EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL); - private final boolean hasValidTargets; - - ReconstructAndTransferBlock(BlockECReconstructionInfo reconstructionInfo) { - ErasureCodingPolicy ecPolicy = reconstructionInfo - .getErasureCodingPolicy(); - dataBlkNum = ecPolicy.getNumDataUnits(); - parityBlkNum = ecPolicy.getNumParityUnits(); - cellSize = ecPolicy.getCellSize(); - - blockGroup = reconstructionInfo.getExtendedBlock(); - final int cellsNum = (int)((blockGroup.getNumBytes() - 1) / cellSize + 1); - minRequiredSources = Math.min(cellsNum, dataBlkNum); - - liveIndices = reconstructionInfo.getLiveBlockIndices(); - sources = reconstructionInfo.getSourceDnInfos(); - stripedReaders = new ArrayList<>(sources.length); - - Preconditions.checkArgument(liveIndices.length >= minRequiredSources, - "No enough live striped blocks."); - Preconditions.checkArgument(liveIndices.length == sources.length, - "liveBlockIndices and source dns should match"); - - if (minRequiredSources < dataBlkNum) { - zeroStripeBuffers = - new ByteBuffer[dataBlkNum - minRequiredSources]; - zeroStripeIndices = new short[dataBlkNum - minRequiredSources]; - } - - targets = reconstructionInfo.getTargetDnInfos(); - targetStorageTypes = reconstructionInfo.getTargetStorageTypes(); - targetIndices = new short[targets.length]; - targetBuffers = new ByteBuffer[targets.length]; - - Preconditions.checkArgument(targetIndices.length <= parityBlkNum, - "Too much missed striped blocks."); - - targetSockets = new Socket[targets.length]; - targetOutputStreams = new DataOutputStream[targets.length]; - targetInputStreams = new DataInputStream[targets.length]; - - blockOffset4Targets = new long[targets.length]; - seqNo4Targets = new long[targets.length]; - - for (int i = 0; i < targets.length; i++) { - blockOffset4Targets[i] = 0; - seqNo4Targets[i] = 0; - } - - hasValidTargets = getTargetIndices(); - cachingStrategy = CachingStrategy.newDefaultStrategy(); - } - - boolean hasValidTargets() { - return hasValidTargets; - } - - private ByteBuffer allocateBuffer(int length) { - return ByteBuffer.allocate(length); - } - - private ExtendedBlock getBlock(ExtendedBlock blockGroup, int i) { - return StripedBlockUtil.constructInternalBlock(blockGroup, cellSize, - dataBlkNum, i); - } - - private long getBlockLen(ExtendedBlock blockGroup, int i) { - return StripedBlockUtil.getInternalBlockLength(blockGroup.getNumBytes(), - cellSize, dataBlkNum, i); - } - - /** - * StripedReader is used to read from one source DN, it contains a block - * reader, buffer and striped block index. - * Only allocate StripedReader once for one source, and the StripedReader - * has the same array order with sources. Typically we only need to allocate - * minimum number (minRequiredSources) of StripedReader, and allocate - * new for new source DN if some existing DN invalid or slow. - * If some source DN is corrupt, set the corresponding blockReader to - * null and will never read from it again. - * - * @param i the array index of sources - * @param offsetInBlock offset for the internal block - * @return StripedReader - */ - private StripedReader addStripedReader(int i, long offsetInBlock) { - final ExtendedBlock block = getBlock(blockGroup, liveIndices[i]); - StripedReader reader = new StripedReader(liveIndices[i], block, sources[i]); - stripedReaders.add(reader); - - BlockReader blockReader = newBlockReader(block, offsetInBlock, sources[i]); - if (blockReader != null) { - initChecksumAndBufferSizeIfNeeded(blockReader); - reader.blockReader = blockReader; - } - reader.buffer = allocateBuffer(bufferSize); - return reader; - } - - @Override - public void run() { - datanode.incrementXmitsInProgress(); - try { - // Store the array indices of source DNs we have read successfully. - // In each iteration of read, the success list may be updated if - // some source DN is corrupted or slow. And use the updated success - // list of DNs for next iteration read. - int[] success = new int[minRequiredSources]; - - int nsuccess = 0; - for (int i = 0; - i < sources.length && nsuccess < minRequiredSources; i++) { - StripedReader reader = addStripedReader(i, 0); - if (reader.blockReader != null) { - success[nsuccess++] = i; - } - } - - if (nsuccess < minRequiredSources) { - String error = "Can't find minimum sources required by " - + "reconstruction, block id: " + blockGroup.getBlockId(); - throw new IOException(error); - } - - if (zeroStripeBuffers != null) { - for (int i = 0; i < zeroStripeBuffers.length; i++) { - zeroStripeBuffers[i] = allocateBuffer(bufferSize); - } - } - - for (int i = 0; i < targets.length; i++) { - targetBuffers[i] = allocateBuffer(bufferSize); - } - - checksumSize = checksum.getChecksumSize(); - int chunkSize = bytesPerChecksum + checksumSize; - maxChunksPerPacket = Math.max( - (WRITE_PACKET_SIZE - PacketHeader.PKT_MAX_HEADER_LEN)/chunkSize, 1); - int maxPacketSize = chunkSize * maxChunksPerPacket - + PacketHeader.PKT_MAX_HEADER_LEN; - - packetBuf = new byte[maxPacketSize]; - checksumBuf = new byte[checksumSize * (bufferSize / bytesPerChecksum)]; - - // targetsStatus store whether some target is success, it will record - // any failed target once, if some target failed (invalid DN or transfer - // failed), will not transfer data to it any more. - boolean[] targetsStatus = new boolean[targets.length]; - if (initTargetStreams(targetsStatus) == 0) { - String error = "All targets are failed."; - throw new IOException(error); - } - - long maxTargetLength = 0; - for (short targetIndex : targetIndices) { - maxTargetLength = Math.max(maxTargetLength, - getBlockLen(blockGroup, targetIndex)); - } - while (positionInBlock < maxTargetLength) { - final int toReconstruct = (int) Math.min( - bufferSize, maxTargetLength - positionInBlock); - // step1: read from minimum source DNs required for reconstruction. - // The returned success list is the source DNs we do real read from - CorruptedBlocks corruptedBlocks = new CorruptedBlocks(); - try { - success = readMinimumStripedData4Reconstruction(success, - toReconstruct, corruptedBlocks); - } finally { - // report corrupted blocks to NN - datanode.reportCorruptedBlocks(corruptedBlocks); - } - - // step2: decode to reconstruct targets - reconstructTargets(success, targetsStatus, toReconstruct); - - // step3: transfer data - if (transferData2Targets(targetsStatus) == 0) { - String error = "Transfer failed for all targets."; - throw new IOException(error); - } - - clearBuffers(); - positionInBlock += toReconstruct; - } - - endTargetBlocks(targetsStatus); - - // Currently we don't check the acks for packets, this is similar as - // block replication. - } catch (Throwable e) { - LOG.warn("Failed to reconstruct striped block: {}", blockGroup, e); - } finally { - datanode.decrementXmitsInProgress(); - // close block readers - for (StripedReader stripedReader : stripedReaders) { - IOUtils.closeStream(stripedReader.blockReader); - } - for (int i = 0; i < targets.length; i++) { - IOUtils.closeStream(targetOutputStreams[i]); - IOUtils.closeStream(targetInputStreams[i]); - IOUtils.closeStream(targetSockets[i]); - } - } - } - - // init checksum from block reader - private void initChecksumAndBufferSizeIfNeeded(BlockReader blockReader) { - if (checksum == null) { - checksum = blockReader.getDataChecksum(); - bytesPerChecksum = checksum.getBytesPerChecksum(); - // The bufferSize is flat to divide bytesPerChecksum - int readBufferSize = EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE; - bufferSize = readBufferSize < bytesPerChecksum ? bytesPerChecksum : - readBufferSize - readBufferSize % bytesPerChecksum; - } else { - assert blockReader.getDataChecksum().equals(checksum); - } - } - - /** - * @return true if there is valid target for reconstruction - */ - private boolean getTargetIndices() { - BitSet bitset = new BitSet(dataBlkNum + parityBlkNum); - for (int i = 0; i < sources.length; i++) { - bitset.set(liveIndices[i]); - } - int m = 0; - int k = 0; - boolean hasValidTarget = false; - for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { - if (!bitset.get(i)) { - if (getBlockLen(blockGroup, i) > 0) { - if (m < targets.length) { - targetIndices[m++] = (short)i; - hasValidTarget = true; - } - } else { - zeroStripeIndices[k++] = (short)i; - } - } - } - return hasValidTarget; - } - - /** the reading length should not exceed the length for reconstruction. */ - private int getReadLength(int index, int reconstructLength) { - long blockLen = getBlockLen(blockGroup, index); - long remaining = blockLen - positionInBlock; - return (int) Math.min(remaining, reconstructLength); - } - - /** - * Read from minimum source DNs required for reconstruction in the iteration. - * First try the success list which we think they are the best DNs - * If source DN is corrupt or slow, try to read some other source DN, - * and will update the success list. - * - * Remember the updated success list and return it for following - * operations and next iteration read. - * - * @param success the initial success list of source DNs we think best - * @param reconstructLength the length to reconstruct. - * @return updated success list of source DNs we do real read - * @throws IOException - */ - private int[] readMinimumStripedData4Reconstruction(final int[] success, - int reconstructLength, CorruptedBlocks corruptedBlocks) - throws IOException { - Preconditions.checkArgument(reconstructLength >= 0 && - reconstructLength <= bufferSize); - int nsuccess = 0; - int[] newSuccess = new int[minRequiredSources]; - BitSet used = new BitSet(sources.length); - /* - * Read from minimum source DNs required, the success list contains - * source DNs which we think best. - */ - for (int i = 0; i < minRequiredSources; i++) { - StripedReader reader = stripedReaders.get(success[i]); - final int toRead = getReadLength(liveIndices[success[i]], - reconstructLength); - if (toRead > 0) { - Callable readCallable = readFromBlock(reader, reader.buffer, - toRead, corruptedBlocks); - Future f = readService.submit(readCallable); - futures.put(f, success[i]); - } else { - // If the read length is 0, we don't need to do real read - reader.buffer.position(0); - newSuccess[nsuccess++] = success[i]; - } - used.set(success[i]); - } - - while (!futures.isEmpty()) { - try { - StripingChunkReadResult result = StripedBlockUtil - .getNextCompletedStripedRead(readService, futures, - EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS); - int resultIndex = -1; - if (result.state == StripingChunkReadResult.SUCCESSFUL) { - resultIndex = result.index; - } else if (result.state == StripingChunkReadResult.FAILED) { - // If read failed for some source DN, we should not use it anymore - // and schedule read from another source DN. - StripedReader failedReader = stripedReaders.get(result.index); - IOUtils.closeStream(failedReader.blockReader); - failedReader.blockReader = null; - resultIndex = scheduleNewRead(used, reconstructLength, - corruptedBlocks); - } else if (result.state == StripingChunkReadResult.TIMEOUT) { - // If timeout, we also schedule a new read. - resultIndex = scheduleNewRead(used, reconstructLength, - corruptedBlocks); - } - if (resultIndex >= 0) { - newSuccess[nsuccess++] = resultIndex; - if (nsuccess >= minRequiredSources) { - // cancel remaining reads if we read successfully from minimum - // number of source DNs required by reconstruction. - cancelReads(futures.keySet()); - futures.clear(); - break; - } - } - } catch (InterruptedException e) { - LOG.info("Read data interrupted.", e); - cancelReads(futures.keySet()); - futures.clear(); - break; - } - } - - if (nsuccess < minRequiredSources) { - String error = "Can't read data from minimum number of sources " - + "required by reconstruction, block id: " + blockGroup.getBlockId(); - throw new IOException(error); - } - - return newSuccess; - } - - private void paddingBufferToLen(ByteBuffer buffer, int len) { - if (len > buffer.limit()) { - buffer.limit(len); - } - int toPadding = len - buffer.position(); - for (int i = 0; i < toPadding; i++) { - buffer.put((byte) 0); - } - } - - // Initialize decoder - private void initDecoderIfNecessary() { - if (decoder == null) { - decoder = newDecoder(dataBlkNum, parityBlkNum); - } - } - - private int[] getErasedIndices(boolean[] targetsStatus) { - int[] result = new int[targets.length]; - int m = 0; - for (int i = 0; i < targets.length; i++) { - if (targetsStatus[i]) { - result[m++] = targetIndices[i]; - } - } - return Arrays.copyOf(result, m); - } - - private void reconstructTargets(int[] success, boolean[] targetsStatus, - int toReconstructLen) { - initDecoderIfNecessary(); - ByteBuffer[] inputs = new ByteBuffer[dataBlkNum + parityBlkNum]; - for (int i = 0; i < success.length; i++) { - StripedReader reader = stripedReaders.get(success[i]); - ByteBuffer buffer = reader.buffer; - paddingBufferToLen(buffer, toReconstructLen); - inputs[reader.index] = (ByteBuffer)buffer.flip(); - } - if (success.length < dataBlkNum) { - for (int i = 0; i < zeroStripeBuffers.length; i++) { - ByteBuffer buffer = zeroStripeBuffers[i]; - paddingBufferToLen(buffer, toReconstructLen); - int index = zeroStripeIndices[i]; - inputs[index] = (ByteBuffer)buffer.flip(); - } - } - int[] erasedIndices = getErasedIndices(targetsStatus); - ByteBuffer[] outputs = new ByteBuffer[erasedIndices.length]; - int m = 0; - for (int i = 0; i < targetBuffers.length; i++) { - if (targetsStatus[i]) { - targetBuffers[i].limit(toReconstructLen); - outputs[m++] = targetBuffers[i]; - } - } - decoder.decode(inputs, erasedIndices, outputs); - - for (int i = 0; i < targets.length; i++) { - if (targetsStatus[i]) { - long blockLen = getBlockLen(blockGroup, targetIndices[i]); - long remaining = blockLen - positionInBlock; - if (remaining <= 0) { - targetBuffers[i].limit(0); - } else if (remaining < toReconstructLen) { - targetBuffers[i].limit((int)remaining); - } - } - } - } - - /** - * Schedule a read from some new source DN if some DN is corrupted - * or slow, this is called from the read iteration. - * Initially we may only have minRequiredSources number of - * StripedReader. - * If the position is at the end of target block, don't need to do - * real read, and return the array index of source DN, otherwise -1. - * - * @param used the used source DNs in this iteration. - * @return the array index of source DN if don't need to do real read. - */ - private int scheduleNewRead(BitSet used, int reconstructLen, - CorruptedBlocks corruptedBlocks) { - StripedReader reader = null; - // step1: initially we may only have minRequiredSources - // number of StripedReader, and there may be some source DNs we never - // read before, so will try to create StripedReader for one new source DN - // and try to read from it. If found, go to step 3. - int m = stripedReaders.size(); - int toRead = 0; - while (reader == null && m < sources.length) { - reader = addStripedReader(m, positionInBlock); - toRead = getReadLength(liveIndices[m], reconstructLen); - if (toRead > 0) { - if (reader.blockReader == null) { - reader = null; - m++; - } - } else { - used.set(m); - return m; - } - } - - // step2: if there is no new source DN we can use, try to find a source - // DN we ever read from but because some reason, e.g., slow, it - // is not in the success DN list at the begin of this iteration, so - // we have not tried it in this iteration. Now we have a chance to - // revisit it again. - for (int i = 0; reader == null && i < stripedReaders.size(); i++) { - if (!used.get(i)) { - StripedReader r = stripedReaders.get(i); - toRead = getReadLength(liveIndices[i], reconstructLen); - if (toRead > 0) { - IOUtils.closeStream(r.blockReader); - r.blockReader = newBlockReader( - getBlock(blockGroup, liveIndices[i]), positionInBlock, - sources[i]); - if (r.blockReader != null) { - r.buffer.position(0); - m = i; - reader = r; - } - } else { - used.set(i); - r.buffer.position(0); - return i; - } - } - } - - // step3: schedule if find a correct source DN and need to do real read. - if (reader != null) { - Callable readCallable = readFromBlock(reader, reader.buffer, - toRead, corruptedBlocks); - Future f = readService.submit(readCallable); - futures.put(f, m); - used.set(m); - } - - return -1; - } - - // cancel all reads. - private void cancelReads(Collection> futures) { - for (Future future : futures) { - future.cancel(true); - } - } - - private Callable readFromBlock(final StripedReader reader, - final ByteBuffer buf, final int length, - final CorruptedBlocks corruptedBlocks) { - return new Callable() { - - @Override - public Void call() throws Exception { - try { - buf.limit(length); - actualReadFromBlock(reader.blockReader, buf); - return null; - } catch (ChecksumException e) { - LOG.warn("Found Checksum error for {} from {} at {}", reader.block, - reader.source, e.getPos()); - corruptedBlocks.addCorruptedBlock(reader.block, reader.source); - throw e; - } catch (IOException e) { - LOG.info(e.getMessage()); - throw e; - } - } - - }; - } - - /** - * Read bytes from block - */ - private void actualReadFromBlock(BlockReader reader, ByteBuffer buf) - throws IOException { - int len = buf.remaining(); - int n = 0; - while (n < len) { - int nread = reader.read(buf); - if (nread <= 0) { - break; - } - n += nread; - } - } - - private InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) { - return NetUtils.createSocketAddr(dnInfo.getXferAddr( - datanode.getDnConf().getConnectToDnViaHostname())); - } - - private BlockReader newBlockReader(final ExtendedBlock block, - long offsetInBlock, DatanodeInfo dnInfo) { - if (offsetInBlock >= block.getNumBytes()) { - return null; - } - try { - InetSocketAddress dnAddr = getSocketAddress4Transfer(dnInfo); - Token blockToken = datanode.getBlockAccessToken( - block, EnumSet.of(BlockTokenIdentifier.AccessMode.READ)); - /* - * This can be further improved if the replica is local, then we can - * read directly from DN and need to check the replica is FINALIZED - * state, notice we should not use short-circuit local read which - * requires config for domain-socket in UNIX or legacy config in Windows. - * The network distance value isn't used for this scenario. - */ - return RemoteBlockReader2.newBlockReader( - "dummy", block, blockToken, offsetInBlock, - block.getNumBytes() - offsetInBlock, true, - "", newConnectedPeer(block, dnAddr, blockToken, dnInfo), dnInfo, - null, cachingStrategy, datanode.getTracer(), -1); - } catch (IOException e) { - LOG.debug("Exception while creating remote block reader, datanode {}", - dnInfo, e); - return null; - } - } - - private Peer newConnectedPeer(ExtendedBlock b, InetSocketAddress addr, - Token blockToken, DatanodeID datanodeId) - throws IOException { - Peer peer = null; - boolean success = false; - Socket sock = null; - final int socketTimeout = datanode.getDnConf().getSocketTimeout(); - try { - sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); - NetUtils.connect(sock, addr, socketTimeout); - peer = DFSUtilClient.peerFromSocketAndKey(datanode.getSaslClient(), - sock, datanode.getDataEncryptionKeyFactoryForBlock(b), - blockToken, datanodeId, socketTimeout); - success = true; - return peer; - } finally { - if (!success) { - IOUtils.cleanup(null, peer); - IOUtils.closeSocket(sock); - } - } - } - - /** - * Send data to targets - */ - private int transferData2Targets(boolean[] targetsStatus) { - int nsuccess = 0; - for (int i = 0; i < targets.length; i++) { - if (targetsStatus[i]) { - boolean success = false; - try { - ByteBuffer buffer = targetBuffers[i]; - - if (buffer.remaining() == 0) { - continue; - } - - checksum.calculateChunkedSums( - buffer.array(), 0, buffer.remaining(), checksumBuf, 0); - - int ckOff = 0; - while (buffer.remaining() > 0) { - DFSPacket packet = new DFSPacket(packetBuf, maxChunksPerPacket, - blockOffset4Targets[i], seqNo4Targets[i]++, checksumSize, false); - int maxBytesToPacket = maxChunksPerPacket * bytesPerChecksum; - int toWrite = buffer.remaining() > maxBytesToPacket ? - maxBytesToPacket : buffer.remaining(); - int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * checksumSize; - packet.writeChecksum(checksumBuf, ckOff, ckLen); - ckOff += ckLen; - packet.writeData(buffer, toWrite); - - // Send packet - packet.writeTo(targetOutputStreams[i]); - - blockOffset4Targets[i] += toWrite; - nsuccess++; - success = true; - } - } catch (IOException e) { - LOG.warn(e.getMessage()); - } - targetsStatus[i] = success; - } - } - return nsuccess; - } - - /** - * clear all buffers - */ - private void clearBuffers() { - for (StripedReader stripedReader : stripedReaders) { - if (stripedReader.buffer != null) { - stripedReader.buffer.clear(); - } - } - - if (zeroStripeBuffers != null) { - for (ByteBuffer zeroStripeBuffer : zeroStripeBuffers) { - zeroStripeBuffer.clear(); - } - } - - for (ByteBuffer targetBuffer : targetBuffers) { - if (targetBuffer != null) { - targetBuffer.clear(); - } - } - } - - // send an empty packet to mark the end of the block - private void endTargetBlocks(boolean[] targetsStatus) { - for (int i = 0; i < targets.length; i++) { - if (targetsStatus[i]) { - try { - DFSPacket packet = new DFSPacket(packetBuf, 0, - blockOffset4Targets[i], seqNo4Targets[i]++, checksumSize, true); - packet.writeTo(targetOutputStreams[i]); - targetOutputStreams[i].flush(); - } catch (IOException e) { - LOG.warn(e.getMessage()); - } - } - } - } - - /** - * Initialize output/input streams for transferring data to target - * and send create block request. - */ - private int initTargetStreams(boolean[] targetsStatus) { - int nsuccess = 0; - for (int i = 0; i < targets.length; i++) { - Socket socket = null; - DataOutputStream out = null; - DataInputStream in = null; - boolean success = false; - try { - InetSocketAddress targetAddr = - getSocketAddress4Transfer(targets[i]); - socket = datanode.newSocket(); - NetUtils.connect(socket, targetAddr, - datanode.getDnConf().getSocketTimeout()); - socket.setSoTimeout(datanode.getDnConf().getSocketTimeout()); - - ExtendedBlock block = getBlock(blockGroup, targetIndices[i]); - Token blockToken = - datanode.getBlockAccessToken(block, - EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE)); - - long writeTimeout = datanode.getDnConf().getSocketWriteTimeout(); - OutputStream unbufOut = NetUtils.getOutputStream(socket, writeTimeout); - InputStream unbufIn = NetUtils.getInputStream(socket); - DataEncryptionKeyFactory keyFactory = - datanode.getDataEncryptionKeyFactoryForBlock(block); - IOStreamPair saslStreams = datanode.getSaslClient().socketSend( - socket, unbufOut, unbufIn, keyFactory, blockToken, targets[i]); - - unbufOut = saslStreams.out; - unbufIn = saslStreams.in; - - out = new DataOutputStream(new BufferedOutputStream(unbufOut, - DFSUtilClient.getSmallBufferSize(conf))); - in = new DataInputStream(unbufIn); - - DatanodeInfo source = new DatanodeInfo(datanode.getDatanodeId()); - new Sender(out).writeBlock(block, targetStorageTypes[i], - blockToken, "", new DatanodeInfo[]{targets[i]}, - new StorageType[]{targetStorageTypes[i]}, source, - BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0, 0, 0, - checksum, cachingStrategy, false, false, null); - - targetSockets[i] = socket; - targetOutputStreams[i] = out; - targetInputStreams[i] = in; - nsuccess++; - success = true; - } catch (Throwable e) { - LOG.warn(e.getMessage()); - } finally { - if (!success) { - IOUtils.closeStream(out); - IOUtils.closeStream(in); - IOUtils.closeStream(socket); - } - } - targetsStatus[i] = success; - } - return nsuccess; - } + DataNode getDatanode() { + return datanode; } - private static class StripedReader { - private final short index; // internal block index - private BlockReader blockReader; - private ByteBuffer buffer; - private final ExtendedBlock block; - private final DatanodeInfo source; + Configuration getConf() { + return conf; + } - StripedReader(short index, ExtendedBlock block, DatanodeInfo source) { - this.index = index; - this.block = block; - this.source = source; - } + ThreadPoolExecutor getStripedReadPool() { + return stripedReadPool; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java new file mode 100644 index 00000000000..7f71bf76ddf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.erasurecode; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ChecksumException; +import org.apache.hadoop.hdfs.BlockReader; +import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks; +import org.apache.hadoop.hdfs.RemoteBlockReader2; +import org.apache.hadoop.hdfs.net.Peer; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.token.Token; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.EnumSet; +import java.util.concurrent.Callable; + +/** + * StripedBlockReader is used to read block data from one source DN, it contains + * a block reader, read buffer and striped block index. + * Only allocate StripedBlockReader once for one source, and the StripedReader + * has the same array order with sources. Typically we only need to allocate + * minimum number (minRequiredSources) of StripedReader, and allocate + * new for new source DN if some existing DN invalid or slow. + * If some source DN is corrupt, set the corresponding blockReader to + * null and will never read from it again. + */ +@InterfaceAudience.Private +class StripedBlockReader { + private static final Logger LOG = DataNode.LOG; + + private StripedReader stripedReader; + private final DataNode datanode; + private final Configuration conf; + + private final short index; // internal block index + private final ExtendedBlock block; + private final DatanodeInfo source; + private BlockReader blockReader; + private ByteBuffer buffer; + + StripedBlockReader(StripedReader stripedReader, DataNode datanode, + Configuration conf, short index, ExtendedBlock block, + DatanodeInfo source, long offsetInBlock) { + this.stripedReader = stripedReader; + this.datanode = datanode; + this.conf = conf; + + this.index = index; + this.source = source; + this.block = block; + + BlockReader tmpBlockReader = createBlockReader(offsetInBlock); + if (tmpBlockReader != null) { + this.blockReader = tmpBlockReader; + } + } + + ByteBuffer getReadBuffer() { + if (buffer == null) { + this.buffer = stripedReader.allocateReadBuffer(); + } + return buffer; + } + + void resetBlockReader(long offsetInBlock) { + this.blockReader = createBlockReader(offsetInBlock); + } + + private BlockReader createBlockReader(long offsetInBlock) { + if (offsetInBlock >= block.getNumBytes()) { + return null; + } + try { + InetSocketAddress dnAddr = + stripedReader.getSocketAddress4Transfer(source); + Token blockToken = datanode.getBlockAccessToken( + block, EnumSet.of(BlockTokenIdentifier.AccessMode.READ)); + /* + * This can be further improved if the replica is local, then we can + * read directly from DN and need to check the replica is FINALIZED + * state, notice we should not use short-circuit local read which + * requires config for domain-socket in UNIX or legacy config in + * Windows. The network distance value isn't used for this scenario. + * + * TODO: add proper tracer + */ + return RemoteBlockReader2.newBlockReader( + "dummy", block, blockToken, offsetInBlock, + block.getNumBytes() - offsetInBlock, true, + "", newConnectedPeer(block, dnAddr, blockToken, source), source, + null, stripedReader.getCachingStrategy(), datanode.getTracer(), -1); + } catch (IOException e) { + LOG.debug("Exception while creating remote block reader, datanode {}", + source, e); + return null; + } + } + + private Peer newConnectedPeer(ExtendedBlock b, InetSocketAddress addr, + Token blockToken, + DatanodeID datanodeId) + throws IOException { + Peer peer = null; + boolean success = false; + Socket sock = null; + final int socketTimeout = datanode.getDnConf().getSocketTimeout(); + try { + sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); + NetUtils.connect(sock, addr, socketTimeout); + peer = DFSUtilClient.peerFromSocketAndKey(datanode.getSaslClient(), + sock, datanode.getDataEncryptionKeyFactoryForBlock(b), + blockToken, datanodeId, socketTimeout); + success = true; + return peer; + } finally { + if (!success) { + IOUtils.cleanup(null, peer); + IOUtils.closeSocket(sock); + } + } + } + + Callable readFromBlock(final int length, + final CorruptedBlocks corruptedBlocks) { + return new Callable() { + + @Override + public Void call() throws Exception { + try { + getReadBuffer().limit(length); + actualReadFromBlock(); + return null; + } catch (ChecksumException e) { + LOG.warn("Found Checksum error for {} from {} at {}", block, + source, e.getPos()); + corruptedBlocks.addCorruptedBlock(block, source); + throw e; + } catch (IOException e) { + LOG.info(e.getMessage()); + throw e; + } + } + }; + } + + /** + * Perform actual reading of bytes from block. + */ + private void actualReadFromBlock() throws IOException { + int len = buffer.remaining(); + int n = 0; + while (n < len) { + int nread = blockReader.read(buffer); + if (nread <= 0) { + break; + } + n += nread; + } + } + + // close block reader + void closeBlockReader() { + IOUtils.closeStream(blockReader); + blockReader = null; + } + + short getIndex() { + return index; + } + + BlockReader getBlockReader() { + return blockReader; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java new file mode 100644 index 00000000000..a62f3c28c73 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.erasurecode; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSPacket; +import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; +import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; +import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; +import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.token.Token; + +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.EnumSet; + +/** + * A striped block writer that writes reconstructed data to the remote target + * datanode. + */ +@InterfaceAudience.Private +class StripedBlockWriter { + private final StripedWriter stripedWriter; + private final DataNode datanode; + private final Configuration conf; + + private final ExtendedBlock block; + private final DatanodeInfo target; + private final StorageType storageType; + + private Socket targetSocket; + private DataOutputStream targetOutputStream; + private DataInputStream targetInputStream; + private ByteBuffer targetBuffer; + private long blockOffset4Target = 0; + private long seqNo4Target = 0; + + StripedBlockWriter(StripedWriter stripedWriter, DataNode datanode, + Configuration conf, ExtendedBlock block, + DatanodeInfo target, StorageType storageType) + throws IOException { + this.stripedWriter = stripedWriter; + this.datanode = datanode; + this.conf = conf; + + this.block = block; + this.target = target; + this.storageType = storageType; + + this.targetBuffer = stripedWriter.allocateWriteBuffer(); + + init(); + } + + ByteBuffer getTargetBuffer() { + return targetBuffer; + } + + /** + * Initialize output/input streams for transferring data to target + * and send create block request. + */ + private void init() throws IOException { + Socket socket = null; + DataOutputStream out = null; + DataInputStream in = null; + boolean success = false; + try { + InetSocketAddress targetAddr = + stripedWriter.getSocketAddress4Transfer(target); + socket = datanode.newSocket(); + NetUtils.connect(socket, targetAddr, + datanode.getDnConf().getSocketTimeout()); + socket.setSoTimeout(datanode.getDnConf().getSocketTimeout()); + + Token blockToken = + datanode.getBlockAccessToken(block, + EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE)); + + long writeTimeout = datanode.getDnConf().getSocketWriteTimeout(); + OutputStream unbufOut = NetUtils.getOutputStream(socket, writeTimeout); + InputStream unbufIn = NetUtils.getInputStream(socket); + DataEncryptionKeyFactory keyFactory = + datanode.getDataEncryptionKeyFactoryForBlock(block); + IOStreamPair saslStreams = datanode.getSaslClient().socketSend( + socket, unbufOut, unbufIn, keyFactory, blockToken, target); + + unbufOut = saslStreams.out; + unbufIn = saslStreams.in; + + out = new DataOutputStream(new BufferedOutputStream(unbufOut, + DFSUtilClient.getSmallBufferSize(conf))); + in = new DataInputStream(unbufIn); + + DatanodeInfo source = new DatanodeInfo(datanode.getDatanodeId()); + new Sender(out).writeBlock(block, storageType, + blockToken, "", new DatanodeInfo[]{target}, + new StorageType[]{storageType}, source, + BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0, 0, 0, + stripedWriter.getChecksum(), stripedWriter.getCachingStrategy(), + false, false, null); + + targetSocket = socket; + targetOutputStream = out; + targetInputStream = in; + success = true; + } finally { + if (!success) { + IOUtils.closeStream(out); + IOUtils.closeStream(in); + IOUtils.closeStream(socket); + } + } + } + + /** + * Send data to targets. + */ + void transferData2Target(byte[] packetBuf) throws IOException { + if (targetBuffer.remaining() == 0) { + return; + } + + stripedWriter.getChecksum().calculateChunkedSums( + targetBuffer.array(), 0, targetBuffer.remaining(), + stripedWriter.getChecksumBuf(), 0); + + int ckOff = 0; + while (targetBuffer.remaining() > 0) { + DFSPacket packet = new DFSPacket(packetBuf, + stripedWriter.getMaxChunksPerPacket(), + blockOffset4Target, seqNo4Target++, + stripedWriter.getChecksumSize(), false); + int maxBytesToPacket = stripedWriter.getMaxChunksPerPacket() + * stripedWriter.getBytesPerChecksum(); + int toWrite = targetBuffer.remaining() > maxBytesToPacket ? + maxBytesToPacket : targetBuffer.remaining(); + int ckLen = ((toWrite - 1) / stripedWriter.getBytesPerChecksum() + 1) + * stripedWriter.getChecksumSize(); + packet.writeChecksum(stripedWriter.getChecksumBuf(), ckOff, ckLen); + ckOff += ckLen; + packet.writeData(targetBuffer, toWrite); + + // Send packet + packet.writeTo(targetOutputStream); + + blockOffset4Target += toWrite; + } + } + + // send an empty packet to mark the end of the block + void endTargetBlock(byte[] packetBuf) throws IOException { + DFSPacket packet = new DFSPacket(packetBuf, 0, + blockOffset4Target, seqNo4Target++, + stripedWriter.getChecksumSize(), true); + packet.writeTo(targetOutputStream); + targetOutputStream.flush(); + } + + void close() { + IOUtils.closeStream(targetOutputStream); + IOUtils.closeStream(targetInputStream); + IOUtils.closeStream(targetSocket); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java new file mode 100644 index 00000000000..fb7699a5cd4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java @@ -0,0 +1,466 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.erasurecode; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult; +import org.apache.hadoop.util.DataChecksum; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.Future; + +/** + * Manage striped readers that performs reading of block data from remote to + * serve input data for the erasure decoding. + */ +@InterfaceAudience.Private +class StripedReader { + private static final Logger LOG = DataNode.LOG; + + private final int stripedReadTimeoutInMills; + private final int stripedReadBufferSize; + + private StripedReconstructor reconstructor; + private final DataNode datanode; + private final Configuration conf; + + private final int dataBlkNum; + private final int parityBlkNum; + + + private DataChecksum checksum; + // Striped read buffer size + private int bufferSize; + private int[] successList; + + private final int minRequiredSources; + // The buffers and indices for striped blocks whose length is 0 + private ByteBuffer[] zeroStripeBuffers; + private short[] zeroStripeIndices; + + // sources + private final byte[] liveIndices; + private final DatanodeInfo[] sources; + + private final List readers; + + private final Map, Integer> futures = new HashMap<>(); + private final CompletionService readService; + + StripedReader(StripedReconstructor reconstructor, DataNode datanode, + Configuration conf, + BlockECReconstructionInfo reconstructionInfo) { + stripedReadTimeoutInMills = conf.getInt( + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_KEY, + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT); + stripedReadBufferSize = conf.getInt( + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT); + + this.reconstructor = reconstructor; + this.datanode = datanode; + this.conf = conf; + + ErasureCodingPolicy ecPolicy = reconstructionInfo.getErasureCodingPolicy(); + dataBlkNum = ecPolicy.getNumDataUnits(); + parityBlkNum = ecPolicy.getNumParityUnits(); + + ExtendedBlock blockGroup = reconstructionInfo.getExtendedBlock(); + int cellsNum = (int)((blockGroup.getNumBytes() - 1) / ecPolicy.getCellSize() + + 1); + minRequiredSources = Math.min(cellsNum, dataBlkNum); + + if (minRequiredSources < dataBlkNum) { + int zeroStripNum = dataBlkNum - minRequiredSources; + zeroStripeBuffers = new ByteBuffer[zeroStripNum]; + zeroStripeIndices = new short[zeroStripNum]; + } + + liveIndices = reconstructionInfo.getLiveBlockIndices(); + sources = reconstructionInfo.getSourceDnInfos(); + + readers = new ArrayList<>(sources.length); + readService = reconstructor.createReadService(); + + Preconditions.checkArgument(liveIndices.length >= minRequiredSources, + "No enough live striped blocks."); + Preconditions.checkArgument(liveIndices.length == sources.length, + "liveBlockIndices and source datanodes should match"); + } + + void init() throws IOException { + initReaders(); + + initBufferSize(); + + initZeroStrip(); + } + + private void initReaders() throws IOException { + // Store the array indices of source DNs we have read successfully. + // In each iteration of read, the successList list may be updated if + // some source DN is corrupted or slow. And use the updated successList + // list of DNs for next iteration read. + successList = new int[minRequiredSources]; + + StripedBlockReader reader; + int nSuccess = 0; + for (int i = 0; i < sources.length && nSuccess < minRequiredSources; i++) { + reader = createReader(i, 0); + readers.add(reader); + if (reader.getBlockReader() != null) { + initOrVerifyChecksum(reader); + successList[nSuccess++] = i; + } + } + + if (nSuccess < minRequiredSources) { + String error = "Can't find minimum sources required by " + + "reconstruction, block id: " + + reconstructor.getBlockGroup().getBlockId(); + throw new IOException(error); + } + } + + StripedBlockReader createReader(int idxInSources, long offsetInBlock) { + return new StripedBlockReader(this, datanode, + conf, liveIndices[idxInSources], + reconstructor.getBlock(liveIndices[idxInSources]), + sources[idxInSources], offsetInBlock); + } + + private void initBufferSize() { + int bytesPerChecksum = checksum.getBytesPerChecksum(); + // The bufferSize is flat to divide bytesPerChecksum + int readBufferSize = stripedReadBufferSize; + bufferSize = readBufferSize < bytesPerChecksum ? bytesPerChecksum : + readBufferSize - readBufferSize % bytesPerChecksum; + } + + // init checksum from block reader + private void initOrVerifyChecksum(StripedBlockReader reader) { + if (checksum == null) { + checksum = reader.getBlockReader().getDataChecksum(); + } else { + assert reader.getBlockReader().getDataChecksum().equals(checksum); + } + } + + protected ByteBuffer allocateReadBuffer() { + return ByteBuffer.allocate(getBufferSize()); + } + + private void initZeroStrip() { + if (zeroStripeBuffers != null) { + for (int i = 0; i < zeroStripeBuffers.length; i++) { + zeroStripeBuffers[i] = reconstructor.allocateBuffer(bufferSize); + } + } + + BitSet bitset = reconstructor.getLiveBitSet(); + int k = 0; + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + if (!bitset.get(i)) { + if (reconstructor.getBlockLen(i) <= 0) { + zeroStripeIndices[k++] = (short)i; + } + } + } + } + + private int getReadLength(int index, int reconstructLength) { + // the reading length should not exceed the length for reconstruction + long blockLen = reconstructor.getBlockLen(index); + long remaining = blockLen - reconstructor.getPositionInBlock(); + return (int) Math.min(remaining, reconstructLength); + } + + ByteBuffer[] getInputBuffers(int toReconstructLen) { + ByteBuffer[] inputs = new ByteBuffer[dataBlkNum + parityBlkNum]; + + for (int i = 0; i < successList.length; i++) { + int index = successList[i]; + StripedBlockReader reader = getReader(index); + ByteBuffer buffer = reader.getReadBuffer(); + paddingBufferToLen(buffer, toReconstructLen); + inputs[reader.getIndex()] = (ByteBuffer)buffer.flip(); + } + + if (successList.length < dataBlkNum) { + for (int i = 0; i < zeroStripeBuffers.length; i++) { + ByteBuffer buffer = zeroStripeBuffers[i]; + paddingBufferToLen(buffer, toReconstructLen); + int index = zeroStripeIndices[i]; + inputs[index] = (ByteBuffer)buffer.flip(); + } + } + + return inputs; + } + + private void paddingBufferToLen(ByteBuffer buffer, int len) { + if (len > buffer.limit()) { + buffer.limit(len); + } + int toPadding = len - buffer.position(); + for (int i = 0; i < toPadding; i++) { + buffer.put((byte) 0); + } + } + + /** + * Read from minimum source DNs required for reconstruction in the iteration. + * First try the success list which we think they are the best DNs + * If source DN is corrupt or slow, try to read some other source DN, + * and will update the success list. + * + * Remember the updated success list and return it for following + * operations and next iteration read. + * + * @param reconstructLength the length to reconstruct. + * @return updated success list of source DNs we do real read + * @throws IOException + */ + void readMinimumSources(int reconstructLength) throws IOException { + CorruptedBlocks corruptedBlocks = new CorruptedBlocks(); + try { + successList = doReadMinimumSources(reconstructLength, corruptedBlocks); + } finally { + // report corrupted blocks to NN + datanode.reportCorruptedBlocks(corruptedBlocks); + } + } + + int[] doReadMinimumSources(int reconstructLength, + CorruptedBlocks corruptedBlocks) + throws IOException { + Preconditions.checkArgument(reconstructLength >= 0 && + reconstructLength <= bufferSize); + int nSuccess = 0; + int[] newSuccess = new int[minRequiredSources]; + BitSet usedFlag = new BitSet(sources.length); + /* + * Read from minimum source DNs required, the success list contains + * source DNs which we think best. + */ + for (int i = 0; i < minRequiredSources; i++) { + StripedBlockReader reader = readers.get(successList[i]); + int toRead = getReadLength(liveIndices[successList[i]], + reconstructLength); + if (toRead > 0) { + Callable readCallable = + reader.readFromBlock(toRead, corruptedBlocks); + Future f = readService.submit(readCallable); + futures.put(f, successList[i]); + } else { + // If the read length is 0, we don't need to do real read + reader.getReadBuffer().position(0); + newSuccess[nSuccess++] = successList[i]; + } + usedFlag.set(successList[i]); + } + + while (!futures.isEmpty()) { + try { + StripingChunkReadResult result = + StripedBlockUtil.getNextCompletedStripedRead( + readService, futures, stripedReadTimeoutInMills); + int resultIndex = -1; + if (result.state == StripingChunkReadResult.SUCCESSFUL) { + resultIndex = result.index; + } else if (result.state == StripingChunkReadResult.FAILED) { + // If read failed for some source DN, we should not use it anymore + // and schedule read from another source DN. + StripedBlockReader failedReader = readers.get(result.index); + failedReader.closeBlockReader(); + resultIndex = scheduleNewRead(usedFlag, + reconstructLength, corruptedBlocks); + } else if (result.state == StripingChunkReadResult.TIMEOUT) { + // If timeout, we also schedule a new read. + resultIndex = scheduleNewRead(usedFlag, + reconstructLength, corruptedBlocks); + } + if (resultIndex >= 0) { + newSuccess[nSuccess++] = resultIndex; + if (nSuccess >= minRequiredSources) { + // cancel remaining reads if we read successfully from minimum + // number of source DNs required by reconstruction. + cancelReads(futures.keySet()); + futures.clear(); + break; + } + } + } catch (InterruptedException e) { + LOG.info("Read data interrupted.", e); + cancelReads(futures.keySet()); + futures.clear(); + break; + } + } + + if (nSuccess < minRequiredSources) { + String error = "Can't read data from minimum number of sources " + + "required by reconstruction, block id: " + + reconstructor.getBlockGroup().getBlockId(); + throw new IOException(error); + } + + return newSuccess; + } + + /** + * Schedule a read from some new source DN if some DN is corrupted + * or slow, this is called from the read iteration. + * Initially we may only have minRequiredSources number of + * StripedBlockReader. + * If the position is at the end of target block, don't need to do + * real read, and return the array index of source DN, otherwise -1. + * + * @param used the used source DNs in this iteration. + * @return the array index of source DN if don't need to do real read. + */ + private int scheduleNewRead(BitSet used, int reconstructLength, + CorruptedBlocks corruptedBlocks) { + StripedBlockReader reader = null; + // step1: initially we may only have minRequiredSources + // number of StripedBlockReader, and there may be some source DNs we never + // read before, so will try to create StripedBlockReader for one new source + // DN and try to read from it. If found, go to step 3. + int m = readers.size(); + int toRead = 0; + while (reader == null && m < sources.length) { + reader = createReader(m, reconstructor.getPositionInBlock()); + readers.add(reader); + toRead = getReadLength(liveIndices[m], reconstructLength); + if (toRead > 0) { + if (reader.getBlockReader() == null) { + reader = null; + m++; + } + } else { + used.set(m); + return m; + } + } + + // step2: if there is no new source DN we can use, try to find a source + // DN we ever read from but because some reason, e.g., slow, it + // is not in the success DN list at the begin of this iteration, so + // we have not tried it in this iteration. Now we have a chance to + // revisit it again. + for (int i = 0; reader == null && i < readers.size(); i++) { + if (!used.get(i)) { + StripedBlockReader stripedReader = readers.get(i); + toRead = getReadLength(liveIndices[i], reconstructLength); + if (toRead > 0) { + stripedReader.closeBlockReader(); + stripedReader.resetBlockReader(reconstructor.getPositionInBlock()); + if (stripedReader.getBlockReader() != null) { + stripedReader.getReadBuffer().position(0); + m = i; + reader = stripedReader; + } + } else { + used.set(i); + stripedReader.getReadBuffer().position(0); + return i; + } + } + } + + // step3: schedule if find a correct source DN and need to do real read. + if (reader != null) { + Callable readCallable = + reader.readFromBlock(toRead, corruptedBlocks); + Future f = readService.submit(readCallable); + futures.put(f, m); + used.set(m); + } + + return -1; + } + + // Cancel all reads. + private static void cancelReads(Collection> futures) { + for (Future future : futures) { + future.cancel(true); + } + } + + void close() { + for (StripedBlockReader reader : readers) { + reader.closeBlockReader(); + } + } + + StripedBlockReader getReader(int i) { + return readers.get(i); + } + + int getBufferSize() { + return bufferSize; + } + + DataChecksum getChecksum() { + return checksum; + } + + void clearBuffers() { + if (zeroStripeBuffers != null) { + for (ByteBuffer zeroStripeBuffer : zeroStripeBuffers) { + zeroStripeBuffer.clear(); + } + } + + for (StripedBlockReader reader : readers) { + if (reader.getReadBuffer() != null) { + reader.getReadBuffer().clear(); + } + } + } + + InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) { + return reconstructor.getSocketAddress4Transfer(dnInfo); + } + + CachingStrategy getCachingStrategy() { + return reconstructor.getCachingStrategy(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java new file mode 100644 index 00000000000..a0a5f836b66 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -0,0 +1,273 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.erasurecode; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.io.erasurecode.CodecUtil; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.DataChecksum; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.BitSet; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; + +/** + * StripedReconstructor reconstruct one or more missed striped block in the + * striped block group, the minimum number of live striped blocks should be + * no less than data block number. + * + * | <- Striped Block Group -> | + * blk_0 blk_1 blk_2(*) blk_3 ... <- A striped block group + * | | | | + * v v v v + * +------+ +------+ +------+ +------+ + * |cell_0| |cell_1| |cell_2| |cell_3| ... + * +------+ +------+ +------+ +------+ + * |cell_4| |cell_5| |cell_6| |cell_7| ... + * +------+ +------+ +------+ +------+ + * |cell_8| |cell_9| |cell10| |cell11| ... + * +------+ +------+ +------+ +------+ + * ... ... ... ... + * + * + * We use following steps to reconstruct striped block group, in each round, we + * reconstruct bufferSize data until finish, the + * bufferSize is configurable and may be less or larger than + * cell size: + * step1: read bufferSize data from minimum number of sources + * required by reconstruction. + * step2: decode data for targets. + * step3: transfer data to targets. + * + * In step1, try to read bufferSize data from minimum number + * of sources , if there is corrupt or stale sources, read from new source + * will be scheduled. The best sources are remembered for next round and + * may be updated in each round. + * + * In step2, typically if source blocks we read are all data blocks, we + * need to call encode, and if there is one parity block, we need to call + * decode. Notice we only read once and reconstruct all missed striped block + * if they are more than one. + * + * In step3, send the reconstructed data to targets by constructing packet + * and send them directly. Same as continuous block replication, we + * don't check the packet ack. Since the datanode doing the reconstruction work + * are one of the source datanodes, so the reconstructed data are sent + * remotely. + * + * There are some points we can do further improvements in next phase: + * 1. we can read the block file directly on the local datanode, + * currently we use remote block reader. (Notice short-circuit is not + * a good choice, see inline comments). + * 2. We need to check the packet ack for EC reconstruction? Since EC + * reconstruction is more expensive than continuous block replication, + * it needs to read from several other datanodes, should we make sure the + * reconstructed result received by targets? + */ +@InterfaceAudience.Private +class StripedReconstructor implements Runnable { + private static final Logger LOG = DataNode.LOG; + + private final ErasureCodingWorker worker; + private final DataNode datanode; + private final Configuration conf; + + private final ErasureCodingPolicy ecPolicy; + + private RawErasureDecoder decoder; + + private final ExtendedBlock blockGroup; + private final BitSet liveBitSet; + + // position in striped internal block + private long positionInBlock; + + private StripedReader stripedReader; + + private StripedWriter stripedWriter; + + private final CachingStrategy cachingStrategy; + + StripedReconstructor(ErasureCodingWorker worker, + BlockECReconstructionInfo reconstructionInfo) { + this.worker = worker; + this.datanode = worker.getDatanode(); + this.conf = worker.getConf(); + + ecPolicy = reconstructionInfo.getErasureCodingPolicy(); + + blockGroup = reconstructionInfo.getExtendedBlock(); + byte[] liveIndices = reconstructionInfo.getLiveBlockIndices(); + liveBitSet = new BitSet(ecPolicy.getNumDataUnits() + + ecPolicy.getNumParityUnits()); + for (int i = 0; i < liveIndices.length; i++) { + liveBitSet.set(liveIndices[i]); + } + + stripedReader = new StripedReader(this, datanode, + conf, reconstructionInfo); + stripedWriter = new StripedWriter(this, datanode, + conf, reconstructionInfo); + + cachingStrategy = CachingStrategy.newDefaultStrategy(); + + positionInBlock = 0L; + } + + BitSet getLiveBitSet() { + return liveBitSet; + } + + ByteBuffer allocateBuffer(int length) { + return ByteBuffer.allocate(length); + } + + ExtendedBlock getBlock(int i) { + return StripedBlockUtil.constructInternalBlock(blockGroup, ecPolicy, i); + } + + long getBlockLen(int i) { + return StripedBlockUtil.getInternalBlockLength(blockGroup.getNumBytes(), + ecPolicy, i); + } + + boolean hasValidTargets() { + return stripedWriter.hasValidTargets(); + } + + @Override + public void run() { + datanode.incrementXmitsInProgress(); + try { + stripedReader.init(); + + stripedWriter.init(); + + reconstructAndTransfer(); + + stripedWriter.endTargetBlocks(); + + // Currently we don't check the acks for packets, this is similar as + // block replication. + } catch (Throwable e) { + LOG.warn("Failed to reconstruct striped block: {}", blockGroup, e); + } finally { + datanode.decrementXmitsInProgress(); + + stripedReader.close(); + + stripedWriter.close(); + } + } + + void reconstructAndTransfer() throws IOException { + while (positionInBlock < stripedWriter.getMaxTargetLength()) { + long remaining = stripedWriter.getMaxTargetLength() - positionInBlock; + final int toReconstructLen = + (int) Math.min(stripedReader.getBufferSize(), remaining); + // step1: read from minimum source DNs required for reconstruction. + // The returned success list is the source DNs we do real read from + stripedReader.readMinimumSources(toReconstructLen); + + // step2: decode to reconstruct targets + reconstructTargets(toReconstructLen); + + // step3: transfer data + if (stripedWriter.transferData2Targets() == 0) { + String error = "Transfer failed for all targets."; + throw new IOException(error); + } + + positionInBlock += toReconstructLen; + + clearBuffers(); + } + } + + // Initialize decoder + private void initDecoderIfNecessary() { + if (decoder == null) { + decoder = CodecUtil.createRSRawDecoder(conf, ecPolicy.getNumDataUnits(), + ecPolicy.getNumParityUnits()); + } + } + + private void reconstructTargets(int toReconstructLen) { + initDecoderIfNecessary(); + + ByteBuffer[] inputs = stripedReader.getInputBuffers(toReconstructLen); + + int[] erasedIndices = stripedWriter.getRealTargetIndices(); + ByteBuffer[] outputs = stripedWriter.getRealTargetBuffers(toReconstructLen); + + decoder.decode(inputs, erasedIndices, outputs); + + stripedWriter.updateRealTargetBuffers(toReconstructLen); + } + + long getPositionInBlock() { + return positionInBlock; + } + + /** + * Clear all associated buffers. + */ + private void clearBuffers() { + stripedReader.clearBuffers(); + + stripedWriter.clearBuffers(); + } + + InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) { + return NetUtils.createSocketAddr(dnInfo.getXferAddr( + datanode.getDnConf().getConnectToDnViaHostname())); + } + + int getBufferSize() { + return stripedReader.getBufferSize(); + } + + DataChecksum getChecksum() { + return stripedReader.getChecksum(); + } + + CachingStrategy getCachingStrategy() { + return cachingStrategy; + } + + CompletionService createReadService() { + return new ExecutorCompletionService<>(worker.getStripedReadPool()); + } + + ExtendedBlock getBlockGroup() { + return blockGroup; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java new file mode 100644 index 00000000000..e2052a39704 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -0,0 +1,313 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.erasurecode; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; +import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; +import org.apache.hadoop.util.DataChecksum; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.BitSet; + +/** + * Manage striped writers that writes to a target with reconstructed data. + */ +@InterfaceAudience.Private +class StripedWriter { + private static final Logger LOG = DataNode.LOG; + private final static int WRITE_PACKET_SIZE = 64 * 1024; + + private final StripedReconstructor reconstructor; + private final DataNode datanode; + private final Configuration conf; + + private final int dataBlkNum; + private final int parityBlkNum; + + private boolean[] targetsStatus; + + // targets + private final DatanodeInfo[] targets; + private final short[] targetIndices; + private boolean hasValidTargets; + private final StorageType[] targetStorageTypes; + private long maxTargetLength; + + private StripedBlockWriter[] writers; + + private int maxChunksPerPacket; + private byte[] packetBuf; + private byte[] checksumBuf; + private int bytesPerChecksum; + private int checksumSize; + + StripedWriter(StripedReconstructor reconstructor, + DataNode datanode, + Configuration conf, + BlockECReconstructionInfo reconstructionInfo) { + this.reconstructor = reconstructor; + this.datanode = datanode; + this.conf = conf; + + ErasureCodingPolicy ecPolicy = reconstructionInfo.getErasureCodingPolicy(); + dataBlkNum = ecPolicy.getNumDataUnits(); + parityBlkNum = ecPolicy.getNumParityUnits(); + + targets = reconstructionInfo.getTargetDnInfos(); + targetStorageTypes = reconstructionInfo.getTargetStorageTypes(); + + writers = new StripedBlockWriter[targets.length]; + + targetIndices = new short[targets.length]; + Preconditions.checkArgument(targetIndices.length <= parityBlkNum, + "Too much missed striped blocks."); + initTargetIndices(); + + maxTargetLength = 0L; + for (short targetIndex : targetIndices) { + maxTargetLength = Math.max(maxTargetLength, + reconstructor.getBlockLen(targetIndex)); + } + + // targetsStatus store whether some target is success, it will record + // any failed target once, if some target failed (invalid DN or transfer + // failed), will not transfer data to it any more. + targetsStatus = new boolean[targets.length]; + } + + void init() throws IOException { + DataChecksum checksum = reconstructor.getChecksum(); + checksumSize = checksum.getChecksumSize(); + bytesPerChecksum = checksum.getBytesPerChecksum(); + int chunkSize = bytesPerChecksum + checksumSize; + maxChunksPerPacket = Math.max( + (WRITE_PACKET_SIZE - PacketHeader.PKT_MAX_HEADER_LEN) / chunkSize, 1); + int maxPacketSize = chunkSize * maxChunksPerPacket + + PacketHeader.PKT_MAX_HEADER_LEN; + + packetBuf = new byte[maxPacketSize]; + int tmpLen = checksumSize * + (reconstructor.getBufferSize() / bytesPerChecksum); + checksumBuf = new byte[tmpLen]; + + if (initTargetStreams() == 0) { + String error = "All targets are failed."; + throw new IOException(error); + } + } + + private void initTargetIndices() { + BitSet bitset = reconstructor.getLiveBitSet(); + + int m = 0; + int k = 0; + hasValidTargets = false; + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + if (!bitset.get(i)) { + if (reconstructor.getBlockLen(i) > 0) { + if (m < targets.length) { + targetIndices[m++] = (short)i; + hasValidTargets = true; + } + } + } + } + } + + /** + * Send reconstructed data to targets. + */ + int transferData2Targets() { + int nSuccess = 0; + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + boolean success = false; + try { + writers[i].transferData2Target(packetBuf); + nSuccess++; + success = true; + } catch (IOException e) { + LOG.warn(e.getMessage()); + } + targetsStatus[i] = success; + } + } + return nSuccess; + } + + /** + * Send an empty packet to mark the end of the block. + */ + void endTargetBlocks() { + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + try { + writers[i].endTargetBlock(packetBuf); + } catch (IOException e) { + LOG.warn(e.getMessage()); + } + } + } + } + + /** + * Initialize output/input streams for transferring data to target + * and send create block request. + */ + int initTargetStreams() { + int nSuccess = 0; + for (short i = 0; i < targets.length; i++) { + try { + writers[i] = createWriter(i); + nSuccess++; + targetsStatus[i] = true; + } catch (Throwable e) { + LOG.warn(e.getMessage()); + } + } + return nSuccess; + } + + private StripedBlockWriter createWriter(short index) throws IOException { + return new StripedBlockWriter(this, datanode, conf, + reconstructor.getBlock(targetIndices[index]), targets[index], + targetStorageTypes[index]); + } + + ByteBuffer allocateWriteBuffer() { + return reconstructor.allocateBuffer(reconstructor.getBufferSize()); + } + + int getTargets() { + return targets.length; + } + + private int getRealTargets() { + int m = 0; + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + m++; + } + } + return m; + } + + int[] getRealTargetIndices() { + int realTargets = getRealTargets(); + int[] results = new int[realTargets]; + int m = 0; + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + results[m++] = targetIndices[i]; + } + } + return results; + } + + ByteBuffer[] getRealTargetBuffers(int toReconstructLen) { + int numGood = getRealTargets(); + ByteBuffer[] outputs = new ByteBuffer[numGood]; + int m = 0; + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + writers[i].getTargetBuffer().limit(toReconstructLen); + outputs[m++] = writers[i].getTargetBuffer(); + } + } + return outputs; + } + + void updateRealTargetBuffers(int toReconstructLen) { + for (int i = 0; i < targets.length; i++) { + if (targetsStatus[i]) { + long blockLen = reconstructor.getBlockLen(targetIndices[i]); + long remaining = blockLen - reconstructor.getPositionInBlock(); + if (remaining <= 0) { + writers[i].getTargetBuffer().limit(0); + } else if (remaining < toReconstructLen) { + writers[i].getTargetBuffer().limit((int)remaining); + } + } + } + } + + long getMaxTargetLength() { + return maxTargetLength; + } + + byte[] getChecksumBuf() { + return checksumBuf; + } + + int getBytesPerChecksum() { + return bytesPerChecksum; + } + + int getChecksumSize() { + return checksumSize; + } + + DataChecksum getChecksum() { + return reconstructor.getChecksum(); + } + + int getMaxChunksPerPacket() { + return maxChunksPerPacket; + } + + CachingStrategy getCachingStrategy() { + return reconstructor.getCachingStrategy(); + } + + InetSocketAddress getSocketAddress4Transfer(DatanodeInfo target) { + return reconstructor.getSocketAddress4Transfer(target); + } + + boolean hasValidTargets() { + return hasValidTargets; + } + + /** + * Clear all buffers. + */ + void clearBuffers() { + for (StripedBlockWriter writer : writers) { + ByteBuffer targetBuffer = writer.getTargetBuffer(); + if (targetBuffer != null) { + targetBuffer.clear(); + } + } + } + + void close() { + for (int i = 0; i < targets.length; i++) { + writers[i].close(); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/package-info.java new file mode 100644 index 00000000000..3150fcedb08 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/package-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Datanode side striping + erasure coding related task processing. + */ +@InterfaceAudience.LimitedPrivate({"HDFS"}) +@InterfaceStability.Evolving +package org.apache.hadoop.hdfs.server.datanode.erasurecode; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 38ca8ceb8ce..7155e744631 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -230,22 +230,23 @@ public class TestReconstructStripedFile { private int generateErrors(Map corruptTargets, ReconstructionType type) throws IOException { - int stoppedDN = 0; - for (Map.Entry target : corruptTargets.entrySet()) { - if (stoppedDN == 0 || type != ReconstructionType.DataOnly + int stoppedDNs = 0; + for (Map.Entry target : + corruptTargets.entrySet()) { + if (stoppedDNs == 0 || type != ReconstructionType.DataOnly || random.nextBoolean()) { // stop at least one DN to trigger reconstruction LOG.info("Note: stop DataNode " + target.getValue().getDisplayName() + " with internal block " + target.getKey()); shutdownDataNode(target.getValue()); - stoppedDN++; + stoppedDNs++; } else { // corrupt the data on the DN LOG.info("Note: corrupt data on " + target.getValue().getDisplayName() + " with internal block " + target.getKey()); cluster.corruptReplica(target.getValue(), target.getKey()); } } - return stoppedDN; + return stoppedDNs; } /** From a62637a413ad88c4273d3251892b8fc1c05afa34 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 7 Apr 2016 14:01:33 +0800 Subject: [PATCH 75/75] HADOOP-12909. Change ipc.Client to support asynchronous calls. Contributed by Xiaobing Zhou --- .../java/org/apache/hadoop/ipc/Client.java | 73 +++- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 346 ++++++++++++++++++ .../java/org/apache/hadoop/ipc/TestIPC.java | 29 +- 3 files changed, 436 insertions(+), 12 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestAsyncIPC.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index fb11cb7a70b..489c35496fa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -62,6 +62,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -96,6 +97,7 @@ import org.apache.htrace.core.Tracer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.protobuf.CodedOutputStream; @@ -107,7 +109,7 @@ import com.google.protobuf.CodedOutputStream; */ @InterfaceAudience.LimitedPrivate(value = { "Common", "HDFS", "MapReduce", "Yarn" }) @InterfaceStability.Evolving -public class Client { +public class Client implements AutoCloseable { public static final Log LOG = LogFactory.getLog(Client.class); @@ -116,6 +118,20 @@ public class Client { private static final ThreadLocal callId = new ThreadLocal(); private static final ThreadLocal retryCount = new ThreadLocal(); + private static final ThreadLocal> returnValue = new ThreadLocal<>(); + private static final ThreadLocal asynchronousMode = + new ThreadLocal() { + @Override + protected Boolean initialValue() { + return false; + } + }; + + @SuppressWarnings("unchecked") + @Unstable + public static Future getReturnValue() { + return (Future) returnValue.get(); + } /** Set call id and retry count for the next call. */ public static void setCallIdAndRetryCount(int cid, int rc) { @@ -1354,8 +1370,8 @@ public class Client { ConnectionId remoteId, int serviceClass, AtomicBoolean fallbackToSimpleAuth) throws IOException { final Call call = createCall(rpcKind, rpcRequest); - Connection connection = getConnection(remoteId, call, serviceClass, - fallbackToSimpleAuth); + final Connection connection = getConnection(remoteId, call, serviceClass, + fallbackToSimpleAuth); try { connection.sendRpcRequest(call); // send the rpc request } catch (RejectedExecutionException e) { @@ -1366,6 +1382,51 @@ public class Client { throw new IOException(e); } + if (isAsynchronousMode()) { + Future returnFuture = new AbstractFuture() { + @Override + public Writable get() throws InterruptedException, ExecutionException { + try { + set(getRpcResponse(call, connection)); + } catch (IOException ie) { + setException(ie); + } + return super.get(); + } + }; + + returnValue.set(returnFuture); + return null; + } else { + return getRpcResponse(call, connection); + } + } + + /** + * Check if RPC is in asynchronous mode or not. + * + * @returns true, if RPC is in asynchronous mode, otherwise false for + * synchronous mode. + */ + @Unstable + static boolean isAsynchronousMode() { + return asynchronousMode.get(); + } + + /** + * Set RPC to asynchronous or synchronous mode. + * + * @param async + * true, RPC will be in asynchronous mode, otherwise false for + * synchronous mode + */ + @Unstable + public static void setAsynchronousMode(boolean async) { + asynchronousMode.set(async); + } + + private Writable getRpcResponse(final Call call, final Connection connection) + throws IOException { synchronized (call) { while (!call.done) { try { @@ -1640,4 +1701,10 @@ public class Client { public static int nextCallId() { return callIdCounter.getAndIncrement() & 0x7FFFFFFF; } + + @Override + @Unstable + public void close() throws Exception { + stop(); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestAsyncIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestAsyncIPC.java new file mode 100644 index 00000000000..de4395e8f25 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestAsyncIPC.java @@ -0,0 +1,346 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.ipc.RPC.RpcKind; +import org.apache.hadoop.ipc.TestIPC.CallInfo; +import org.apache.hadoop.ipc.TestIPC.TestServer; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAsyncIPC { + + private static Configuration conf; + private static final Log LOG = LogFactory.getLog(TestAsyncIPC.class); + + @Before + public void setupConf() { + conf = new Configuration(); + Client.setPingInterval(conf, TestIPC.PING_INTERVAL); + // set asynchronous mode for main thread + Client.setAsynchronousMode(true); + } + + protected static class SerialCaller extends Thread { + private Client client; + private InetSocketAddress server; + private int count; + private boolean failed; + Map> returnFutures = + new HashMap>(); + Map expectedValues = new HashMap(); + + public SerialCaller(Client client, InetSocketAddress server, int count) { + this.client = client; + this.server = server; + this.count = count; + // set asynchronous mode, since SerialCaller extends Thread + Client.setAsynchronousMode(true); + } + + @Override + public void run() { + // in case Thread#Start is called, which will spawn new thread + Client.setAsynchronousMode(true); + for (int i = 0; i < count; i++) { + try { + final long param = TestIPC.RANDOM.nextLong(); + TestIPC.call(client, param, server, conf); + Future returnFuture = Client.getReturnValue(); + returnFutures.put(i, returnFuture); + expectedValues.put(i, param); + } catch (Exception e) { + LOG.fatal("Caught: " + StringUtils.stringifyException(e)); + failed = true; + } + } + } + + public void waitForReturnValues() throws InterruptedException, + ExecutionException { + for (int i = 0; i < count; i++) { + LongWritable value = returnFutures.get(i).get(); + if (expectedValues.get(i) != value.get()) { + LOG.fatal(String.format("Call-%d failed!", i)); + failed = true; + break; + } + } + } + } + + @Test + public void testSerial() throws IOException, InterruptedException, + ExecutionException { + internalTestSerial(3, false, 2, 5, 100); + internalTestSerial(3, true, 2, 5, 10); + } + + public void internalTestSerial(int handlerCount, boolean handlerSleep, + int clientCount, int callerCount, int callCount) throws IOException, + InterruptedException, ExecutionException { + Server server = new TestIPC.TestServer(handlerCount, handlerSleep, conf); + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + + Client[] clients = new Client[clientCount]; + for (int i = 0; i < clientCount; i++) { + clients[i] = new Client(LongWritable.class, conf); + } + + SerialCaller[] callers = new SerialCaller[callerCount]; + for (int i = 0; i < callerCount; i++) { + callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount); + callers[i].start(); + } + for (int i = 0; i < callerCount; i++) { + callers[i].join(); + callers[i].waitForReturnValues(); + String msg = String.format("Expected not failed for caller-%d: %s.", i, + callers[i]); + assertFalse(msg, callers[i].failed); + } + for (int i = 0; i < clientCount; i++) { + clients[i].stop(); + } + server.stop(); + } + + /** + * Test if (1) the rpc server uses the call id/retry provided by the rpc + * client, and (2) the rpc client receives the same call id/retry from the rpc + * server. + * + * @throws ExecutionException + * @throws InterruptedException + */ + @Test(timeout = 60000) + public void testCallIdAndRetry() throws IOException, InterruptedException, + ExecutionException { + final Map infoMap = new HashMap(); + + // Override client to store the call info and check response + final Client client = new Client(LongWritable.class, conf) { + @Override + Call createCall(RpcKind rpcKind, Writable rpcRequest) { + // Set different call id and retry count for the next call + Client.setCallIdAndRetryCount(Client.nextCallId(), + TestIPC.RANDOM.nextInt(255)); + + final Call call = super.createCall(rpcKind, rpcRequest); + + CallInfo info = new CallInfo(); + info.id = call.id; + info.retry = call.retry; + infoMap.put(call.id, info); + + return call; + } + + @Override + void checkResponse(RpcResponseHeaderProto header) throws IOException { + super.checkResponse(header); + Assert.assertEquals(infoMap.get(header.getCallId()).retry, + header.getRetryCount()); + } + }; + + // Attach a listener that tracks every call received by the server. + final TestServer server = new TestIPC.TestServer(1, false, conf); + server.callListener = new Runnable() { + @Override + public void run() { + Assert.assertEquals(infoMap.get(Server.getCallId()).retry, + Server.getCallRetryCount()); + } + }; + + try { + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + final SerialCaller caller = new SerialCaller(client, addr, 4); + caller.run(); + caller.waitForReturnValues(); + String msg = String.format("Expected not failed for caller: %s.", caller); + assertFalse(msg, caller.failed); + } finally { + client.stop(); + server.stop(); + } + } + + /** + * Test if the rpc server gets the retry count from client. + * + * @throws ExecutionException + * @throws InterruptedException + */ + @Test(timeout = 60000) + public void testCallRetryCount() throws IOException, InterruptedException, + ExecutionException { + final int retryCount = 255; + // Override client to store the call id + final Client client = new Client(LongWritable.class, conf); + Client.setCallIdAndRetryCount(Client.nextCallId(), retryCount); + + // Attach a listener that tracks every call ID received by the server. + final TestServer server = new TestIPC.TestServer(1, false, conf); + server.callListener = new Runnable() { + @Override + public void run() { + // we have not set the retry count for the client, thus on the server + // side we should see retry count as 0 + Assert.assertEquals(retryCount, Server.getCallRetryCount()); + } + }; + + try { + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + final SerialCaller caller = new SerialCaller(client, addr, 10); + caller.run(); + caller.waitForReturnValues(); + String msg = String.format("Expected not failed for caller: %s.", caller); + assertFalse(msg, caller.failed); + } finally { + client.stop(); + server.stop(); + } + } + + /** + * Test if the rpc server gets the default retry count (0) from client. + * + * @throws ExecutionException + * @throws InterruptedException + */ + @Test(timeout = 60000) + public void testInitialCallRetryCount() throws IOException, + InterruptedException, ExecutionException { + // Override client to store the call id + final Client client = new Client(LongWritable.class, conf); + + // Attach a listener that tracks every call ID received by the server. + final TestServer server = new TestIPC.TestServer(1, false, conf); + server.callListener = new Runnable() { + @Override + public void run() { + // we have not set the retry count for the client, thus on the server + // side we should see retry count as 0 + Assert.assertEquals(0, Server.getCallRetryCount()); + } + }; + + try { + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + final SerialCaller caller = new SerialCaller(client, addr, 10); + caller.run(); + caller.waitForReturnValues(); + String msg = String.format("Expected not failed for caller: %s.", caller); + assertFalse(msg, caller.failed); + } finally { + client.stop(); + server.stop(); + } + } + + /** + * Tests that client generates a unique sequential call ID for each RPC call, + * even if multiple threads are using the same client. + * + * @throws InterruptedException + * @throws ExecutionException + */ + @Test(timeout = 60000) + public void testUniqueSequentialCallIds() throws IOException, + InterruptedException, ExecutionException { + int serverThreads = 10, callerCount = 100, perCallerCallCount = 100; + TestServer server = new TestIPC.TestServer(serverThreads, false, conf); + + // Attach a listener that tracks every call ID received by the server. This + // list must be synchronized, because multiple server threads will add to + // it. + final List callIds = Collections + .synchronizedList(new ArrayList()); + server.callListener = new Runnable() { + @Override + public void run() { + callIds.add(Server.getCallId()); + } + }; + + Client client = new Client(LongWritable.class, conf); + + try { + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + SerialCaller[] callers = new SerialCaller[callerCount]; + for (int i = 0; i < callerCount; ++i) { + callers[i] = new SerialCaller(client, addr, perCallerCallCount); + callers[i].start(); + } + for (int i = 0; i < callerCount; ++i) { + callers[i].join(); + callers[i].waitForReturnValues(); + String msg = String.format("Expected not failed for caller-%d: %s.", i, + callers[i]); + assertFalse(msg, callers[i].failed); + } + } finally { + client.stop(); + server.stop(); + } + + int expectedCallCount = callerCount * perCallerCallCount; + assertEquals(expectedCallCount, callIds.size()); + + // It is not guaranteed that the server executes requests in sequential + // order + // of client call ID, so we must sort the call IDs before checking that it + // contains every expected value. + Collections.sort(callIds); + final int startID = callIds.get(0).intValue(); + for (int i = 0; i < expectedCallCount; ++i) { + assertEquals(startID + i, callIds.get(i).intValue()); + } + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index d65818238eb..6bfcc537da4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -99,7 +99,7 @@ public class TestIPC { LogFactory.getLog(TestIPC.class); private static Configuration conf; - final static private int PING_INTERVAL = 1000; + final static int PING_INTERVAL = 1000; final static private int MIN_SLEEP_TIME = 1000; /** * Flag used to turn off the fault injection behavior @@ -114,7 +114,7 @@ public class TestIPC { Client.setPingInterval(conf, PING_INTERVAL); } - private static final Random RANDOM = new Random(); + static final Random RANDOM = new Random(); private static final String ADDRESS = "0.0.0.0"; @@ -148,22 +148,33 @@ public class TestIPC { RPC.RPC_SERVICE_CLASS_DEFAULT, null); } - private static class TestServer extends Server { + static class TestServer extends Server { // Tests can set callListener to run a piece of code each time the server // receives a call. This code executes on the server thread, so it has // visibility of that thread's thread-local storage. - private Runnable callListener; + Runnable callListener; private boolean sleep; private Class responseClass; public TestServer(int handlerCount, boolean sleep) throws IOException { this(handlerCount, sleep, LongWritable.class, null); } - + + public TestServer(int handlerCount, boolean sleep, Configuration conf) + throws IOException { + this(handlerCount, sleep, LongWritable.class, null, conf); + } + public TestServer(int handlerCount, boolean sleep, Class paramClass, - Class responseClass) - throws IOException { + Class responseClass) throws IOException { + this(handlerCount, sleep, paramClass, responseClass, conf); + } + + public TestServer(int handlerCount, boolean sleep, + Class paramClass, + Class responseClass, Configuration conf) + throws IOException { super(ADDRESS, 0, paramClass, handlerCount, conf); this.sleep = sleep; this.responseClass = responseClass; @@ -1070,7 +1081,7 @@ public class TestIPC { assertRetriesOnSocketTimeouts(conf, 4); } - private static class CallInfo { + static class CallInfo { int id = RpcConstants.INVALID_CALL_ID; int retry = RpcConstants.INVALID_RETRY_COUNT; } @@ -1125,7 +1136,7 @@ public class TestIPC { } /** A dummy protocol */ - private interface DummyProtocol { + interface DummyProtocol { @Idempotent public void dummyRun() throws IOException; }