From 817a26b6c8518674bce326b85b5a65e538476f0c Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 7 Nov 2012 06:29:37 +0000 Subject: [PATCH 01/31] HDFS-4153. Add START_MSG/SHUTDOWN_MSG for JournalNode. Contributed by liang xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406473 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/qjournal/server/JournalNode.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b2766bd3b0c..2342fd91938 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -157,6 +157,8 @@ Trunk (Unreleased) HDFS-4152. Add a new class BlocksMapUpdateInfo for the parameter in INode.collectSubtreeBlocksAndClear(..). (Jing Zhao via szetszwo) + HDFS-4153. Add START_MSG/SHUTDOWN_MSG for JournalNode. (liang xie via atm) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index e70f4690efc..91629d930c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -35,6 +35,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -230,6 +231,7 @@ public class JournalNode implements Tool, Configurable { } public static void main(String[] args) throws Exception { + StringUtils.startupShutdownMessage(JournalNode.class, args, LOG); System.exit(ToolRunner.run(new JournalNode(), args)); } } From 8d483fff2ce71006a4390fd1938f7dcf0a0978b1 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 7 Nov 2012 14:11:59 +0000 Subject: [PATCH 02/31] MAPREDUCE-4777. In TestIFile, testIFileReaderWithCodec relies on testIFileWriterWithCodec. Contributed by Sandy Ryza git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/test/java/org/apache/hadoop/mapred/TestIFile.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 644796d4bd8..258a07ca75b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -194,6 +194,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-1806. CombineFileInputFormat does not work with paths not on default FS. (Gera Shegalov via tucu) + MAPREDUCE-4777. In TestIFile, testIFileReaderWithCodec relies on + testIFileWriterWithCodec. (Sandy Ryza via tomwhite) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java index 0411711b5fa..3514b15becf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java @@ -56,6 +56,10 @@ public class TestIFile { Path path = new Path(new Path("build/test.ifile"), "data"); DefaultCodec codec = new GzipCodec(); codec.setConf(conf); + IFile.Writer writer = + new IFile.Writer(conf, rfs, path, Text.class, Text.class, + codec, null); + writer.close(); IFile.Reader reader = new IFile.Reader(conf, rfs, path, codec, null); reader.close(); From cb293a62a52bd780488176bd924fa11c8bb7839c Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 7 Nov 2012 15:56:46 +0000 Subject: [PATCH 03/31] HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406684 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/security/UserGroupInformation.java | 26 ++++++---- .../org/apache/hadoop/ipc/TestSaslRPC.java | 50 ++++++++++++------- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 669c9e64d82..65b6ceb357a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -342,6 +342,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9010. Map UGI authenticationMethod to RPC authMethod (daryn via bobby) + HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn + via bobby) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 88f82912025..f630d695a90 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -238,14 +238,17 @@ public class UserGroupInformation { */ private static synchronized void initUGI(Configuration conf) { AuthenticationMethod auth = SecurityUtil.getAuthenticationMethod(conf); - if (auth == AuthenticationMethod.SIMPLE) { - useKerberos = false; - } else if (auth == AuthenticationMethod.KERBEROS) { - useKerberos = true; - } else { - throw new IllegalArgumentException("Invalid attribute value for " + - HADOOP_SECURITY_AUTHENTICATION + - " of " + auth); + switch (auth) { + case SIMPLE: + useKerberos = false; + break; + case KERBEROS: + useKerberos = true; + break; + default: + throw new IllegalArgumentException("Invalid attribute value for " + + HADOOP_SECURITY_AUTHENTICATION + + " of " + auth); } try { kerberosMinSecondsBeforeRelogin = 1000L * conf.getLong( @@ -637,19 +640,20 @@ public class UserGroupInformation { try { Subject subject = new Subject(); LoginContext login; + AuthenticationMethod authenticationMethod; if (isSecurityEnabled()) { + authenticationMethod = AuthenticationMethod.KERBEROS; login = newLoginContext(HadoopConfiguration.USER_KERBEROS_CONFIG_NAME, subject, new HadoopConfiguration()); } else { + authenticationMethod = AuthenticationMethod.SIMPLE; login = newLoginContext(HadoopConfiguration.SIMPLE_CONFIG_NAME, subject, new HadoopConfiguration()); } login.login(); loginUser = new UserGroupInformation(subject); loginUser.setLogin(login); - loginUser.setAuthenticationMethod(isSecurityEnabled() ? - AuthenticationMethod.KERBEROS : - AuthenticationMethod.SIMPLE); + loginUser.setAuthenticationMethod(authenticationMethod); loginUser = new UserGroupInformation(login.getSubject()); String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); if (fileLocation != null) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java index 7abd6e9dacb..848de86132b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java @@ -43,14 +43,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.KerberosInfo; -import org.apache.hadoop.security.SaslInputStream; -import org.apache.hadoop.security.SaslRpcClient; -import org.apache.hadoop.security.SaslRpcServer; -import org.apache.hadoop.security.SecurityInfo; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.TestUserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.*; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; @@ -58,8 +51,10 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenInfo; import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.security.token.SecretManager.InvalidToken; + import org.apache.log4j.Level; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; /** Unit tests for using Sasl over RPC. */ @@ -76,6 +71,12 @@ public class TestSaslRPC { static final String SERVER_PRINCIPAL_2 = "p2/foo@BAR"; private static Configuration conf; + + @BeforeClass + public static void setupKerb() { + System.setProperty("java.security.krb5.kdc", ""); + System.setProperty("java.security.krb5.realm", "NONE"); + } @Before public void setup() { @@ -539,21 +540,39 @@ public class TestSaslRPC { final boolean useToken, final boolean useValidToken) throws Exception { - Configuration serverConf = new Configuration(conf); + String currentUser = UserGroupInformation.getCurrentUser().getUserName(); + + final Configuration serverConf = new Configuration(conf); SecurityUtil.setAuthenticationMethod(serverAuth, serverConf); UserGroupInformation.setConfiguration(serverConf); - TestTokenSecretManager sm = new TestTokenSecretManager(); - Server server = new RPC.Builder(serverConf).setProtocol(TestSaslProtocol.class) + final UserGroupInformation serverUgi = + UserGroupInformation.createRemoteUser(currentUser + "-SERVER"); + serverUgi.setAuthenticationMethod(serverAuth); + + final TestTokenSecretManager sm = new TestTokenSecretManager(); + Server server = serverUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Server run() throws IOException { + Server server = new RPC.Builder(serverConf) + .setProtocol(TestSaslProtocol.class) .setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0) .setNumHandlers(5).setVerbose(true) .setSecretManager((serverAuth != SIMPLE) ? sm : null) .build(); - server.start(); + server.start(); + return server; + } + }); + final Configuration clientConf = new Configuration(conf); + SecurityUtil.setAuthenticationMethod(clientAuth, clientConf); + UserGroupInformation.setConfiguration(clientConf); + final UserGroupInformation clientUgi = - UserGroupInformation.createRemoteUser( - UserGroupInformation.getCurrentUser().getUserName()+"-CLIENT"); + UserGroupInformation.createRemoteUser(currentUser + "-CLIENT"); + clientUgi.setAuthenticationMethod(clientAuth); + final InetSocketAddress addr = NetUtils.getConnectAddress(server); if (useToken) { TestTokenIdentifier tokenId = new TestTokenIdentifier( @@ -568,9 +587,6 @@ public class TestSaslRPC { clientUgi.addToken(token); } - final Configuration clientConf = new Configuration(conf); - SecurityUtil.setAuthenticationMethod(clientAuth, clientConf); - UserGroupInformation.setConfiguration(clientConf); try { return clientUgi.doAs(new PrivilegedExceptionAction() { From 3c02237459359463c32df228c5e525bd1cf94e29 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 7 Nov 2012 16:05:03 +0000 Subject: [PATCH 04/31] HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406689 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../apache/hadoop/security/SaslRpcClient.java | 68 +++++++++++-------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 65b6ceb357a..5d87915b5fb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -345,6 +345,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn via bobby) + HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java index 98b3f5db295..8365fa7ccd0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java @@ -25,6 +25,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Map; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; @@ -45,6 +46,7 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SaslRpcServer.SaslStatus; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -69,40 +71,48 @@ public class SaslRpcClient { public SaslRpcClient(AuthMethod method, Token token, String serverPrincipal) throws IOException { + String saslUser = null; + String saslProtocol = null; + String saslServerName = null; + Map saslProperties = SaslRpcServer.SASL_PROPS; + CallbackHandler saslCallback = null; + switch (method) { - case DIGEST: - if (LOG.isDebugEnabled()) - LOG.debug("Creating SASL " + AuthMethod.DIGEST.getMechanismName() - + " client to authenticate to service at " + token.getService()); - saslClient = Sasl.createSaslClient(new String[] { AuthMethod.DIGEST - .getMechanismName() }, null, null, SaslRpcServer.SASL_DEFAULT_REALM, - SaslRpcServer.SASL_PROPS, new SaslClientCallbackHandler(token)); - break; - case KERBEROS: - if (LOG.isDebugEnabled()) { - LOG.debug("Creating SASL " + AuthMethod.KERBEROS.getMechanismName() - + " client. Server's Kerberos principal name is " - + serverPrincipal); + case DIGEST: { + saslServerName = SaslRpcServer.SASL_DEFAULT_REALM; + saslCallback = new SaslClientCallbackHandler(token); + break; } - if (serverPrincipal == null || serverPrincipal.length() == 0) { - throw new IOException( - "Failed to specify server's Kerberos principal name"); + case KERBEROS: { + if (serverPrincipal == null || serverPrincipal.isEmpty()) { + throw new IOException( + "Failed to specify server's Kerberos principal name"); + } + KerberosName name = new KerberosName(serverPrincipal); + saslProtocol = name.getServiceName(); + saslServerName = name.getHostName(); + if (saslServerName == null) { + throw new IOException( + "Kerberos principal name does NOT have the expected hostname part: " + + serverPrincipal); + } + break; } - String names[] = SaslRpcServer.splitKerberosName(serverPrincipal); - if (names.length != 3) { - throw new IOException( - "Kerberos principal name does NOT have the expected hostname part: " - + serverPrincipal); - } - saslClient = Sasl.createSaslClient(new String[] { AuthMethod.KERBEROS - .getMechanismName() }, null, names[0], names[1], - SaslRpcServer.SASL_PROPS, null); - break; - default: - throw new IOException("Unknown authentication method " + method); + default: + throw new IOException("Unknown authentication method " + method); } - if (saslClient == null) + + String mechanism = method.getMechanismName(); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating SASL " + mechanism + + " client to authenticate to service at " + saslServerName); + } + saslClient = Sasl.createSaslClient( + new String[] { mechanism }, saslUser, saslProtocol, saslServerName, + saslProperties, saslCallback); + if (saslClient == null) { throw new IOException("Unable to find SASL client implementation"); + } } private static void readStatus(DataInputStream inStream) throws IOException { From 1ab97373deea303e3f43af36b974db52e0ba6d13 Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Wed, 7 Nov 2012 17:45:01 +0000 Subject: [PATCH 05/31] HDFS-4138. BackupNode startup fails due to uninitialized edit log. Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406734 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/BackupNode.java | 18 ++---------------- .../hdfs/server/namenode/FSNamesystem.java | 2 +- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2342fd91938..5b5863986b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1966,6 +1966,9 @@ Release 0.23.5 - UNRELEASED HDFS-4090. getFileChecksum() result incompatible when called against zero-byte files. (Kihwal Lee via daryn) + HDFS-4138. BackupNode startup fails due to uninitialized edit log. + (Kihwal Lee via shv) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index cfc841902c2..0d93227d70c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -78,10 +78,6 @@ public class BackupNode extends NameNode { String nnHttpAddress; /** Checkpoint manager */ Checkpointer checkpointManager; - /** ClusterID to which BackupNode belongs to */ - String clusterId; - /** Block pool Id of the peer namenode of this BackupNode */ - String blockPoolId; BackupNode(Configuration conf, NamenodeRole role) throws IOException { super(conf, role); @@ -145,6 +141,7 @@ public class BackupNode extends NameNode { CommonConfigurationKeys.FS_TRASH_INTERVAL_DEFAULT); NamespaceInfo nsInfo = handshake(conf); super.initialize(conf); + namesystem.setBlockPoolId(nsInfo.getBlockPoolID()); if (false == namesystem.isInSafeMode()) { namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); @@ -154,9 +151,6 @@ public class BackupNode extends NameNode { // therefore lease hard limit should never expire. namesystem.leaseManager.setLeasePeriod( HdfsConstants.LEASE_SOFTLIMIT_PERIOD, Long.MAX_VALUE); - - clusterId = nsInfo.getClusterID(); - blockPoolId = nsInfo.getBlockPoolID(); // register with the active name-node registerWith(nsInfo); @@ -219,7 +213,7 @@ public class BackupNode extends NameNode { } /* @Override */// NameNode - public boolean setSafeMode(@SuppressWarnings("unused") SafeModeAction action) + public boolean setSafeMode(SafeModeAction action) throws IOException { throw new UnsupportedActionException("setSafeMode"); } @@ -415,14 +409,6 @@ public class BackupNode extends NameNode { return nsInfo; } - String getBlockPoolId() { - return blockPoolId; - } - - String getClusterId() { - return clusterId; - } - @Override protected NameNodeHAContext createHAContext() { return new BNHAContext(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 106718fe2f4..e24603018b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3507,7 +3507,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, @Metric({"TransactionsSinceLastLogRoll", "Number of transactions since last edit log roll"}) public long getTransactionsSinceLastLogRoll() { - if (isInStandbyState()) { + if (isInStandbyState() || !getEditLog().isSegmentOpen()) { return 0; } else { return getEditLog().getLastWrittenTxId() - From c44f37a7a55216eb98c32e1ab9958bc5120214fc Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Wed, 7 Nov 2012 17:59:35 +0000 Subject: [PATCH 06/31] Move HDFS-3625 under 0.23.5 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406739 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5b5863986b7..67c3fbb156c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -225,9 +225,6 @@ Trunk (Unreleased) HDFS-3614. Revert unused MiniDFSCluster constructor from HDFS-3049. (acmurthy via eli) - HDFS-3625. Fix TestBackupNode by properly initializing edit log during - startup. (Junping Du via todd) - HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd) HDFS-3827. TestHASafeMode#assertSafemode method should be made static. @@ -1966,6 +1963,9 @@ Release 0.23.5 - UNRELEASED HDFS-4090. getFileChecksum() result incompatible when called against zero-byte files. (Kihwal Lee via daryn) + HDFS-3625. Fix TestBackupNode by properly initializing edit log during + startup. (Junping Du via todd) + HDFS-4138. BackupNode startup fails due to uninitialized edit log. (Kihwal Lee via shv) From fb5b96dfc324f999e8b3698288c110a1c3b71c30 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 7 Nov 2012 21:56:18 +0000 Subject: [PATCH 07/31] YARN-201. Fix CapacityScheduler to be less conservative for starved off-switch requests. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406834 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/scheduler/capacity/LeafQueue.java | 8 ++++++-- .../resourcemanager/scheduler/capacity/TestLeafQueue.java | 8 ++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a6102b216c4..7d1dce459ff 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -202,6 +202,9 @@ Release 0.23.5 - UNRELEASED YARN-202. Log Aggregation generates a storm of fsync() for namenode (Kihwal Lee via bobby) + YARN-201. Fix CapacityScheduler to be less conservative for starved + off-switch requests. (jlowe via acmurthy) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index e183a2d7020..28a81f7af26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -833,8 +833,12 @@ public class LeafQueue implements CSQueue { // Note: Update headroom to account for current allocation too... allocateResource(clusterResource, application, assigned); - // Reset scheduling opportunities - application.resetSchedulingOpportunities(priority); + // Don't reset scheduling opportunities for non-local assignments + // otherwise the app will be delayed for each non-local assignment. + // This helps apps with many off-cluster requests schedule faster. + if (assignment.getType() != NodeType.OFF_SWITCH) { + application.resetSchedulingOpportunities(priority); + } // Done return assignment; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index f9d6c60479f..868c2bbce87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1259,7 +1259,7 @@ public class TestLeafQueue { assignment = a.assignContainers(clusterResource, node_2); verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), any(Priority.class), any(ResourceRequest.class), any(Container.class)); - assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset + assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset assertEquals(2, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.OFF_SWITCH, assignment.getType()); @@ -1408,11 +1408,11 @@ public class TestLeafQueue { assertEquals(0, app_0.getSchedulingOpportunities(priority_2)); assertEquals(1, app_0.getTotalRequiredResources(priority_2)); - // Another off-switch, shouldn allocate OFF_SWITCH P1 + // Another off-switch, shouldn't allocate OFF_SWITCH P1 a.assignContainers(clusterResource, node_2); verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), eq(priority_1), any(ResourceRequest.class), any(Container.class)); - assertEquals(0, app_0.getSchedulingOpportunities(priority_1)); + assertEquals(3, app_0.getSchedulingOpportunities(priority_1)); assertEquals(1, app_0.getTotalRequiredResources(priority_1)); verify(app_0, never()).allocate(any(NodeType.class), eq(node_2), eq(priority_2), any(ResourceRequest.class), any(Container.class)); @@ -1438,7 +1438,7 @@ public class TestLeafQueue { assertEquals(0, app_0.getTotalRequiredResources(priority_1)); verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1), eq(priority_2), any(ResourceRequest.class), any(Container.class)); - assertEquals(0, app_0.getSchedulingOpportunities(priority_2)); + assertEquals(1, app_0.getSchedulingOpportunities(priority_2)); assertEquals(0, app_0.getTotalRequiredResources(priority_2)); } From 1594dd6965c412a08339a0079ee1416dd6f595f9 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 7 Nov 2012 22:50:48 +0000 Subject: [PATCH 08/31] HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406851 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../java/org/apache/hadoop/ipc/Server.java | 157 ++++++++++-------- .../org/apache/hadoop/ipc/TestSaslRPC.java | 44 +++-- 3 files changed, 124 insertions(+), 79 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5d87915b5fb..fd5faf7f632 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -347,6 +347,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby) + HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index efaf6028ea9..60155e71ce8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -57,6 +57,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; +import javax.security.auth.callback.CallbackHandler; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; import javax.security.sasl.SaslServer; @@ -87,6 +88,7 @@ import org.apache.hadoop.security.SaslRpcServer.SaslDigestCallbackHandler; import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler; import org.apache.hadoop.security.SaslRpcServer.SaslStatus; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; @@ -1078,7 +1080,6 @@ public abstract class Server { IpcConnectionContextProto connectionContext; String protocolName; - boolean useSasl; SaslServer saslServer; private AuthMethod authMethod; private boolean saslContextEstablished; @@ -1194,49 +1195,6 @@ public abstract class Server { if (!saslContextEstablished) { byte[] replyToken = null; try { - if (saslServer == null) { - switch (authMethod) { - case DIGEST: - if (secretManager == null) { - throw new AccessControlException( - "Server is not configured to do DIGEST authentication."); - } - secretManager.checkAvailableForRead(); - saslServer = Sasl.createSaslServer(AuthMethod.DIGEST - .getMechanismName(), null, SaslRpcServer.SASL_DEFAULT_REALM, - SaslRpcServer.SASL_PROPS, new SaslDigestCallbackHandler( - secretManager, this)); - break; - default: - UserGroupInformation current = UserGroupInformation - .getCurrentUser(); - String fullName = current.getUserName(); - if (LOG.isDebugEnabled()) - LOG.debug("Kerberos principal name is " + fullName); - final String names[] = SaslRpcServer.splitKerberosName(fullName); - if (names.length != 3) { - throw new AccessControlException( - "Kerberos principal name does NOT have the expected " - + "hostname part: " + fullName); - } - current.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws SaslException { - saslServer = Sasl.createSaslServer(AuthMethod.KERBEROS - .getMechanismName(), names[0], names[1], - SaslRpcServer.SASL_PROPS, new SaslGssCallbackHandler()); - return null; - } - }); - } - if (saslServer == null) - throw new AccessControlException( - "Unable to find SASL server implementation for " - + authMethod.getMechanismName()); - if (LOG.isDebugEnabled()) - LOG.debug("Created SASL server with mechanism = " - + authMethod.getMechanismName()); - } if (LOG.isDebugEnabled()) LOG.debug("Have read input token of size " + saslToken.length + " for processing by saslServer.evaluateResponse()"); @@ -1375,38 +1333,27 @@ public abstract class Server { dataLengthBuffer.clear(); if (authMethod == null) { throw new IOException("Unable to read authentication method"); - } + } + boolean useSaslServer = isSecurityEnabled; final boolean clientUsingSasl; switch (authMethod) { case SIMPLE: { // no sasl for simple - if (isSecurityEnabled) { - AccessControlException ae = new AccessControlException("Authorization (" - + CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION - + ") is enabled but authentication (" - + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION - + ") is configured as simple. Please configure another method " - + "like kerberos or digest."); - setupResponse(authFailedResponse, authFailedCall, RpcStatusProto.FATAL, - null, ae.getClass().getName(), ae.getMessage()); - responder.doRespond(authFailedCall); - throw ae; - } clientUsingSasl = false; - useSasl = false; break; } - case DIGEST: { + case DIGEST: { // always allow tokens if there's a secret manager + useSaslServer |= (secretManager != null); clientUsingSasl = true; - useSasl = (secretManager != null); break; } default: { clientUsingSasl = true; - useSasl = isSecurityEnabled; break; } - } - if (clientUsingSasl && !useSasl) { + } + if (useSaslServer) { + saslServer = createSaslServer(authMethod); + } else if (clientUsingSasl) { // security is off doSaslReply(SaslStatus.SUCCESS, new IntWritable( SaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null); authMethod = AuthMethod.SIMPLE; @@ -1448,7 +1395,7 @@ public abstract class Server { continue; } boolean isHeaderRead = connectionContextRead; - if (useSasl) { + if (saslServer != null) { saslReadAndProcess(data.array()); } else { processOneRpc(data.array()); @@ -1462,6 +1409,84 @@ public abstract class Server { } } + private SaslServer createSaslServer(AuthMethod authMethod) + throws IOException { + try { + return createSaslServerInternal(authMethod); + } catch (IOException ioe) { + final String ioeClass = ioe.getClass().getName(); + final String ioeMessage = ioe.getLocalizedMessage(); + if (authMethod == AuthMethod.SIMPLE) { + setupResponse(authFailedResponse, authFailedCall, + RpcStatusProto.FATAL, null, ioeClass, ioeMessage); + responder.doRespond(authFailedCall); + } else { + doSaslReply(SaslStatus.ERROR, null, ioeClass, ioeMessage); + } + throw ioe; + } + } + + private SaslServer createSaslServerInternal(AuthMethod authMethod) + throws IOException { + SaslServer saslServer = null; + String hostname = null; + String saslProtocol = null; + CallbackHandler saslCallback = null; + + switch (authMethod) { + case SIMPLE: { + throw new AccessControlException("Authorization (" + + CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION + + ") is enabled but authentication (" + + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + + ") is configured as simple. Please configure another method " + + "like kerberos or digest."); + } + case DIGEST: { + if (secretManager == null) { + throw new AccessControlException( + "Server is not configured to do DIGEST authentication."); + } + secretManager.checkAvailableForRead(); + hostname = SaslRpcServer.SASL_DEFAULT_REALM; + saslCallback = new SaslDigestCallbackHandler(secretManager, this); + break; + } + case KERBEROS: { + String fullName = UserGroupInformation.getCurrentUser().getUserName(); + if (LOG.isDebugEnabled()) + LOG.debug("Kerberos principal name is " + fullName); + KerberosName krbName = new KerberosName(fullName); + hostname = krbName.getHostName(); + if (hostname == null) { + throw new AccessControlException( + "Kerberos principal name does NOT have the expected " + + "hostname part: " + fullName); + } + saslProtocol = krbName.getServiceName(); + saslCallback = new SaslGssCallbackHandler(); + break; + } + default: + throw new AccessControlException( + "Server does not support SASL " + authMethod); + } + + String mechanism = authMethod.getMechanismName(); + saslServer = Sasl.createSaslServer( + mechanism, saslProtocol, hostname, + SaslRpcServer.SASL_PROPS, saslCallback); + if (saslServer == null) { + throw new AccessControlException( + "Unable to find SASL server implementation for " + mechanism); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Created SASL server with mechanism = " + mechanism); + } + return saslServer; + } + /** * Try to set up the response to indicate that the client version * is incompatible with the server. This can contain special-case @@ -1523,7 +1548,7 @@ public abstract class Server { .getProtocol() : null; UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext); - if (!useSasl) { + if (saslServer == null) { user = protocolUser; if (user != null) { user.setAuthenticationMethod(AuthMethod.SIMPLE); @@ -1999,7 +2024,7 @@ public abstract class Server { private void wrapWithSasl(ByteArrayOutputStream response, Call call) throws IOException { - if (call.connection.useSasl) { + if (call.connection.saslServer != null) { byte[] token = response.toByteArray(); // synchronization may be needed since there can be multiple Handler // threads using saslServer to wrap responses. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java index 848de86132b..68dd2c83930 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java @@ -53,6 +53,7 @@ import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.log4j.Level; +import org.apache.tools.ant.types.Assertions.EnabledAssertion; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -69,8 +70,8 @@ public class TestSaslRPC { static final String SERVER_KEYTAB_KEY = "test.ipc.server.keytab"; static final String SERVER_PRINCIPAL_1 = "p1/foo@BAR"; static final String SERVER_PRINCIPAL_2 = "p2/foo@BAR"; - private static Configuration conf; + static Boolean forceSecretManager = null; @BeforeClass public static void setupKerb() { @@ -83,6 +84,7 @@ public class TestSaslRPC { conf = new Configuration(); SecurityUtil.setAuthenticationMethod(KERBEROS, conf); UserGroupInformation.setConfiguration(conf); + forceSecretManager = null; } static { @@ -266,16 +268,6 @@ public class TestSaslRPC { } } - @Test - public void testSecureToInsecureRpc() throws Exception { - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.SIMPLE, conf); - Server server = new RPC.Builder(conf).setProtocol(TestSaslProtocol.class) - .setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0) - .setNumHandlers(5).setVerbose(true).build(); - TestTokenSecretManager sm = new TestTokenSecretManager(); - doDigestRpc(server, sm); - } - @Test public void testErrorMessage() throws Exception { BadTokenSecretManager sm = new BadTokenSecretManager(); @@ -463,6 +455,8 @@ public class TestSaslRPC { "Failed to specify server's Kerberos principal name.*"); private static Pattern Denied = Pattern.compile(".*Authorization .* is enabled .*"); + private static Pattern NoDigest = + Pattern.compile(".*Server is not configured to do DIGEST auth.*"); /* * simple server @@ -479,6 +473,9 @@ public class TestSaslRPC { // Tokens are ignored because client is reverted to simple assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE, SIMPLE, true)); assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, true)); + forceSecretManager = true; + assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE, SIMPLE, true)); + assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, true)); } @Test @@ -486,6 +483,9 @@ public class TestSaslRPC { // Tokens are ignored because client is reverted to simple assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE, SIMPLE, false)); assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, false)); + forceSecretManager = true; + assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE, SIMPLE, false)); + assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, false)); } /* @@ -502,12 +502,19 @@ public class TestSaslRPC { // can use tokens regardless of auth assertAuthEquals(TOKEN, getAuthMethod(SIMPLE, KERBEROS, true)); assertAuthEquals(TOKEN, getAuthMethod(KERBEROS, KERBEROS, true)); + // can't fallback to simple when using kerberos w/o tokens + forceSecretManager = false; + assertAuthEquals(NoDigest, getAuthMethod(SIMPLE, KERBEROS, true)); + assertAuthEquals(NoDigest, getAuthMethod(KERBEROS, KERBEROS, true)); } @Test public void testKerberosServerWithInvalidTokens() throws Exception { assertAuthEquals(BadToken, getAuthMethod(SIMPLE, KERBEROS, false)); assertAuthEquals(BadToken, getAuthMethod(KERBEROS, KERBEROS, false)); + forceSecretManager = false; + assertAuthEquals(NoDigest, getAuthMethod(SIMPLE, KERBEROS, true)); + assertAuthEquals(NoDigest, getAuthMethod(KERBEROS, KERBEROS, true)); } @@ -551,6 +558,12 @@ public class TestSaslRPC { serverUgi.setAuthenticationMethod(serverAuth); final TestTokenSecretManager sm = new TestTokenSecretManager(); + boolean useSecretManager = (serverAuth != SIMPLE); + if (forceSecretManager != null) { + useSecretManager &= forceSecretManager.booleanValue(); + } + final SecretManager serverSm = useSecretManager ? sm : null; + Server server = serverUgi.doAs(new PrivilegedExceptionAction() { @Override public Server run() throws IOException { @@ -558,7 +571,7 @@ public class TestSaslRPC { .setProtocol(TestSaslProtocol.class) .setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0) .setNumHandlers(5).setVerbose(true) - .setSecretManager((serverAuth != SIMPLE) ? sm : null) + .setSecretManager(serverSm) .build(); server.start(); return server; @@ -587,7 +600,6 @@ public class TestSaslRPC { clientUgi.addToken(token); } - try { return clientUgi.doAs(new PrivilegedExceptionAction() { @Override @@ -597,6 +609,12 @@ public class TestSaslRPC { proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class, TestSaslProtocol.versionID, addr, clientConf); + proxy.ping(); + // verify sasl completed + if (serverAuth != SIMPLE) { + assertEquals(SaslRpcServer.SASL_PROPS.get(Sasl.QOP), "auth"); + } + // make sure the other side thinks we are who we said we are!!! assertEquals(clientUgi.getUserName(), proxy.getAuthUser()); return proxy.getAuthMethod().toString(); From 76d43a08edd3cd3bb07903b9ee930218b55bd12e Mon Sep 17 00:00:00 2001 From: Sanjay Radia Date: Thu, 8 Nov 2012 06:16:02 +0000 Subject: [PATCH 09/31] HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406939 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/fs/DelegateToFileSystem.java | 5 ++ .../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 -- .../hadoop/fs/viewfs/ViewFileSystem.java | 5 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 5 +- .../hadoop/fs/FileSystemTestHelper.java | 13 ++- .../fs/viewfs/TestChRootedFileSystem.java | 4 +- .../hadoop/fs/viewfs/TestChRootedFs.java | 4 +- .../viewfs/TestFcMainOperationsLocalFs.java | 43 +-------- .../fs/viewfs/ViewFileSystemTestSetup.java | 75 ++++++++++++---- .../hadoop/fs/viewfs/ViewFsTestSetup.java | 88 ++++++++++++++----- 11 files changed, 154 insertions(+), 97 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fd5faf7f632..db09a93efa9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -272,10 +272,13 @@ Trunk (Unreleased) HADOOP-8918. test-patch.sh is parsing modified files wrong. (Raja Aluri via suresh) + HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) + HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia) Release 2.0.3-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 962847154ac..1293448eea3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -125,6 +125,11 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { public FsServerDefaults getServerDefaults() throws IOException { return fsImpl.getServerDefaults(); } + + @Override + public Path getHomeDirectory() { + return fsImpl.getHomeDirectory(); + } @Override public int getUriDefaultPort() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index b3acf506f6a..e4988efeaff 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -153,12 +153,6 @@ class ChRootedFileSystem extends FilterFileSystem { return makeQualified( new Path(chRootPathPartString + f.toUri().toString())); } - - @Override - public Path getHomeDirectory() { - return new Path("/user/"+System.getProperty("user.name")).makeQualified( - getUri(), null); - } @Override public Path getWorkingDirectory() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index d4de01efa0e..f4fbc66b530 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -256,8 +256,9 @@ public class ViewFileSystem extends FileSystem { if (base == null) { base = "/user"; } - homeDir = - this.makeQualified(new Path(base + "/" + ugi.getShortUserName())); + homeDir = (base.equals("/") ? + this.makeQualified(new Path(base + ugi.getShortUserName())): + this.makeQualified(new Path(base + "/" + ugi.getShortUserName()))); } return homeDir; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index a8a77bec5d0..dcfe5f32031 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -248,8 +248,9 @@ public class ViewFs extends AbstractFileSystem { if (base == null) { base = "/user"; } - homeDir = - this.makeQualified(new Path(base + "/" + ugi.getShortUserName())); + homeDir = (base.equals("/") ? + this.makeQualified(new Path(base + ugi.getShortUserName())): + this.makeQualified(new Path(base + "/" + ugi.getShortUserName()))); } return homeDir; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index 38e07d8aace..2c058ca3098 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -61,19 +61,28 @@ public final class FileSystemTestHelper { return data; } + + /* + * get testRootPath qualified for fSys + */ public static Path getTestRootPath(FileSystem fSys) { return fSys.makeQualified(new Path(TEST_ROOT_DIR)); } + /* + * get testRootPath + pathString qualified for fSys + */ public static Path getTestRootPath(FileSystem fSys, String pathString) { return fSys.makeQualified(new Path(TEST_ROOT_DIR, pathString)); } // the getAbsolutexxx method is needed because the root test dir - // can be messed up by changing the working dir. + // can be messed up by changing the working dir since the TEST_ROOT_PATH + // is often relative to the working directory of process + // running the unit tests. - public static String getAbsoluteTestRootDir(FileSystem fSys) + static String getAbsoluteTestRootDir(FileSystem fSys) throws IOException { // NOTE: can't cache because of different filesystems! //if (absTestRootDir == null) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index 35e2cb7649c..f2aace9472d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -73,10 +73,10 @@ public class TestChRootedFileSystem { URI uri = fSys.getUri(); Assert.assertEquals(chrootedTo.toUri(), uri); Assert.assertEquals(fSys.makeQualified( - new Path("/user/" + System.getProperty("user.name"))), + new Path(System.getProperty("user.home"))), fSys.getWorkingDirectory()); Assert.assertEquals(fSys.makeQualified( - new Path("/user/" + System.getProperty("user.name"))), + new Path(System.getProperty("user.home"))), fSys.getHomeDirectory()); /* * ChRootedFs as its uri like file:///chrootRoot. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java index c47308fbc09..c52280154d6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java @@ -70,10 +70,10 @@ public class TestChRootedFs { URI uri = fc.getDefaultFileSystem().getUri(); Assert.assertEquals(chrootedTo.toUri(), uri); Assert.assertEquals(fc.makeQualified( - new Path("/user/" + System.getProperty("user.name"))), + new Path(System.getProperty("user.home"))), fc.getWorkingDirectory()); Assert.assertEquals(fc.makeQualified( - new Path("/user/" + System.getProperty("user.name"))), + new Path(System.getProperty("user.home"))), fc.getHomeDirectory()); /* * ChRootedFs as its uri like file:///chrootRoot. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java index 5641c9d70bf..74f558509f1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java @@ -39,44 +39,7 @@ public class TestFcMainOperationsLocalFs extends @Override @Before public void setUp() throws Exception { - /** - * create the test root on local_fs - the mount table will point here - */ - fclocal = FileContext.getLocalFSFileContext(); - targetOfTests = FileContextTestHelper.getTestRootPath(fclocal); - // In case previous test was killed before cleanup - fclocal.delete(targetOfTests, true); - - fclocal.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true); - - - - - // We create mount table so that the test root on the viewFs points to - // to the test root on the target. - // DOing this helps verify the FileStatus.path. - // - // The test root by default when running eclipse - // is a test dir below the working directory. - // (see FileContextTestHelper). - // Since viewFs has no built-in wd, its wd is /user/. - // If this test launched via ant (build.xml) the test root is absolute path - - String srcTestRoot; - if (FileContextTestHelper.TEST_ROOT_DIR.startsWith("/")) { - srcTestRoot = FileContextTestHelper.TEST_ROOT_DIR; - } else { - srcTestRoot = "/user/" + System.getProperty("user.name") + "/" + - FileContextTestHelper.TEST_ROOT_DIR; - } - - Configuration conf = new Configuration(); - ConfigUtil.addLink(conf, srcTestRoot, - targetOfTests.toUri()); - - fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf); - //System.out.println("SRCOfTests = "+ FileContextTestHelper.getTestRootPath(fc, "test")); - //System.out.println("TargetOfTests = "+ targetOfTests.toUri()); + fc = ViewFsTestSetup.setupForViewFsLocalFs(); super.setUp(); } @@ -84,6 +47,6 @@ public class TestFcMainOperationsLocalFs extends @After public void tearDown() throws Exception { super.tearDown(); - fclocal.delete(targetOfTests, true); + ViewFsTestSetup.tearDownForViewFsLocalFs(); } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java index 525f28bea7c..446b38e60b8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.fs.viewfs; +import java.net.URI; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; @@ -32,14 +35,19 @@ import org.mortbay.log.Log; * * If tests launched via ant (build.xml) the test root is absolute path * If tests launched via eclipse, the test root is - * is a test dir below the working directory. (see FileSystemTestHelper). - * Since viewFs has no built-in wd, its wd is /user/ - * (or /User/ on mac) + * is a test dir below the working directory. (see FileContextTestHelper) * - * We set a viewFileSystems with mount point for - * /" pointing to the target fs's testdir + * We set a viewFileSystems with 3 mount points: + * 1) /" of testdir pointing to same in target fs + * 2) /" of home pointing to same in target fs + * 3) /" of wd pointing to same in target fs + * (note in many cases the link may be the same - viewFileSytem handles this) + * + * We also set the view file system's wd to point to the wd. */ public class ViewFileSystemTestSetup { + + static public String ViewFSTestDir = "/testDir"; /** * @@ -56,24 +64,26 @@ public class ViewFileSystemTestSetup { fsTarget.delete(targetOfTests, true); fsTarget.mkdirs(targetOfTests); - // Setup a link from viewfs to targetfs for the first component of - // path of testdir. + + // Set up viewfs link for test dir as described above String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri() .getPath(); - int indexOf2ndSlash = testDir.indexOf('/', 1); - String testDirFirstComponent = testDir.substring(0, indexOf2ndSlash); - ConfigUtil.addLink(conf, testDirFirstComponent, fsTarget.makeQualified( - new Path(testDirFirstComponent)).toUri()); + linkUpFirstComponents(conf, testDir, fsTarget, "test dir"); + + + // Set up viewfs link for home dir as described above + setUpHomeDir(conf, fsTarget); + + + // the test path may be relative to working dir - we need to make that work: + // Set up viewfs link for wd as described above + String wdDir = fsTarget.getWorkingDirectory().toUri().getPath(); + linkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); - // viewFs://home => fsTarget://home - String homeDirRoot = fsTarget.getHomeDirectory() - .getParent().toUri().getPath(); - ConfigUtil.addLink(conf, homeDirRoot, - fsTarget.makeQualified(new Path(homeDirRoot)).toUri()); - ConfigUtil.setHomeDirConf(conf, homeDirRoot); - Log.info("Home dir base " + homeDirRoot); FileSystem fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf); + fsView.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. + Log.info("Working dir is: " + fsView.getWorkingDirectory()); return fsView; } @@ -91,4 +101,33 @@ public class ViewFileSystemTestSetup { conf.set("fs.viewfs.impl", ViewFileSystem.class.getName()); return conf; } + + static void setUpHomeDir(Configuration conf, FileSystem fsTarget) { + String homeDir = fsTarget.getHomeDirectory().toUri().getPath(); + int indexOf2ndSlash = homeDir.indexOf('/', 1); + if (indexOf2ndSlash >0) { + linkUpFirstComponents(conf, homeDir, fsTarget, "home dir"); + } else { // home dir is at root. Just link the home dir itse + URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri(); + ConfigUtil.addLink(conf, homeDir, linkTarget); + Log.info("Added link for home dir " + homeDir + "->" + linkTarget); + } + // Now set the root of the home dir for viewfs + String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath(); + ConfigUtil.setHomeDirConf(conf, homeDirRoot); + Log.info("Home dir base for viewfs" + homeDirRoot); + } + + /* + * Set up link in config for first component of path to the same + * in the target file system. + */ + static void linkUpFirstComponents(Configuration conf, String path, FileSystem fsTarget, String info) { + int indexOf2ndSlash = path.indexOf('/', 1); + String firstComponent = path.substring(0, indexOf2ndSlash); + URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); + ConfigUtil.addLink(conf, firstComponent, linkTarget); + Log.info("Added link for " + info + " " + + firstComponent + "->" + linkTarget); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java index 4ab91158778..ac63217fd43 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.fs.viewfs; +import java.net.URI; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.mortbay.log.Log; /** @@ -31,13 +34,20 @@ import org.apache.hadoop.fs.viewfs.ConfigUtil; * * If tests launched via ant (build.xml) the test root is absolute path * If tests launched via eclipse, the test root is - * is a test dir below the working directory. (see FileContextTestHelper). - * Since viewFs has no built-in wd, its wd is /user/. + * is a test dir below the working directory. (see FileContextTestHelper) * - * We set up fc to be the viewFs with mount point for - * /" pointing to the local file system's testdir + * We set a viewfs with 3 mount points: + * 1) /" of testdir pointing to same in target fs + * 2) /" of home pointing to same in target fs + * 3) /" of wd pointing to same in target fs + * (note in many cases the link may be the same - viewfs handles this) + * + * We also set the view file system's wd to point to the wd. */ + public class ViewFsTestSetup { + + static public String ViewFSTestDir = "/testDir"; /* @@ -47,30 +57,31 @@ public class ViewFsTestSetup { /** * create the test root on local_fs - the mount table will point here */ - FileContext fclocal = FileContext.getLocalFSFileContext(); - Path targetOfTests = FileContextTestHelper.getTestRootPath(fclocal); + FileContext fsTarget = FileContext.getLocalFSFileContext(); + Path targetOfTests = FileContextTestHelper.getTestRootPath(fsTarget); // In case previous test was killed before cleanup - fclocal.delete(targetOfTests, true); + fsTarget.delete(targetOfTests, true); - fclocal.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true); - - String srcTestFirstDir; - if (FileContextTestHelper.TEST_ROOT_DIR.startsWith("/")) { - int indexOf2ndSlash = FileContextTestHelper.TEST_ROOT_DIR.indexOf('/', 1); - srcTestFirstDir = FileContextTestHelper.TEST_ROOT_DIR.substring(0, indexOf2ndSlash); - } else { - srcTestFirstDir = "/user"; - - } - //System.out.println("srcTestFirstDir=" + srcTestFirstDir); - - // Set up the defaultMT in the config with mount point links - // The test dir is root is below /user/ + fsTarget.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true); Configuration conf = new Configuration(); - ConfigUtil.addLink(conf, srcTestFirstDir, - targetOfTests.toUri()); + + // Set up viewfs link for test dir as described above + String testDir = FileContextTestHelper.getTestRootPath(fsTarget).toUri() + .getPath(); + linkUpFirstComponents(conf, testDir, fsTarget, "test dir"); + + + // Set up viewfs link for home dir as described above + setUpHomeDir(conf, fsTarget); + + // the test path may be relative to working dir - we need to make that work: + // Set up viewfs link for wd as described above + String wdDir = fsTarget.getWorkingDirectory().toUri().getPath(); + linkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); FileContext fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf); + fc.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. + Log.info("Working dir is: " + fc.getWorkingDirectory()); //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test")); //System.out.println("TargetOfTests = "+ targetOfTests.toUri()); return fc; @@ -85,5 +96,36 @@ public class ViewFsTestSetup { Path targetOfTests = FileContextTestHelper.getTestRootPath(fclocal); fclocal.delete(targetOfTests, true); } + + + static void setUpHomeDir(Configuration conf, FileContext fsTarget) { + String homeDir = fsTarget.getHomeDirectory().toUri().getPath(); + int indexOf2ndSlash = homeDir.indexOf('/', 1); + if (indexOf2ndSlash >0) { + linkUpFirstComponents(conf, homeDir, fsTarget, "home dir"); + } else { // home dir is at root. Just link the home dir itse + URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri(); + ConfigUtil.addLink(conf, homeDir, linkTarget); + Log.info("Added link for home dir " + homeDir + "->" + linkTarget); + } + // Now set the root of the home dir for viewfs + String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath(); + ConfigUtil.setHomeDirConf(conf, homeDirRoot); + Log.info("Home dir base for viewfs" + homeDirRoot); + } + + /* + * Set up link in config for first component of path to the same + * in the target file system. + */ + static void linkUpFirstComponents(Configuration conf, String path, + FileContext fsTarget, String info) { + int indexOf2ndSlash = path.indexOf('/', 1); + String firstComponent = path.substring(0, indexOf2ndSlash); + URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); + ConfigUtil.addLink(conf, firstComponent, linkTarget); + Log.info("Added link for " + info + " " + + firstComponent + "->" + linkTarget); + } } From 5a324b18edbfd4cf2305737acdfba23135879545 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Thu, 8 Nov 2012 10:33:29 +0000 Subject: [PATCH 10/31] YARN-169. Update log4j.appender.EventCounter to use org.apache.hadoop.log.metrics.EventCounter. Contributed by Anthony Rojas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1406999 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/resources/container-log4j.properties | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7d1dce459ff..ec030b2ba76 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,9 @@ Release 2.0.3-alpha - Unreleased YARN-181. Fixed eclipse settings broken by capacity-scheduler.xml move via YARN-140. (Siddharth Seth via vinodkv) + YARN-169. Update log4j.appender.EventCounter to use + org.apache.hadoop.log.metrics.EventCounter (Anthony Rojas via tomwhite) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index ea485741951..7c859535c88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -26,4 +26,4 @@ log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n # Event Counter Appender # Sends counts of logging messages at different severity levels to Hadoop Metrics. # -log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter +log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter From 6ee6eb843013324788f30384d9d967ff8743a970 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 8 Nov 2012 15:17:34 +0000 Subject: [PATCH 11/31] MAPREDUCE-4772. Fetch failures can take way too long for a map to be restarted (bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407118 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/impl/JobImpl.java | 19 +- .../mapreduce/v2/app/TestFetchFailure.java | 169 ++++++++++++++++++ .../apache/hadoop/mapreduce/MRJobConfig.java | 3 + .../hadoop/mapreduce/task/reduce/Fetcher.java | 8 +- .../task/reduce/ShuffleScheduler.java | 16 +- .../src/main/resources/mapred-default.xml | 8 + .../mapreduce/task/reduce/TestFetcher.java | 8 +- 8 files changed, 217 insertions(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 258a07ca75b..bd40d5ab947 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -637,6 +637,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4771. KeyFieldBasedPartitioner not partitioning properly when configured (jlowe via bobby) + + MAPREDUCE-4772. Fetch failures can take way too long for a map to be + restarted (bobby) Release 0.23.4 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 3053f1901f8..4612d066ee3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -68,6 +68,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; @@ -1409,16 +1410,22 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1); job.fetchFailuresMapping.put(mapId, fetchFailures); - //get number of running reduces - int runningReduceTasks = 0; + //get number of shuffling reduces + int shufflingReduceTasks = 0; for (TaskId taskId : job.reduceTasks) { - if (TaskState.RUNNING.equals(job.tasks.get(taskId).getState())) { - runningReduceTasks++; + Task task = job.tasks.get(taskId); + if (TaskState.RUNNING.equals(task.getState())) { + for(TaskAttempt attempt : task.getAttempts().values()) { + if(attempt.getReport().getPhase() == Phase.SHUFFLE) { + shufflingReduceTasks++; + break; + } + } } } - float failureRate = runningReduceTasks == 0 ? 1.0f : - (float) fetchFailures / runningReduceTasks; + float failureRate = shufflingReduceTasks == 0 ? 1.0f : + (float) fetchFailures / shufflingReduceTasks; // declare faulty if fetch-failures >= max-allowed-failures boolean isMapFaulty = (failureRate >= MAX_ALLOWED_FETCH_FAILURES_FRACTION); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java index cdc8537c103..4b1a1a0bd96 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java @@ -18,14 +18,19 @@ package org.apache.hadoop.mapreduce.v2.app; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler; import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; @@ -37,6 +42,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; import org.apache.hadoop.yarn.event.EventHandler; import org.junit.Assert; import org.junit.Test; @@ -254,6 +260,169 @@ public class TestFetchFailure { events = job.getTaskAttemptCompletionEvents(0, 100); Assert.assertEquals("Num completion events not correct", 2, events.length); } + + @Test + public void testFetchFailureMultipleReduces() throws Exception { + MRApp app = new MRApp(1, 3, false, this.getClass().getName(), true); + Configuration conf = new Configuration(); + // map -> reduce -> fetch-failure -> map retry is incompatible with + // sequential, single-task-attempt approach in uber-AM, so disable: + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + Job job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + //all maps would be running + Assert.assertEquals("Num tasks not correct", + 4, job.getTasks().size()); + Iterator it = job.getTasks().values().iterator(); + Task mapTask = it.next(); + Task reduceTask = it.next(); + Task reduceTask2 = it.next(); + Task reduceTask3 = it.next(); + + //wait for Task state move to RUNNING + app.waitForState(mapTask, TaskState.RUNNING); + TaskAttempt mapAttempt1 = mapTask.getAttempts().values().iterator().next(); + app.waitForState(mapAttempt1, TaskAttemptState.RUNNING); + + //send the done signal to the map attempt + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(mapAttempt1.getID(), + TaskAttemptEventType.TA_DONE)); + + // wait for map success + app.waitForState(mapTask, TaskState.SUCCEEDED); + + TaskAttemptCompletionEvent[] events = + job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Num completion events not correct", + 1, events.length); + Assert.assertEquals("Event status not correct", + TaskAttemptCompletionEventStatus.SUCCEEDED, events[0].getStatus()); + + // wait for reduce to start running + app.waitForState(reduceTask, TaskState.RUNNING); + app.waitForState(reduceTask2, TaskState.RUNNING); + app.waitForState(reduceTask3, TaskState.RUNNING); + TaskAttempt reduceAttempt = + reduceTask.getAttempts().values().iterator().next(); + app.waitForState(reduceAttempt, TaskAttemptState.RUNNING); + + updateStatus(app, reduceAttempt, Phase.SHUFFLE); + + TaskAttempt reduceAttempt2 = + reduceTask2.getAttempts().values().iterator().next(); + app.waitForState(reduceAttempt2, TaskAttemptState.RUNNING); + updateStatus(app, reduceAttempt2, Phase.SHUFFLE); + + TaskAttempt reduceAttempt3 = + reduceTask3.getAttempts().values().iterator().next(); + app.waitForState(reduceAttempt3, TaskAttemptState.RUNNING); + updateStatus(app, reduceAttempt3, Phase.SHUFFLE); + + //send 3 fetch failures from reduce to trigger map re execution + sendFetchFailure(app, reduceAttempt, mapAttempt1); + sendFetchFailure(app, reduceAttempt, mapAttempt1); + sendFetchFailure(app, reduceAttempt, mapAttempt1); + + //We should not re-launch the map task yet + assertEquals(TaskState.SUCCEEDED, mapTask.getState()); + updateStatus(app, reduceAttempt2, Phase.REDUCE); + updateStatus(app, reduceAttempt3, Phase.REDUCE); + + sendFetchFailure(app, reduceAttempt, mapAttempt1); + + //wait for map Task state move back to RUNNING + app.waitForState(mapTask, TaskState.RUNNING); + + //map attempt must have become FAILED + Assert.assertEquals("Map TaskAttempt state not correct", + TaskAttemptState.FAILED, mapAttempt1.getState()); + + Assert.assertEquals("Num attempts in Map Task not correct", + 2, mapTask.getAttempts().size()); + + Iterator atIt = mapTask.getAttempts().values().iterator(); + atIt.next(); + TaskAttempt mapAttempt2 = atIt.next(); + + app.waitForState(mapAttempt2, TaskAttemptState.RUNNING); + //send the done signal to the second map attempt + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(mapAttempt2.getID(), + TaskAttemptEventType.TA_DONE)); + + // wait for map success + app.waitForState(mapTask, TaskState.SUCCEEDED); + + //send done to reduce + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(reduceAttempt.getID(), + TaskAttemptEventType.TA_DONE)); + + //send done to reduce + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(reduceAttempt2.getID(), + TaskAttemptEventType.TA_DONE)); + + //send done to reduce + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(reduceAttempt3.getID(), + TaskAttemptEventType.TA_DONE)); + + app.waitForState(job, JobState.SUCCEEDED); + + //previous completion event now becomes obsolete + Assert.assertEquals("Event status not correct", + TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); + + events = job.getTaskAttemptCompletionEvents(0, 100); + Assert.assertEquals("Num completion events not correct", + 6, events.length); + Assert.assertEquals("Event map attempt id not correct", + mapAttempt1.getID(), events[0].getAttemptId()); + Assert.assertEquals("Event map attempt id not correct", + mapAttempt1.getID(), events[1].getAttemptId()); + Assert.assertEquals("Event map attempt id not correct", + mapAttempt2.getID(), events[2].getAttemptId()); + Assert.assertEquals("Event reduce attempt id not correct", + reduceAttempt.getID(), events[3].getAttemptId()); + Assert.assertEquals("Event status not correct for map attempt1", + TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); + Assert.assertEquals("Event status not correct for map attempt1", + TaskAttemptCompletionEventStatus.FAILED, events[1].getStatus()); + Assert.assertEquals("Event status not correct for map attempt2", + TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus()); + Assert.assertEquals("Event status not correct for reduce attempt1", + TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus()); + + TaskAttemptCompletionEvent mapEvents[] = + job.getMapAttemptCompletionEvents(0, 2); + Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length); + Assert.assertArrayEquals("Unexpected map events", + Arrays.copyOfRange(events, 0, 2), mapEvents); + mapEvents = job.getMapAttemptCompletionEvents(2, 200); + Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length); + Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]); + } + + + private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) { + TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus(); + status.counters = new Counters(); + status.fetchFailedMaps = new ArrayList(); + status.id = attempt.getID(); + status.mapFinishTime = 0; + status.outputSize = 0; + status.phase = phase; + status.progress = 0.5f; + status.shuffleFinishTime = 0; + status.sortFinishTime = 0; + status.stateString = "OK"; + status.taskState = attempt.getState(); + TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(attempt.getID(), + status); + app.getContext().getEventHandler().handle(event); + } private void sendFetchFailure(MRApp app, TaskAttempt reduceAttempt, TaskAttempt mapAttempt) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index cf865cf8720..b086a12b614 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -262,6 +262,9 @@ public interface MRJobConfig { public static final String SHUFFLE_FETCH_FAILURES = "mapreduce.reduce.shuffle.maxfetchfailures"; public static final String SHUFFLE_NOTIFY_READERROR = "mapreduce.reduce.shuffle.notify.readerror"; + + public static final String MAX_SHUFFLE_FETCH_RETRY_DELAY = "mapreduce.reduce.shuffle.retry-delay.max.ms"; + public static final long DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY = 60000; public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 27c7a49069e..ef8a2a95256 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -21,6 +21,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; @@ -283,6 +284,7 @@ class Fetcher extends Thread { SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret); LOG.info("for url="+msgToEncode+" sent hash and receievd reply"); } catch (IOException ie) { + boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); @@ -291,14 +293,14 @@ class Fetcher extends Thread { // indirectly penalizing the host if (!connectSucceeded) { for(TaskAttemptID left: remaining) { - scheduler.copyFailed(left, host, connectSucceeded); + scheduler.copyFailed(left, host, connectSucceeded, connectExcpt); } } else { // If we got a read error at this stage, it implies there was a problem // with the first map, typically lost map. So, penalize only that map // and add the rest TaskAttemptID firstMap = maps.get(0); - scheduler.copyFailed(firstMap, host, connectSucceeded); + scheduler.copyFailed(firstMap, host, connectSucceeded, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed @@ -322,7 +324,7 @@ class Fetcher extends Thread { if(failedTasks != null && failedTasks.length > 0) { LOG.warn("copyMapOutput failed for tasks "+Arrays.toString(failedTasks)); for(TaskAttemptID left: failedTasks) { - scheduler.copyFailed(left, host, true); + scheduler.copyFailed(left, host, true, false); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java index 8d5bc3b9050..03378a478d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java @@ -89,6 +89,7 @@ class ShuffleScheduler { private DecimalFormat mbpsFormat = new DecimalFormat("0.00"); private boolean reportReadErrorImmediately = true; + private long maxDelay = MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY; public ShuffleScheduler(JobConf job, TaskStatus status, ExceptionReporter reporter, @@ -115,6 +116,9 @@ class ShuffleScheduler { MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT); this.reportReadErrorImmediately = job.getBoolean( MRJobConfig.SHUFFLE_NOTIFY_READERROR, true); + + this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY, + MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY); } public synchronized void copySucceeded(TaskAttemptID mapId, @@ -159,7 +163,7 @@ class ShuffleScheduler { } public synchronized void copyFailed(TaskAttemptID mapId, MapHost host, - boolean readError) { + boolean readError, boolean connectExcpt) { host.penalize(); int failures = 1; if (failureCounts.containsKey(mapId)) { @@ -184,12 +188,15 @@ class ShuffleScheduler { } } - checkAndInformJobTracker(failures, mapId, readError); + checkAndInformJobTracker(failures, mapId, readError, connectExcpt); checkReducerHealth(); long delay = (long) (INITIAL_PENALTY * Math.pow(PENALTY_GROWTH_RATE, failures)); + if (delay > maxDelay) { + delay = maxDelay; + } penalties.add(new Penalty(host, delay)); @@ -200,8 +207,9 @@ class ShuffleScheduler { // after every read error, if 'reportReadErrorImmediately' is true or // after every 'maxFetchFailuresBeforeReporting' failures private void checkAndInformJobTracker( - int failures, TaskAttemptID mapId, boolean readError) { - if ((reportReadErrorImmediately && readError) + int failures, TaskAttemptID mapId, boolean readError, + boolean connectExcpt) { + if (connectExcpt || (reportReadErrorImmediately && readError) || ((failures % maxFetchFailuresBeforeReporting) == 0)) { LOG.info("Reporting fetch failure for " + mapId + " to jobtracker."); status.addFetchFailedMap((org.apache.hadoop.mapred.TaskAttemptID) mapId); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 6979855adbd..8852d3980b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -110,6 +110,14 @@ + + mapreduce.reduce.shuffle.retry-delay.max.ms + 60000 + The maximum number of ms the reducer will delay before retrying + to download map data. + + + mapreduce.reduce.shuffle.parallelcopies 5 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java index 097f120dd21..92bdc74fb31 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java @@ -118,8 +118,8 @@ public class TestFetcher { encHash); verify(allErrs).increment(1); - verify(ss).copyFailed(map1ID, host, true); - verify(ss).copyFailed(map2ID, host, true); + verify(ss).copyFailed(map1ID, host, true, false); + verify(ss).copyFailed(map2ID, host, true, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); @@ -178,8 +178,8 @@ public class TestFetcher { .addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); verify(allErrs, never()).increment(1); - verify(ss, never()).copyFailed(map1ID, host, true); - verify(ss, never()).copyFailed(map2ID, host, true); + verify(ss, never()).copyFailed(map1ID, host, true, false); + verify(ss, never()).copyFailed(map2ID, host, true, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); From 8a1f3337071b7bd16d274623353a21cb64f85140 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 8 Nov 2012 17:22:52 +0000 Subject: [PATCH 12/31] YARN-186. Coverage fixing LinuxContainerExecutor (Aleksey Gorshkov via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407171 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../TestLinuxContainerExecutorWithMocks.java | 98 ++++++++++++++++++- .../mock-container-executer-with-error | 7 ++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100755 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ec030b2ba76..7ff7e44efe8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -169,6 +169,9 @@ Release 0.23.5 - UNRELEASED YARN-32. Fix TestApplicationTokens to not depend on test order and thus pass on JDK7. (vinodkv) + YARN-186. Coverage fixing LinuxContainerExecutor (Aleksey Gorshkov via + bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 9b98290d909..46ee4aa0c54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -26,11 +26,13 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.LineNumberReader; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,7 +48,6 @@ import org.junit.Test; public class TestLinuxContainerExecutorWithMocks { - @SuppressWarnings("unused") private static final Log LOG = LogFactory .getLog(TestLinuxContainerExecutorWithMocks.class); @@ -54,6 +55,7 @@ public class TestLinuxContainerExecutorWithMocks { private final File mockParamFile = new File("./params.txt"); private LocalDirsHandlerService dirsHandler; + private void deleteMockParamFile() { if(mockParamFile.exists()) { mockParamFile.delete(); @@ -126,8 +128,102 @@ public class TestLinuxContainerExecutorWithMocks { StringUtils.join(",", dirsHandler.getLocalDirs()), StringUtils.join(",", dirsHandler.getLogDirs())), readMockParams()); + } + + @Test + public void testStartLocalizer() throws IOException { + + + InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 8040); + Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath"); + + try { + mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs()); + List result=readMockParams(); + Assert.assertEquals(result.size(), 16); + Assert.assertEquals(result.get(0), "test"); + Assert.assertEquals(result.get(1), "0" ); + Assert.assertEquals(result.get(2),"application_0" ); + Assert.assertEquals(result.get(3), "/bin/nmPrivateCTokensPath"); + Assert.assertEquals(result.get(7), "-classpath" ); + Assert.assertEquals(result.get(10),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(11), "test"); + Assert.assertEquals(result.get(12), "application_0"); + Assert.assertEquals(result.get(13),"12345" ); + Assert.assertEquals(result.get(14),"localhost" ); + Assert.assertEquals(result.get(15),"8040" ); + + } catch (InterruptedException e) { + LOG.error("Error:"+e.getMessage(),e); + Assert.fail(); + } + } + + + @Test + public void testContainerLaunchError() throws IOException { + + // reinitialize executer + File f = new File("./src/test/resources/mock-container-executer-with-error"); + if (!f.canExecute()) { + f.setExecutable(true); + } + String executorPath = f.getAbsolutePath(); + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo"); + conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null"); + + mockExec = new LinuxContainerExecutor(); + dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + mockExec.setConf(conf); + + String appSubmitter = "nobody"; + String cmd = String + .valueOf(LinuxContainerExecutor.Commands.LAUNCH_CONTAINER.getValue()); + String appId = "APP_ID"; + String containerId = "CONTAINER_ID"; + Container container = mock(Container.class); + ContainerId cId = mock(ContainerId.class); + ContainerLaunchContext context = mock(ContainerLaunchContext.class); + HashMap env = new HashMap(); + + when(container.getContainerID()).thenReturn(cId); + when(container.getLaunchContext()).thenReturn(context); + + when(cId.toString()).thenReturn(containerId); + + when(context.getEnvironment()).thenReturn(env); + + Path scriptPath = new Path("file:///bin/echo"); + Path tokensPath = new Path("file:///dev/null"); + Path workDir = new Path("/tmp"); + Path pidFile = new Path(workDir, "pid.txt"); + + mockExec.activateContainer(cId, pidFile); + int ret = mockExec.launchContainer(container, scriptPath, tokensPath, + appSubmitter, appId, workDir, dirsHandler.getLocalDirs(), + dirsHandler.getLogDirs()); + Assert.assertNotSame(0, ret); + assertEquals(Arrays.asList(appSubmitter, cmd, appId, containerId, + workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), + StringUtils.join(",", dirsHandler.getLocalDirs()), + StringUtils.join(",", dirsHandler.getLogDirs())), readMockParams()); + + } + + @Test + public void testInit() throws Exception { + + mockExec.init(); + assertEquals(Arrays.asList("--checksetup"), readMockParams()); + + } + + @Test public void testContainerKill() throws IOException { String appSubmitter = "nobody"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error new file mode 100755 index 00000000000..4f3432cbb80 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error @@ -0,0 +1,7 @@ +#!/bin/sh +for PARAM in "$@" +do + echo $PARAM; +done > params.txt + +exec badcommand From f1fe91ec9519535153f05acd3e6402bdb8e12c56 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Thu, 8 Nov 2012 17:45:46 +0000 Subject: [PATCH 13/31] HDFS-3810. Implement format() for BKJM. Contributed by Ivan Kelly. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407182 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../bkjournal/BookKeeperJournalManager.java | 173 ++++++++++++------ .../contrib/bkjournal/CurrentInprogress.java | 18 +- .../TestBookKeeperConfiguration.java | 2 + .../TestBookKeeperJournalManager.java | 99 ++++++++++ .../bkjournal/TestCurrentInprogress.java | 5 +- 6 files changed, 232 insertions(+), 67 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 67c3fbb156c..c174dc5dc21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -560,6 +560,8 @@ Release 2.0.3-alpha - Unreleased HDFS-3979. For hsync, datanode should wait for the local sync to complete before sending ack. (Lars Hofhansl via szetszwo) + HDFS-3810. Implement format() for BKJM (Ivan Kelly via umamahesh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java index 62dbbd031f4..67f0b4b2293 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java @@ -39,6 +39,7 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.AsyncCallback.StringCallback; +import org.apache.zookeeper.ZKUtil; import java.util.Collection; import java.util.Collections; @@ -46,6 +47,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.io.IOException; import java.net.URI; @@ -142,13 +144,16 @@ public class BookKeeperJournalManager implements JournalManager { private final Configuration conf; private final BookKeeper bkc; private final CurrentInprogress ci; + private final String basePath; private final String ledgerPath; + private final String versionPath; private final MaxTxId maxTxId; private final int ensembleSize; private final int quorumSize; private final String digestpw; private final CountDownLatch zkConnectLatch; private final NamespaceInfo nsInfo; + private boolean initialized = false; private LedgerHandle currentLedger = null; /** @@ -160,16 +165,16 @@ public class BookKeeperJournalManager implements JournalManager { this.nsInfo = nsInfo; String zkConnect = uri.getAuthority().replace(";", ","); - String zkPath = uri.getPath(); + basePath = uri.getPath(); ensembleSize = conf.getInt(BKJM_BOOKKEEPER_ENSEMBLE_SIZE, BKJM_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT); quorumSize = conf.getInt(BKJM_BOOKKEEPER_QUORUM_SIZE, BKJM_BOOKKEEPER_QUORUM_SIZE_DEFAULT); - ledgerPath = zkPath + "/ledgers"; - String maxTxIdPath = zkPath + "/maxtxid"; - String currentInprogressNodePath = zkPath + "/CurrentInprogress"; - String versionPath = zkPath + "/version"; + ledgerPath = basePath + "/ledgers"; + String maxTxIdPath = basePath + "/maxtxid"; + String currentInprogressNodePath = basePath + "/CurrentInprogress"; + versionPath = basePath + "/version"; digestpw = conf.get(BKJM_BOOKKEEPER_DIGEST_PW, BKJM_BOOKKEEPER_DIGEST_PW_DEFAULT); @@ -180,47 +185,7 @@ public class BookKeeperJournalManager implements JournalManager { if (!zkConnectLatch.await(6000, TimeUnit.MILLISECONDS)) { throw new IOException("Error connecting to zookeeper"); } - if (zkc.exists(zkPath, false) == null) { - zkc.create(zkPath, new byte[] {'0'}, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - Stat versionStat = zkc.exists(versionPath, false); - if (versionStat != null) { - byte[] d = zkc.getData(versionPath, false, versionStat); - VersionProto.Builder builder = VersionProto.newBuilder(); - TextFormat.merge(new String(d, UTF_8), builder); - if (!builder.isInitialized()) { - throw new IOException("Invalid/Incomplete data in znode"); - } - VersionProto vp = builder.build(); - - // There's only one version at the moment - assert vp.getLayoutVersion() == BKJM_LAYOUT_VERSION; - - NamespaceInfo readns = PBHelper.convert(vp.getNamespaceInfo()); - - if (nsInfo.getNamespaceID() != readns.getNamespaceID() || - !nsInfo.clusterID.equals(readns.getClusterID()) || - !nsInfo.getBlockPoolID().equals(readns.getBlockPoolID())) { - String err = String.format("Environment mismatch. Running process %s" - +", stored in ZK %s", nsInfo, readns); - LOG.error(err); - throw new IOException(err); - } - } else if (nsInfo.getNamespaceID() > 0) { - VersionProto.Builder builder = VersionProto.newBuilder(); - builder.setNamespaceInfo(PBHelper.convert(nsInfo)) - .setLayoutVersion(BKJM_LAYOUT_VERSION); - byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); - zkc.create(versionPath, data, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - - if (zkc.exists(ledgerPath, false) == null) { - zkc.create(ledgerPath, new byte[] {'0'}, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } prepareBookKeeperEnv(); bkc = new BookKeeper(new ClientConfiguration(), zkc); } catch (KeeperException e) { @@ -244,6 +209,7 @@ public class BookKeeperJournalManager implements JournalManager { BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT); final CountDownLatch zkPathLatch = new CountDownLatch(1); + final AtomicBoolean success = new AtomicBoolean(false); StringCallback callback = new StringCallback() { @Override public void processResult(int rc, String path, Object ctx, String name) { @@ -251,22 +217,23 @@ public class BookKeeperJournalManager implements JournalManager { || KeeperException.Code.NODEEXISTS.intValue() == rc) { LOG.info("Successfully created bookie available path : " + zkAvailablePath); - zkPathLatch.countDown(); + success.set(true); } else { KeeperException.Code code = KeeperException.Code.get(rc); - LOG - .error("Error : " + LOG.error("Error : " + KeeperException.create(code, path).getMessage() + ", failed to create bookie available path : " + zkAvailablePath); } + zkPathLatch.countDown(); } }; ZkUtils.createFullPathOptimistic(zkc, zkAvailablePath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, callback, null); try { - if (!zkPathLatch.await(zkc.getSessionTimeout(), TimeUnit.MILLISECONDS)) { + if (!zkPathLatch.await(zkc.getSessionTimeout(), TimeUnit.MILLISECONDS) + || !success.get()) { throw new IOException("Couldn't create bookie available path :" + zkAvailablePath + ", timed out " + zkc.getSessionTimeout() + " millis"); @@ -281,19 +248,101 @@ public class BookKeeperJournalManager implements JournalManager { @Override public void format(NamespaceInfo ns) throws IOException { - // Currently, BKJM automatically formats itself when first accessed. - // TODO: change over to explicit formatting so that the admin can - // clear out the BK storage when reformatting a cluster. - LOG.info("Not formatting " + this + " - BKJM does not currently " + - "support reformatting. If it has not been used before, it will" + - "be formatted automatically upon first use."); + try { + // delete old info + Stat baseStat = null; + Stat ledgerStat = null; + if ((baseStat = zkc.exists(basePath, false)) != null) { + if ((ledgerStat = zkc.exists(ledgerPath, false)) != null) { + for (EditLogLedgerMetadata l : getLedgerList(true)) { + try { + bkc.deleteLedger(l.getLedgerId()); + } catch (BKException.BKNoSuchLedgerExistsException bke) { + LOG.warn("Ledger " + l.getLedgerId() + " does not exist;" + + " Cannot delete."); + } + } + } + ZKUtil.deleteRecursive(zkc, basePath); + } + + // should be clean now. + zkc.create(basePath, new byte[] {'0'}, + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + VersionProto.Builder builder = VersionProto.newBuilder(); + builder.setNamespaceInfo(PBHelper.convert(ns)) + .setLayoutVersion(BKJM_LAYOUT_VERSION); + + byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); + zkc.create(versionPath, data, + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + zkc.create(ledgerPath, new byte[] {'0'}, + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } catch (KeeperException ke) { + LOG.error("Error accessing zookeeper to format", ke); + throw new IOException("Error accessing zookeeper to format", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted during format", ie); + } catch (BKException bke) { + throw new IOException("Error cleaning up ledgers during format", bke); + } } @Override public boolean hasSomeData() throws IOException { - // Don't confirm format on BKJM, since format() is currently a - // no-op anyway - return false; + try { + return zkc.exists(basePath, false) != null; + } catch (KeeperException ke) { + throw new IOException("Couldn't contact zookeeper", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while checking for data", ie); + } + } + + synchronized private void checkEnv() throws IOException { + if (!initialized) { + try { + Stat versionStat = zkc.exists(versionPath, false); + if (versionStat == null) { + throw new IOException("Environment not initialized. " + +"Have you forgotten to format?"); + } + byte[] d = zkc.getData(versionPath, false, versionStat); + + VersionProto.Builder builder = VersionProto.newBuilder(); + TextFormat.merge(new String(d, UTF_8), builder); + if (!builder.isInitialized()) { + throw new IOException("Invalid/Incomplete data in znode"); + } + VersionProto vp = builder.build(); + + // There's only one version at the moment + assert vp.getLayoutVersion() == BKJM_LAYOUT_VERSION; + + NamespaceInfo readns = PBHelper.convert(vp.getNamespaceInfo()); + + if (nsInfo.getNamespaceID() != readns.getNamespaceID() || + !nsInfo.clusterID.equals(readns.getClusterID()) || + !nsInfo.getBlockPoolID().equals(readns.getBlockPoolID())) { + String err = String.format("Environment mismatch. Running process %s" + +", stored in ZK %s", nsInfo, readns); + LOG.error(err); + throw new IOException(err); + } + + ci.init(); + initialized = true; + } catch (KeeperException ke) { + throw new IOException("Cannot access ZooKeeper", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while checking environment", ie); + } + } } /** @@ -307,6 +356,8 @@ public class BookKeeperJournalManager implements JournalManager { */ @Override public EditLogOutputStream startLogSegment(long txId) throws IOException { + checkEnv(); + if (txId <= maxTxId.get()) { throw new IOException("We've already seen " + txId + ". A new stream cannot be created with it"); @@ -384,6 +435,8 @@ public class BookKeeperJournalManager implements JournalManager { @Override public void finalizeLogSegment(long firstTxId, long lastTxId) throws IOException { + checkEnv(); + String inprogressPath = inprogressZNode(firstTxId); try { Stat inprogressStat = zkc.exists(inprogressPath, false); @@ -537,6 +590,8 @@ public class BookKeeperJournalManager implements JournalManager { @Override public void recoverUnfinalizedSegments() throws IOException { + checkEnv(); + synchronized (this) { try { List children = zkc.getChildren(ledgerPath, false); @@ -589,6 +644,8 @@ public class BookKeeperJournalManager implements JournalManager { @Override public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException { + checkEnv(); + for (EditLogLedgerMetadata l : getLedgerList(false)) { if (l.getLastTxId() < minTxIdToKeep) { try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java index 8477f7c4e56..32d65cbf8d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java @@ -56,6 +56,9 @@ class CurrentInprogress { CurrentInprogress(ZooKeeper zkc, String lockpath) throws IOException { this.currentInprogressNode = lockpath; this.zkc = zkc; + } + + void init() throws IOException { try { Stat isCurrentInprogressNodeExists = zkc.exists(currentInprogressNode, false); @@ -96,15 +99,14 @@ class CurrentInprogress { this.versionNumberForPermission); } catch (KeeperException e) { throw new IOException("Exception when setting the data " - + "[layout version number,hostname,inprogressNode path]= [" + content - + "] to CurrentInprogress. ", e); + + "[" + content + "] to CurrentInprogress. ", e); } catch (InterruptedException e) { throw new IOException("Interrupted while setting the data " - + "[layout version number,hostname,inprogressNode path]= [" + content - + "] to CurrentInprogress", e); + + "[" + content + "] to CurrentInprogress", e); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Updated data[" + content + "] to CurrentInprogress"); } - LOG.info("Updated data[layout version number,hostname,inprogressNode path]" - + "= [" + content + "] to CurrentInprogress"); } /** @@ -136,7 +138,7 @@ class CurrentInprogress { } return builder.build().getPath(); } else { - LOG.info("No data available in CurrentInprogress"); + LOG.debug("No data available in CurrentInprogress"); } return null; } @@ -152,7 +154,7 @@ class CurrentInprogress { throw new IOException( "Interrupted when setting the data to CurrentInprogress node", e); } - LOG.info("Cleared the data from CurrentInprogress"); + LOG.debug("Cleared the data from CurrentInprogress"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java index 4ea5074092c..f3f6ce5674f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java @@ -149,6 +149,7 @@ public class TestBookKeeperConfiguration { bkjm = new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-WithBKPath"), nsi); + bkjm.format(nsi); Assert.assertNotNull("Bookie available path : " + bkAvailablePath + " doesn't exists", zkc.exists(bkAvailablePath, false)); } @@ -166,6 +167,7 @@ public class TestBookKeeperConfiguration { bkjm = new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-DefaultBKPath"), nsi); + bkjm.format(nsi); Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH + " doesn't exists", zkc.exists(BK_ROOT_PATH, false)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java index 954f2a54098..9da904007d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java @@ -29,8 +29,16 @@ import org.mockito.Mockito; import java.io.IOException; import java.net.URI; import java.util.List; +import java.util.ArrayList; import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; @@ -90,6 +98,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { @@ -112,6 +121,8 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi); + bkjm.format(nsi); + EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -130,6 +141,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-gaps"), nsi); + bkjm.format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { @@ -167,6 +179,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"), nsi); + bkjm.format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { @@ -208,6 +221,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi); + bkjm.format(nsi); long txid = 1; long start = txid; @@ -266,6 +280,7 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); + bkjm1.format(nsi); BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); @@ -288,6 +303,7 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simpleread"), nsi); + bkjm.format(nsi); final long numTransactions = 10000; EditLogOutputStream out = bkjm.startLogSegment(1); @@ -315,6 +331,7 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"), nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { @@ -365,6 +382,7 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"), nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { @@ -450,6 +468,7 @@ public class TestBookKeeperJournalManager { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { @@ -500,6 +519,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -541,6 +561,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -583,6 +604,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -622,6 +644,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -669,6 +692,7 @@ public class TestBookKeeperJournalManager { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); try { // start new inprogress log segment with txid=1 @@ -697,6 +721,81 @@ public class TestBookKeeperJournalManager { } } + private enum ThreadStatus { + COMPLETED, GOODEXCEPTION, BADEXCEPTION; + }; + + /** + * Tests that concurrent calls to format will still allow one to succeed. + */ + @Test + public void testConcurrentFormat() throws Exception { + final URI uri = BKJMUtil.createJournalURI("/hdfsjournal-concurrentformat"); + final NamespaceInfo nsi = newNSInfo(); + + // populate with data first + BookKeeperJournalManager bkjm + = new BookKeeperJournalManager(conf, uri, nsi); + bkjm.format(nsi); + for (int i = 1; i < 100*2; i += 2) { + bkjm.startLogSegment(i); + bkjm.finalizeLogSegment(i, i+1); + } + bkjm.close(); + + final int numThreads = 40; + List> threads + = new ArrayList>(); + final CyclicBarrier barrier = new CyclicBarrier(numThreads); + + for (int i = 0; i < numThreads; i++) { + threads.add(new Callable() { + public ThreadStatus call() { + BookKeeperJournalManager bkjm = null; + try { + bkjm = new BookKeeperJournalManager(conf, uri, nsi); + barrier.await(); + bkjm.format(nsi); + return ThreadStatus.COMPLETED; + } catch (IOException ioe) { + LOG.info("Exception formatting ", ioe); + return ThreadStatus.GOODEXCEPTION; + } catch (InterruptedException ie) { + LOG.error("Interrupted. Something is broken", ie); + Thread.currentThread().interrupt(); + return ThreadStatus.BADEXCEPTION; + } catch (Exception e) { + LOG.error("Some other bad exception", e); + return ThreadStatus.BADEXCEPTION; + } finally { + if (bkjm != null) { + try { + bkjm.close(); + } catch (IOException ioe) { + LOG.error("Error closing journal manager", ioe); + } + } + } + } + }); + } + ExecutorService service = Executors.newFixedThreadPool(numThreads); + List> statuses = service.invokeAll(threads, 60, + TimeUnit.SECONDS); + int numCompleted = 0; + for (Future s : statuses) { + assertTrue(s.isDone()); + assertTrue("Thread threw invalid exception", + s.get() == ThreadStatus.COMPLETED + || s.get() == ThreadStatus.GOODEXCEPTION); + if (s.get() == ThreadStatus.COMPLETED) { + numCompleted++; + } + } + LOG.info("Completed " + numCompleted + " formats"); + assertTrue("No thread managed to complete formatting", numCompleted > 0); + } + private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid, int endTxid) throws IOException, KeeperException, InterruptedException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java index 00497b7798f..169a8a8f691 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java @@ -118,6 +118,7 @@ public class TestCurrentInprogress { public void testReadShouldReturnTheZnodePathAfterUpdate() throws Exception { String data = "inprogressNode"; CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); + ci.init(); ci.update(data); String inprogressNodePath = ci.read(); assertEquals("Not returning inprogressZnode", "inprogressNode", @@ -131,6 +132,7 @@ public class TestCurrentInprogress { @Test public void testReadShouldReturnNullAfterClear() throws Exception { CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); + ci.init(); ci.update("myInprogressZnode"); ci.read(); ci.clear(); @@ -146,6 +148,7 @@ public class TestCurrentInprogress { public void testUpdateShouldFailWithIOEIfVersionNumberChangedAfterRead() throws Exception { CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); + ci.init(); ci.update("myInprogressZnode"); assertEquals("Not returning myInprogressZnode", "myInprogressZnode", ci .read()); @@ -154,4 +157,4 @@ public class TestCurrentInprogress { ci.update("myInprogressZnode"); } -} \ No newline at end of file +} From d511052be53426c6ccb1d39e36ac340409f45b5b Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Thu, 8 Nov 2012 18:01:38 +0000 Subject: [PATCH 14/31] Move HDFS-3625 and HDFS-4138 under 2.0.3 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407189 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c174dc5dc21..c03bda351a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -560,6 +560,12 @@ Release 2.0.3-alpha - Unreleased HDFS-3979. For hsync, datanode should wait for the local sync to complete before sending ack. (Lars Hofhansl via szetszwo) + HDFS-3625. Fix TestBackupNode by properly initializing edit log during + startup. (Junping Du via todd) + + HDFS-4138. BackupNode startup fails due to uninitialized edit log. + (Kihwal Lee via shv) + HDFS-3810. Implement format() for BKJM (Ivan Kelly via umamahesh) Release 2.0.2-alpha - 2012-09-07 @@ -1965,12 +1971,6 @@ Release 0.23.5 - UNRELEASED HDFS-4090. getFileChecksum() result incompatible when called against zero-byte files. (Kihwal Lee via daryn) - HDFS-3625. Fix TestBackupNode by properly initializing edit log during - startup. (Junping Du via todd) - - HDFS-4138. BackupNode startup fails due to uninitialized edit log. - (Kihwal Lee via shv) - Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES From 1813fb0132f017441379a29b2d4d40a44eb08430 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 8 Nov 2012 19:12:42 +0000 Subject: [PATCH 15/31] HADOOP-8974. TestDFVariations fails on Windows. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407222 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 5 ++++- .../test/java/org/apache/hadoop/fs/TestDFVariations.java | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index db09a93efa9..4d6b88c93d0 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -272,7 +272,10 @@ Trunk (Unreleased) HADOOP-8918. test-patch.sh is parsing modified files wrong. (Raja Aluri via suresh) - HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia) + HADOOP-8589 ViewFs tests fail when tests and home dirs are nested. + (sanjay Radia) + + HADOOP-8974. TestDFVariations fails on Windows. (Chris Nauroth via suresh) OPTIMIZATIONS diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java index b5bd65bc1a9..b291dd2200f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; +import org.apache.hadoop.util.Shell; + public class TestDFVariations extends TestCase { public static class XXDF extends DF { @@ -51,7 +53,9 @@ public class TestDFVariations extends TestCase { public void testOSParsing() throws Exception { for (DF.OSType ost : EnumSet.allOf(DF.OSType.class)) { XXDF df = new XXDF(ost.getId()); - assertEquals(ost.getId() + " mount", "/foo/bar", df.getMount()); + assertEquals(ost.getId() + " mount", + Shell.WINDOWS ? df.getDirPath().substring(0, 2) : "/foo/bar", + df.getMount()); } } From 0f7a4337ac96141db4a3400fd8308436d87042f4 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 8 Nov 2012 19:54:55 +0000 Subject: [PATCH 16/31] HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota. Contributed by Binglin Chang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407245 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c03bda351a6..7dffd5c7e06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -248,6 +248,9 @@ Trunk (Unreleased) HDFS-4106. BPServiceActor#lastHeartbeat, lastBlockReport and lastDeletedReport should be volatile. (Jing Zhao via suresh) + HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota. + (Binglin Chang via suresh) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 66103bcf32e..ecd4286fa41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1849,9 +1849,9 @@ public class FSDirectory implements Closeable { assert hasWriteLock(); // sanity check if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET && - nsQuota < HdfsConstants.QUOTA_RESET) || + nsQuota != HdfsConstants.QUOTA_RESET) || (dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET && - dsQuota < HdfsConstants.QUOTA_RESET)) { + dsQuota != HdfsConstants.QUOTA_RESET)) { throw new IllegalArgumentException("Illegal value for nsQuota or " + "dsQuota : " + nsQuota + " and " + dsQuota); From be94bf6b57895846853d3e0ebc5c33b4f725ae2c Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Fri, 9 Nov 2012 00:53:11 +0000 Subject: [PATCH 17/31] HDFS-3990. NN's health report has severe performance problems (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407333 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/protocol/DatanodeID.java | 16 +++- .../blockmanagement/DatanodeManager.java | 78 +++++++++---------- .../hadoop/hdfs/TestDatanodeRegistration.java | 63 ++++++++++++++- 4 files changed, 117 insertions(+), 42 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7dffd5c7e06..cec087011da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1960,6 +1960,8 @@ Release 0.23.5 - UNRELEASED HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn) + HDFS-3990. NN's health report has severe performance problems (daryn) + BUG FIXES HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index ad1567fd39b..2a0578ca93f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -38,7 +38,8 @@ public class DatanodeID implements Comparable { public static final DatanodeID[] EMPTY_ARRAY = {}; private String ipAddr; // IP address - private String hostName; // hostname + private String hostName; // hostname claimed by datanode + private String peerHostName; // hostname from the actual connection private String storageID; // unique per cluster storageID private int xferPort; // data streaming port private int infoPort; // info server port @@ -51,6 +52,7 @@ public class DatanodeID implements Comparable { from.getXferPort(), from.getInfoPort(), from.getIpcPort()); + this.peerHostName = from.getPeerHostName(); } /** @@ -76,6 +78,10 @@ public class DatanodeID implements Comparable { this.ipAddr = ipAddr; } + public void setPeerHostName(String peerHostName) { + this.peerHostName = peerHostName; + } + public void setStorageID(String storageID) { this.storageID = storageID; } @@ -94,6 +100,13 @@ public class DatanodeID implements Comparable { return hostName; } + /** + * @return hostname from the actual connection + */ + public String getPeerHostName() { + return peerHostName; + } + /** * @return IP:xferPort string */ @@ -202,6 +215,7 @@ public class DatanodeID implements Comparable { public void updateRegInfo(DatanodeID nodeReg) { ipAddr = nodeReg.getIpAddr(); hostName = nodeReg.getHostName(); + peerHostName = nodeReg.getPeerHostName(); xferPort = nodeReg.getXferPort(); infoPort = nodeReg.getInfoPort(); ipcPort = nodeReg.getIpcPort(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 23013d7d911..804fdf21105 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -540,28 +540,16 @@ public class DatanodeManager { private static boolean checkInList(final DatanodeID node, final Set hostsList, final boolean isExcludeList) { - final InetAddress iaddr; - - try { - iaddr = InetAddress.getByName(node.getIpAddr()); - } catch (UnknownHostException e) { - LOG.warn("Unknown IP: " + node.getIpAddr(), e); - return isExcludeList; - } - // if include list is empty, host is in include list if ( (!isExcludeList) && (hostsList.isEmpty()) ){ return true; } - return // compare ipaddress(:port) - (hostsList.contains(iaddr.getHostAddress().toString())) - || (hostsList.contains(iaddr.getHostAddress().toString() + ":" - + node.getXferPort())) - // compare hostname(:port) - || (hostsList.contains(iaddr.getHostName())) - || (hostsList.contains(iaddr.getHostName() + ":" + node.getXferPort())) - || ((node instanceof DatanodeInfo) && hostsList - .contains(((DatanodeInfo) node).getHostName())); + for (String name : getNodeNamesForHostFiltering(node)) { + if (hostsList.contains(name)) { + return true; + } + } + return false; } /** @@ -644,16 +632,20 @@ public class DatanodeManager { */ public void registerDatanode(DatanodeRegistration nodeReg) throws DisallowedDatanodeException { - String dnAddress = Server.getRemoteAddress(); - if (dnAddress == null) { - // Mostly called inside an RPC. - // But if not, use address passed by the data-node. - dnAddress = nodeReg.getIpAddr(); + InetAddress dnAddress = Server.getRemoteIp(); + if (dnAddress != null) { + // Mostly called inside an RPC, update ip and peer hostname + String hostname = dnAddress.getHostName(); + String ip = dnAddress.getHostAddress(); + if (hostname.equals(ip)) { + LOG.warn("Unresolved datanode registration from " + ip); + throw new DisallowedDatanodeException(nodeReg); + } + // update node registration with the ip and hostname from rpc request + nodeReg.setIpAddr(ip); + nodeReg.setPeerHostName(hostname); } - // Update the IP to the address of the RPC request that is - // registering this datanode. - nodeReg.setIpAddr(dnAddress); nodeReg.setExportedKeys(blockManager.getBlockKeys()); // Checks if the node is not on the hosts list. If it is not, then @@ -1033,19 +1025,8 @@ public class DatanodeManager { if ( (isDead && listDeadNodes) || (!isDead && listLiveNodes) ) { nodes.add(dn); } - // Remove any nodes we know about from the map - try { - InetAddress inet = InetAddress.getByName(dn.getIpAddr()); - // compare hostname(:port) - mustList.remove(inet.getHostName()); - mustList.remove(inet.getHostName()+":"+dn.getXferPort()); - // compare ipaddress(:port) - mustList.remove(inet.getHostAddress().toString()); - mustList.remove(inet.getHostAddress().toString()+ ":" +dn.getXferPort()); - } catch (UnknownHostException e) { - mustList.remove(dn.getName()); - mustList.remove(dn.getIpAddr()); - LOG.warn(e); + for (String name : getNodeNamesForHostFiltering(dn)) { + mustList.remove(name); } } } @@ -1066,6 +1047,25 @@ public class DatanodeManager { return nodes; } + private static List getNodeNamesForHostFiltering(DatanodeID node) { + String ip = node.getIpAddr(); + String regHostName = node.getHostName(); + int xferPort = node.getXferPort(); + + List names = new ArrayList(); + names.add(ip); + names.add(ip + ":" + xferPort); + names.add(regHostName); + names.add(regHostName + ":" + xferPort); + + String peerHostName = node.getPeerHostName(); + if (peerHostName != null) { + names.add(peerHostName); + names.add(peerHostName + ":" + xferPort); + } + return names; + } + private void setDatanodeDead(DatanodeDescriptor node) { node.setLastUpdate(0); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java index 63371ec4741..934342eea9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hdfs; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import java.net.InetSocketAddress; +import java.security.Permission; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -46,6 +47,64 @@ public class TestDatanodeRegistration { public static final Log LOG = LogFactory.getLog(TestDatanodeRegistration.class); + private static class MonitorDNS extends SecurityManager { + int lookups = 0; + @Override + public void checkPermission(Permission perm) {} + @Override + public void checkConnect(String host, int port) { + if (port == -1) { + lookups++; + } + } + } + + /** + * Ensure the datanode manager does not do host lookup after registration, + * especially for node reports. + * @throws Exception + */ + @Test + public void testDNSLookups() throws Exception { + MonitorDNS sm = new MonitorDNS(); + System.setSecurityManager(sm); + + MiniDFSCluster cluster = null; + try { + HdfsConfiguration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(8).build(); + cluster.waitActive(); + + int initialLookups = sm.lookups; + assertTrue("dns security manager is active", initialLookups != 0); + + DatanodeManager dm = + cluster.getNamesystem().getBlockManager().getDatanodeManager(); + + // make sure no lookups occur + dm.refreshNodes(conf); + assertEquals(initialLookups, sm.lookups); + + dm.refreshNodes(conf); + assertEquals(initialLookups, sm.lookups); + + // ensure none of the reports trigger lookups + dm.getDatanodeListForReport(DatanodeReportType.ALL); + assertEquals(initialLookups, sm.lookups); + + dm.getDatanodeListForReport(DatanodeReportType.LIVE); + assertEquals(initialLookups, sm.lookups); + + dm.getDatanodeListForReport(DatanodeReportType.DEAD); + assertEquals(initialLookups, sm.lookups); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + System.setSecurityManager(null); + } + } + /** * Regression test for HDFS-894 ensures that, when datanodes * are restarted, the new IPC port is registered with the From 312eb235981166b905a668c82c146ce373ecaffa Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Fri, 9 Nov 2012 02:57:05 +0000 Subject: [PATCH 18/31] HDFS-4048. Use ERROR instead of INFO for volume failure logs. Contributed by Stephen Chu git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407345 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/common/Storage.java | 8 ++++---- .../hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cec087011da..0898deeeb34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -461,6 +461,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4046. Rename ChecksumTypeProto enum NULL since it is illegal in C/C++. (Binglin Chang via suresh) + HDFS-4048. Use ERROR instead of INFO for volume failure logs. + (Stephen Chu via eli) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 4a019b73fbe..cef7d237c24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -433,7 +433,7 @@ public abstract class Storage extends StorageInfo { if (!root.exists()) { // storage directory does not exist if (startOpt != StartupOption.FORMAT) { - LOG.info("Storage directory " + rootPath + " does not exist"); + LOG.warn("Storage directory " + rootPath + " does not exist"); return StorageState.NON_EXISTENT; } LOG.info(rootPath + " does not exist. Creating ..."); @@ -442,15 +442,15 @@ public abstract class Storage extends StorageInfo { } // or is inaccessible if (!root.isDirectory()) { - LOG.info(rootPath + "is not a directory"); + LOG.warn(rootPath + "is not a directory"); return StorageState.NON_EXISTENT; } if (!root.canWrite()) { - LOG.info("Cannot access storage directory " + rootPath); + LOG.warn("Cannot access storage directory " + rootPath); return StorageState.NON_EXISTENT; } } catch(SecurityException ex) { - LOG.info("Cannot access storage directory " + rootPath, ex); + LOG.warn("Cannot access storage directory " + rootPath, ex); return StorageState.NON_EXISTENT; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 107ce5d2f8c..66ecdbc8c1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -137,7 +137,7 @@ class FsVolumeList { if (removedVols != null && removedVols.size() > 0) { // Replace volume list volumes = Collections.unmodifiableList(volumeList); - FsDatasetImpl.LOG.info("Completed checkDirs. Removed " + removedVols.size() + FsDatasetImpl.LOG.warn("Completed checkDirs. Removed " + removedVols.size() + " volumes. Current volumes: " + this); } From 933a6d2c1ec8d3b373674e3e74eb472863fc464d Mon Sep 17 00:00:00 2001 From: Thomas White Date: Fri, 9 Nov 2012 12:38:10 +0000 Subject: [PATCH 19/31] YARN-183. Clean up fair scheduler code. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407433 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../scheduler/fair/AppSchedulable.java | 14 +- .../scheduler/fair/FSQueue.java | 2 +- .../scheduler/fair/FSQueueSchedulable.java | 14 +- .../scheduler/fair/FSSchedulerApp.java | 52 +++---- .../scheduler/fair/FSSchedulerNode.java | 17 +- .../scheduler/fair/FairScheduler.java | 146 ++++++++---------- ...tBooster.java => NewAppWeightBooster.java} | 2 +- .../scheduler/fair/QueueManager.java | 25 +-- .../scheduler/fair/WeightAdjuster.java | 2 +- 10 files changed, 132 insertions(+), 144 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/{NewJobWeightBooster.java => NewAppWeightBooster.java} (97%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7ff7e44efe8..97636dabc70 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -69,6 +69,8 @@ Release 2.0.3-alpha - Unreleased YARN-136. Make ClientToAMTokenSecretManager part of RMContext (Vinod Kumar Vavilapalli via sseth) + YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 3f97c96f457..acad730ee82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -101,7 +101,7 @@ public class AppSchedulable extends Schedulable { @Override public Resource getResourceUsage() { - return this.app.getCurrentConsumption(); + return app.getCurrentConsumption(); } @@ -114,7 +114,7 @@ public class AppSchedulable extends Schedulable { * Get metrics reference from containing queue. */ public QueueMetrics getMetrics() { - return this.queue.getQueueSchedulable().getMetrics(); + return queue.getQueueSchedulable().getMetrics(); } @Override @@ -190,9 +190,9 @@ public class AppSchedulable extends Schedulable { RMContainer rmContainer = application.reserve(node, priority, null, container); node.reserveResource(application, priority, rmContainer); - getMetrics().reserveResource(this.app.getUser(), + getMetrics().reserveResource(app.getUser(), container.getResource()); - scheduler.getRootQueueMetrics().reserveResource(this.app.getUser(), + scheduler.getRootQueueMetrics().reserveResource(app.getUser(), container.getResource()); } @@ -257,13 +257,13 @@ public class AppSchedulable extends Schedulable { // TODO this should subtract resource just assigned // TEMPROARY getMetrics().setAvailableResourcesToQueue( - this.scheduler.getClusterCapacity()); + scheduler.getClusterCapacity()); } // If we had previously made a reservation, delete it if (reserved) { - this.unreserve(application, priority, node); + unreserve(application, priority, node); } // Inform the node @@ -290,7 +290,7 @@ public class AppSchedulable extends Schedulable { // Make sure the application still needs requests at this priority if (app.getTotalRequiredResources(priority) == 0) { - this.unreserve(app, priority, node); + unreserve(app, priority, node); return Resources.none(); } } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 31508d3d162..79395b0d1d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -61,7 +61,7 @@ public class FSQueue { queueSchedulable.addApp(appSchedulable); } - public void removeJob(FSSchedulerApp app) { + public void removeApp(FSSchedulerApp app) { applications.remove(app); queueSchedulable.removeApp(app); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java index 592b310e458..ccac112e6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java @@ -80,7 +80,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue { this.scheduler = scheduler; this.queue = queue; this.queueMgr = scheduler.getQueueManager(); - this.metrics = QueueMetrics.forQueue(this.getName(), null, true, scheduler.getConf()); + this.metrics = QueueMetrics.forQueue(getName(), null, true, scheduler.getConf()); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtHalfFairShare = scheduler.getClock().getTime(); } @@ -113,7 +113,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue { Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + sched.getName() + " " + toAdd - + "; Total resource consumption for " + this.getName() + " now " + + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); @@ -123,7 +123,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue { } } if (LOG.isDebugEnabled()) { - LOG.debug("The updated demand for " + this.getName() + " is " + demand + LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxRes); } } @@ -164,9 +164,9 @@ public class FSQueueSchedulable extends Schedulable implements Queue { @Override public Resource assignContainer(FSSchedulerNode node, boolean reserved) { - LOG.debug("Node offered to queue: " + this.getName() + " reserved: " + reserved); + LOG.debug("Node offered to queue: " + getName() + " reserved: " + reserved); // If this queue is over its limit, reject - if (Resources.greaterThan(this.getResourceUsage(), + if (Resources.greaterThan(getResourceUsage(), queueMgr.getMaxResources(queue.getName()))) { return Resources.none(); } @@ -258,7 +258,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue { @Override public Map getQueueAcls() { - Map acls = this.queueMgr.getQueueAcls(this.getName()); + Map acls = queueMgr.getQueueAcls(getName()); return new HashMap(acls); } @@ -284,7 +284,7 @@ public class FSQueueSchedulable extends Schedulable implements Queue { recordFactory.newRecordInstance(QueueUserACLInfo.class); List operations = new ArrayList(); for (QueueACL operation : QueueACL.values()) { - Map acls = this.queueMgr.getQueueAcls(this.getName()); + Map acls = queueMgr.getQueueAcls(getName()); if (acls.get(operation).isUserAllowed(user)) { operations.add(operation); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index e2a385f26c3..4e164e576f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -112,12 +112,12 @@ public class FSSchedulerApp extends SchedulerApplication { } public ApplicationId getApplicationId() { - return this.appSchedulingInfo.getApplicationId(); + return appSchedulingInfo.getApplicationId(); } @Override public ApplicationAttemptId getApplicationAttemptId() { - return this.appSchedulingInfo.getApplicationAttemptId(); + return appSchedulingInfo.getApplicationAttemptId(); } public void setAppSchedulable(AppSchedulable appSchedulable) { @@ -129,7 +129,7 @@ public class FSSchedulerApp extends SchedulerApplication { } public String getUser() { - return this.appSchedulingInfo.getUser(); + return appSchedulingInfo.getUser(); } public synchronized void updateResourceRequests( @@ -138,19 +138,19 @@ public class FSSchedulerApp extends SchedulerApplication { } public Map getResourceRequests(Priority priority) { - return this.appSchedulingInfo.getResourceRequests(priority); + return appSchedulingInfo.getResourceRequests(priority); } public int getNewContainerId() { - return this.appSchedulingInfo.getNewContainerId(); + return appSchedulingInfo.getNewContainerId(); } public Collection getPriorities() { - return this.appSchedulingInfo.getPriorities(); + return appSchedulingInfo.getPriorities(); } public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) { - return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress); + return appSchedulingInfo.getResourceRequest(priority, nodeAddress); } public synchronized int getTotalRequiredResources(Priority priority) { @@ -158,7 +158,7 @@ public class FSSchedulerApp extends SchedulerApplication { } public Resource getResource(Priority priority) { - return this.appSchedulingInfo.getResource(priority); + return appSchedulingInfo.getResource(priority); } /** @@ -167,11 +167,11 @@ public class FSSchedulerApp extends SchedulerApplication { */ @Override public boolean isPending() { - return this.appSchedulingInfo.isPending(); + return appSchedulingInfo.isPending(); } public String getQueueName() { - return this.appSchedulingInfo.getQueueName(); + return appSchedulingInfo.getQueueName(); } /** @@ -185,7 +185,7 @@ public class FSSchedulerApp extends SchedulerApplication { public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) { // Cleanup all scheduling information - this.appSchedulingInfo.stop(rmAppAttemptFinalState); + appSchedulingInfo.stop(rmAppAttemptFinalState); } @SuppressWarnings("unchecked") @@ -196,7 +196,7 @@ public class FSSchedulerApp extends SchedulerApplication { getRMContainer(containerId); if (rmContainer == null) { // Some unknown container sneaked into the system. Kill it. - this.rmContext.getDispatcher().getEventHandler() + rmContext.getDispatcher().getEventHandler() .handle(new RMNodeCleanContainerEvent(nodeId, containerId)); return; } @@ -272,7 +272,7 @@ public class FSSchedulerApp extends SchedulerApplication { } synchronized public void addSchedulingOpportunity(Priority priority) { - this.schedulingOpportunities.setCount(priority, + schedulingOpportunities.setCount(priority, schedulingOpportunities.count(priority) + 1); } @@ -282,19 +282,19 @@ public class FSSchedulerApp extends SchedulerApplication { * successfully did so. */ synchronized public int getSchedulingOpportunities(Priority priority) { - return this.schedulingOpportunities.count(priority); + return schedulingOpportunities.count(priority); } synchronized void resetReReservations(Priority priority) { - this.reReservations.setCount(priority, 0); + reReservations.setCount(priority, 0); } synchronized void addReReservation(Priority priority) { - this.reReservations.add(priority); + reReservations.add(priority); } synchronized public int getReReservations(Priority priority) { - return this.reReservations.count(priority); + return reReservations.count(priority); } public synchronized int getNumReservedContainers(Priority priority) { @@ -458,8 +458,8 @@ public class FSSchedulerApp extends SchedulerApplication { * @param priority The priority of the container scheduled. */ synchronized public void resetSchedulingOpportunities(Priority priority) { - this.lastScheduledContainer.put(priority, System.currentTimeMillis()); - this.schedulingOpportunities.setCount(priority, 0); + lastScheduledContainer.put(priority, System.currentTimeMillis()); + schedulingOpportunities.setCount(priority, 0); } /** @@ -494,14 +494,14 @@ public class FSSchedulerApp extends SchedulerApplication { rackLocalityThreshold; // Relax locality constraints once we've surpassed threshold. - if (this.getSchedulingOpportunities(priority) > (numNodes * threshold)) { + if (getSchedulingOpportunities(priority) > (numNodes * threshold)) { if (allowed.equals(NodeType.NODE_LOCAL)) { allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL); - this.resetSchedulingOpportunities(priority); + resetSchedulingOpportunities(priority); } else if (allowed.equals(NodeType.RACK_LOCAL)) { allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH); - this.resetSchedulingOpportunities(priority); + resetSchedulingOpportunities(priority); } } return allowedLocalityLevel.get(priority); @@ -512,7 +512,7 @@ public class FSSchedulerApp extends SchedulerApplication { Priority priority, ResourceRequest request, Container container) { // Update allowed locality level - NodeType allowed = this.allowedLocalityLevel.get(priority); + NodeType allowed = allowedLocalityLevel.get(priority); if (allowed != null) { if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(NodeType.NODE_LOCAL) || @@ -532,9 +532,9 @@ public class FSSchedulerApp extends SchedulerApplication { } // Create RMContainer - RMContainer rmContainer = new RMContainerImpl(container, this - .getApplicationAttemptId(), node.getNodeID(), this.rmContext - .getDispatcher().getEventHandler(), this.rmContext + RMContainer rmContainer = new RMContainerImpl(container, + getApplicationAttemptId(), node.getNodeID(), rmContext + .getDispatcher().getEventHandler(), rmContext .getContainerAllocationExpirer()); // Add it to allContainers list. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index b8cef425d0e..2833ca67b95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -67,25 +67,25 @@ public class FSSchedulerNode extends SchedulerNode { } public RMNode getRMNode() { - return this.rmNode; + return rmNode; } public NodeId getNodeID() { - return this.rmNode.getNodeID(); + return rmNode.getNodeID(); } public String getHttpAddress() { - return this.rmNode.getHttpAddress(); + return rmNode.getHttpAddress(); } @Override public String getHostName() { - return this.rmNode.getHostName(); + return rmNode.getHostName(); } @Override public String getRackName() { - return this.rmNode.getRackName(); + return rmNode.getRackName(); } /** @@ -112,17 +112,18 @@ public class FSSchedulerNode extends SchedulerNode { @Override public synchronized Resource getAvailableResource() { - return this.availableResource; + return availableResource; } @Override public synchronized Resource getUsedResource() { - return this.usedResource; + return usedResource; } private synchronized boolean isValidContainer(Container c) { - if (launchedContainers.containsKey(c.getId())) + if (launchedContainers.containsKey(c.getId())) { return true; + } return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1d2412e5ffb..f011802f77e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -139,11 +139,11 @@ public class FairScheduler implements ResourceScheduler { public FairSchedulerConfiguration getConf() { - return this.conf; + return conf; } public QueueManager getQueueManager() { - return this.queueMgr; + return queueMgr; } public List getQueueSchedulables() { @@ -183,36 +183,34 @@ public class FairScheduler implements ResourceScheduler { * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ - protected void update() { - synchronized (this) { - queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file - updateRunnability(); // Set job runnability based on user/queue limits - updatePreemptionVariables(); // Determine if any queues merit preemption + protected synchronized void update() { + queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file + updateRunnability(); // Set job runnability based on user/queue limits + updatePreemptionVariables(); // Determine if any queues merit preemption - // Update demands of apps and queues - for (FSQueue queue: queueMgr.getQueues()) { - queue.getQueueSchedulable().updateDemand(); - } - - // Compute fair shares based on updated demands - List queueScheds = this.getQueueSchedulables(); - SchedulingAlgorithms.computeFairShares( - queueScheds, clusterCapacity); - - // Update queue metrics for this queue - for (FSQueueSchedulable sched : queueScheds) { - sched.getMetrics().setAvailableResourcesToQueue(sched.getFairShare()); - } - - // Use the computed shares to assign shares within each queue - for (FSQueue queue: queueMgr.getQueues()) { - queue.getQueueSchedulable().redistributeShare(); - } - - // Update recorded capacity of root queue (child queues are updated - // when fair share is calculated). - rootMetrics.setAvailableResourcesToQueue(clusterCapacity); + // Update demands of apps and queues + for (FSQueue queue: queueMgr.getQueues()) { + queue.getQueueSchedulable().updateDemand(); } + + // Compute fair shares based on updated demands + List queueScheds = getQueueSchedulables(); + SchedulingAlgorithms.computeFairShares( + queueScheds, clusterCapacity); + + // Update queue metrics for this queue + for (FSQueueSchedulable sched : queueScheds) { + sched.getMetrics().setAvailableResourcesToQueue(sched.getFairShare()); + } + + // Use the computed shares to assign shares within each queue + for (FSQueue queue: queueMgr.getQueues()) { + queue.getQueueSchedulable().redistributeShare(); + } + + // Update recorded capacity of root queue (child queues are updated + // when fair share is calculated). + rootMetrics.setAvailableResourcesToQueue(clusterCapacity); } /** @@ -257,17 +255,16 @@ public class FairScheduler implements ResourceScheduler { * have been below half their fair share for the fairSharePreemptionTimeout. * If such queues exist, compute how many tasks of each type need to be * preempted and then select the right ones using preemptTasks. - * - * This method computes and logs the number of tasks we want to preempt even - * if preemption is disabled, for debugging purposes. */ protected void preemptTasksIfNecessary() { - if (!preemptionEnabled) + if (!preemptionEnabled) { return; + } long curTime = clock.getTime(); - if (curTime - lastPreemptCheckTime < preemptionInterval) + if (curTime - lastPreemptCheckTime < preemptionInterval) { return; + } lastPreemptCheckTime = curTime; Resource resToPreempt = Resources.none(); @@ -288,8 +285,9 @@ public class FairScheduler implements ResourceScheduler { * lowest priority to preempt. */ protected void preemptResources(List scheds, Resource toPreempt) { - if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) + if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) { return; + } Map apps = new HashMap(); @@ -330,7 +328,7 @@ public class FairScheduler implements ResourceScheduler { // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). - this.completedContainer(container, status, RMContainerEventType.KILL); + completedContainer(container, status, RMContainerEventType.KILL); toPreempt = Resources.subtract(toPreempt, container.getContainer().getResource()); @@ -413,7 +411,7 @@ public class FairScheduler implements ResourceScheduler { } public RMContainerTokenSecretManager getContainerTokenSecretManager() { - return this.rmContext.getContainerTokenSecretManager(); + return rmContext.getContainerTokenSecretManager(); } public double getAppWeight(AppSchedulable app) { @@ -437,28 +435,28 @@ public class FairScheduler implements ResourceScheduler { @Override public Resource getMinimumResourceCapability() { - return this.minimumAllocation; + return minimumAllocation; } @Override public Resource getMaximumResourceCapability() { - return this.maximumAllocation; + return maximumAllocation; } public double getNodeLocalityThreshold() { - return this.nodeLocalityThreshold; + return nodeLocalityThreshold; } public double getRackLocalityThreshold() { - return this.rackLocalityThreshold; + return rackLocalityThreshold; } public Resource getClusterCapacity() { - return this.clusterCapacity; + return clusterCapacity; } public Clock getClock() { - return this.clock; + return clock; } protected void setClock(Clock clock) { @@ -478,11 +476,11 @@ public class FairScheduler implements ResourceScheduler { addApplication(ApplicationAttemptId applicationAttemptId, String queueName, String user) { - FSQueue queue = this.queueMgr.getQueue(queueName); + FSQueue queue = queueMgr.getQueue(queueName); FSSchedulerApp schedulerApp = new FSSchedulerApp(applicationAttemptId, user, - queue.getQueueSchedulable(), new ActiveUsersManager(this.getRootQueueMetrics()), + queue.getQueueSchedulable(), new ActiveUsersManager(getRootQueueMetrics()), rmContext, null); // Inforce ACLs @@ -553,8 +551,8 @@ public class FairScheduler implements ResourceScheduler { application.stop(rmAppAttemptFinalState); // Inform the queue - FSQueue queue = this.queueMgr.getQueue(application.getQueue().getQueueName()); - queue.removeJob(application); + FSQueue queue = queueMgr.getQueue(application.getQueue().getQueueName()); + queue.removeApp(application); // Remove from our data-structure applications.remove(applicationAttemptId); @@ -600,7 +598,7 @@ public class FairScheduler implements ResourceScheduler { } private synchronized void addNode(RMNode node) { - this.nodes.put(node.getNodeID(), new FSSchedulerNode(node)); + nodes.put(node.getNodeID(), new FSSchedulerNode(node)); Resources.addTo(clusterCapacity, node.getTotalCapability()); LOG.info("Added node " + node.getNodeAddress() + @@ -608,7 +606,7 @@ public class FairScheduler implements ResourceScheduler { } private synchronized void removeNode(RMNode rmNode) { - FSSchedulerNode node = this.nodes.get(rmNode.getNodeID()); + FSSchedulerNode node = nodes.get(rmNode.getNodeID()); Resources.subtractFrom(clusterCapacity, rmNode.getTotalCapability()); // Remove running containers @@ -631,7 +629,7 @@ public class FairScheduler implements ResourceScheduler { RMContainerEventType.KILL); } - this.nodes.remove(rmNode.getNodeID()); + nodes.remove(rmNode.getNodeID()); LOG.info("Removed node " + rmNode.getNodeAddress() + " cluster capacity: " + clusterCapacity); } @@ -669,10 +667,8 @@ public class FairScheduler implements ResourceScheduler { } synchronized (application) { - if (!ask.isEmpty()) { - - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("allocate: pre-update" + " applicationAttemptId=" + appAttemptId + " application=" + application.getApplicationId()); @@ -686,7 +682,7 @@ public class FairScheduler implements ResourceScheduler { application.showRequests(); } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("allocate:" + " applicationAttemptId=" + appAttemptId + " #ask=" + ask.size()); @@ -764,7 +760,7 @@ public class FairScheduler implements ResourceScheduler { int assignedContainers = 0; while (true) { // At most one task is scheduled each iteration of this loop - List scheds = this.getQueueSchedulables(); + List scheds = getQueueSchedulables(); Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator()); boolean assignedContainer = false; for (FSQueueSchedulable sched : scheds) { @@ -796,11 +792,11 @@ public class FairScheduler implements ResourceScheduler { @Override public SchedulerAppReport getSchedulerAppInfo( ApplicationAttemptId appAttemptId) { - if (!this.applications.containsKey(appAttemptId)) { + if (!applications.containsKey(appAttemptId)) { LOG.error("Request for appInfo of unknown attempt" + appAttemptId); return null; } - return new SchedulerAppReport(this.applications.get(appAttemptId)); + return new SchedulerAppReport(applications.get(appAttemptId)); } @Override @@ -812,37 +808,30 @@ public class FairScheduler implements ResourceScheduler { public void handle(SchedulerEvent event) { switch(event.getType()) { case NODE_ADDED: - { if (!(event instanceof NodeAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event; addNode(nodeAddedEvent.getAddedRMNode()); - } - break; + break; case NODE_REMOVED: - { if (!(event instanceof NodeRemovedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event; removeNode(nodeRemovedEvent.getRemovedRMNode()); - } - break; + break; case NODE_UPDATE: - { if (!(event instanceof NodeUpdateSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; - this.nodeUpdate(nodeUpdatedEvent.getRMNode(), + nodeUpdate(nodeUpdatedEvent.getRMNode(), nodeUpdatedEvent.getNewlyLaunchedContainers(), nodeUpdatedEvent.getCompletedContainers()); - } - break; + break; case APP_ADDED: - { if (!(event instanceof AppAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } @@ -857,20 +846,16 @@ public class FairScheduler implements ResourceScheduler { addApplication(appAddedEvent.getApplicationAttemptId(), queue, appAddedEvent.getUser()); - } - break; + break; case APP_REMOVED: - { if (!(event instanceof AppRemovedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; - this.removeApplication(appRemovedEvent.getApplicationAttemptID(), + removeApplication(appRemovedEvent.getApplicationAttemptID(), appRemovedEvent.getFinalAttemptState()); - } - break; + break; case CONTAINER_EXPIRED: - { if (!(event instanceof ContainerExpiredSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } @@ -882,8 +867,7 @@ public class FairScheduler implements ResourceScheduler { containerId, SchedulerUtils.EXPIRED_CONTAINER), RMContainerEventType.EXPIRE); - } - break; + break; default: LOG.error("Unknown event arrived at FairScheduler: " + event.toString()); } @@ -897,9 +881,9 @@ public class FairScheduler implements ResourceScheduler { @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { - if (!this.initialized) { + if (!initialized) { this.conf = new FairSchedulerConfiguration(conf); - this.rootMetrics = QueueMetrics.forQueue("root", null, true, conf); + rootMetrics = QueueMetrics.forQueue("root", null, true, conf); this.rmContext = rmContext; this.clock = new SystemClock(); this.eventLog = new FairSchedulerEventLog(); @@ -973,7 +957,7 @@ public class FairScheduler implements ResourceScheduler { @Override public int getNumClusterNodes() { - return this.nodes.size(); + return nodes.size(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewJobWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java similarity index 97% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewJobWeightBooster.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java index c643027904a..e77eed79568 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewJobWeightBooster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java @@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configured; */ @Private @Unstable -public class NewJobWeightBooster extends Configured implements WeightAdjuster { +public class NewAppWeightBooster extends Configured implements WeightAdjuster { private static final float DEFAULT_FACTOR = 3; private static final long DEFAULT_DURATION = 5 * 60 * 1000; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index c765e7f7dab..0395eaad5c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -202,7 +202,7 @@ public class QueueManager { * Get the queue for a given AppSchedulable. */ public FSQueue getQueueForApp(AppSchedulable app) { - return this.getQueue(app.getApp().getQueueName()); + return getQueue(app.getApp().getQueueName()); } /** @@ -388,7 +388,7 @@ public class QueueManager { // Commit the reload; also create any queue defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. - synchronized(this) { + synchronized (this) { setMinResources(minQueueResources); setMaxResources(maxQueueResources); setQueueMaxApps(queueMaxApps); @@ -431,14 +431,14 @@ public class QueueManager { synchronized(minQueueResourcesMO) { if (minQueueResources.containsKey(queue)) { return minQueueResources.get(queue); - } else{ + } else { return Resources.createResource(0); } } } private void setMinResources(Map resources) { - synchronized(minQueueResourcesMO) { + synchronized (minQueueResourcesMO) { minQueueResources = resources; } } @@ -457,7 +457,7 @@ public class QueueManager { } private void setMaxResources(Map resources) { - synchronized(maxQueueResourcesMO) { + synchronized (maxQueueResourcesMO) { maxQueueResources = resources; } } @@ -472,8 +472,8 @@ public class QueueManager { /** * Remove an app */ - public synchronized void removeJob(FSSchedulerApp app) { - getQueue(app.getQueueName()).removeJob(app); + public synchronized void removeApp(FSSchedulerApp app) { + getQueue(app.getQueueName()).removeApp(app); } /** @@ -543,7 +543,7 @@ public class QueueManager { } private int getQueueMaxAppsDefault(){ - synchronized(queueMaxAppsDefaultMO) { + synchronized (queueMaxAppsDefaultMO) { return queueMaxAppsDefault; } } @@ -575,11 +575,12 @@ public class QueueManager { queueWeights = weights; } } + /** - * Get a queue's min share preemption timeout, in milliseconds. This is the - * time after which jobs in the queue may kill other queues' tasks if they - * are below their min share. - */ + * Get a queue's min share preemption timeout, in milliseconds. This is the + * time after which jobs in the queue may kill other queues' tasks if they + * are below their min share. + */ public long getMinSharePreemptionTimeout(String queueName) { synchronized (minSharePreemptionTimeoutsMO) { if (minSharePreemptionTimeouts.containsKey(queueName)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java index 2fa71badbc7..1a9467fc003 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java @@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configurable; /** * A pluggable object for altering the weights of apps in the fair scheduler, - * which is used for example by {@link NewJobWeightBooster} to give higher + * which is used for example by {@link NewAppWeightBooster} to give higher * weight to new jobs so that short jobs finish faster. * * May implement {@link Configurable} to access configuration parameters. From 1f40b8b4e8ce8c4876c1b57012cbd12332d0c096 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 9 Nov 2012 15:52:33 +0000 Subject: [PATCH 20/31] MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit (Mark Fuhs via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407505 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/lib/input/NLineInputFormat.java | 32 +++++++++++-------- .../lib/input/TestNLineInputFormat.java | 32 ++++++++++++------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bd40d5ab947..78fdfb99e2e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -640,6 +640,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4772. Fetch failures can take way too long for a map to be restarted (bobby) + + MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit + (Mark Fuhs via bobby) Release 0.23.4 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java index 2c7c63dd552..758996165f4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java @@ -107,25 +107,14 @@ public class NLineInputFormat extends FileInputFormat { numLines++; length += num; if (numLines == numLinesPerSplit) { - // NLineInputFormat uses LineRecordReader, which always reads - // (and consumes) at least one character out of its upper split - // boundary. So to make sure that each mapper gets N lines, we - // move back the upper split limits of each split - // by one character here. - if (begin == 0) { - splits.add(new FileSplit(fileName, begin, length - 1, - new String[] {})); - } else { - splits.add(new FileSplit(fileName, begin - 1, length, - new String[] {})); - } + splits.add(createFileSplit(fileName, begin, length)); begin += length; length = 0; numLines = 0; } } if (numLines != 0) { - splits.add(new FileSplit(fileName, begin, length, new String[]{})); + splits.add(createFileSplit(fileName, begin, length)); } } finally { if (lr != null) { @@ -134,6 +123,23 @@ public class NLineInputFormat extends FileInputFormat { } return splits; } + + /** + * NLineInputFormat uses LineRecordReader, which always reads + * (and consumes) at least one character out of its upper split + * boundary. So to make sure that each mapper gets N lines, we + * move back the upper split limits of each split + * by one character here. + * @param fileName Path of file + * @param begin the position of the first byte in the file to process + * @param length number of bytes in InputSplit + * @return FileSplit + */ + protected static FileSplit createFileSplit(Path fileName, long begin, long length) { + return (begin == 0) + ? new FileSplit(fileName, begin, length - 1, new String[] {}) + : new FileSplit(fileName, begin - 1, length, new String[] {}); + } /** * Set the number of lines per split diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java index aca8e53af53..7b3878d9475 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java @@ -50,37 +50,40 @@ public class TestNLineInputFormat extends TestCase { Job job = Job.getInstance(conf); Path file = new Path(workDir, "test.txt"); - int seed = new Random().nextInt(); - Random random = new Random(seed); - localFs.delete(workDir, true); FileInputFormat.setInputPaths(job, workDir); int numLinesPerMap = 5; NLineInputFormat.setNumLinesPerSplit(job, numLinesPerMap); - // for a variety of lengths for (int length = 0; length < MAX_LENGTH; - length += random.nextInt(MAX_LENGTH / 10) + 1) { + length += 1) { + // create a file with length entries Writer writer = new OutputStreamWriter(localFs.create(file)); try { for (int i = 0; i < length; i++) { - writer.write(Integer.toString(i)); + writer.write(Integer.toString(i)+" some more text"); writer.write("\n"); } } finally { writer.close(); } - checkFormat(job, numLinesPerMap); + int lastN = 0; + if (length != 0) { + lastN = length % 5; + if (lastN == 0) { + lastN = 5; + } + } + checkFormat(job, numLinesPerMap, lastN); } } - void checkFormat(Job job, int expectedN) + void checkFormat(Job job, int expectedN, int lastN) throws IOException, InterruptedException { NLineInputFormat format = new NLineInputFormat(); List splits = format.getSplits(job); - // check all splits except last one int count = 0; - for (int i = 0; i < splits.size() -1; i++) { + for (int i = 0; i < splits.size(); i++) { assertEquals("There are no split locations", 0, splits.get(i).getLocations().length); TaskAttemptContext context = MapReduceTestUtil. @@ -104,8 +107,13 @@ public class TestNLineInputFormat extends TestCase { } finally { reader.close(); } - assertEquals("number of lines in split is " + expectedN , - expectedN, count); + if ( i == splits.size() - 1) { + assertEquals("number of lines in split(" + i + ") is wrong" , + lastN, count); + } else { + assertEquals("number of lines in split(" + i + ") is wrong" , + expectedN, count); + } } } From ca1c683b4755958b30af7b6d8dc8c5b9fa55c60b Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 9 Nov 2012 17:26:40 +0000 Subject: [PATCH 21/31] MAPREDUCE-4266. remove Ant remnants from MR (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407551 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + hadoop-mapreduce-project/build-utils.xml | 33 - hadoop-mapreduce-project/build.xml | 1912 --- hadoop-mapreduce-project/ivy.xml | 167 - .../ivy/hadoop-mapred-examples-template.xml | 28 - .../hadoop-mapred-instrumented-template.xml | 34 - ...doop-mapred-instrumented-test-template.xml | 34 - .../ivy/hadoop-mapred-template.xml | 34 - .../ivy/hadoop-mapred-test-template.xml | 34 - .../ivy/hadoop-mapred-tools-template.xml | 28 - hadoop-mapreduce-project/ivy/ivysettings.xml | 70 - .../ivy/libraries.properties | 86 - .../src/benchmarks/gridmix/README | 168 - .../src/benchmarks/gridmix/generateData.sh | 90 - .../src/benchmarks/gridmix/gridmix-env | 86 - .../gridmix/javasort/text-sort.large | 25 - .../gridmix/javasort/text-sort.medium | 25 - .../gridmix/javasort/text-sort.small | 25 - .../benchmarks/gridmix/maxent/maxent.large | 37 - .../gridmix/monsterQuery/monster_query.large | 38 - .../gridmix/monsterQuery/monster_query.medium | 38 - .../gridmix/monsterQuery/monster_query.small | 38 - .../gridmix/pipesort/text-sort.large | 27 - .../gridmix/pipesort/text-sort.medium | 27 - .../gridmix/pipesort/text-sort.small | 27 - .../gridmix/streamsort/text-sort.large | 27 - .../gridmix/streamsort/text-sort.medium | 27 - .../gridmix/streamsort/text-sort.small | 27 - .../gridmix/submissionScripts/allThroughHod | 24 - .../submissionScripts/allToSameCluster | 38 - .../gridmix/submissionScripts/maxentHod | 26 - .../submissionScripts/maxentToSameCluster | 30 - .../submissionScripts/monsterQueriesHod | 44 - .../monsterQueriesToSameCluster | 45 - .../submissionScripts/sleep_if_too_busy | 21 - .../gridmix/submissionScripts/textSortHod | 75 - .../submissionScripts/textSortToSameCluster | 64 - .../gridmix/submissionScripts/webdataScanHod | 45 - .../webdataScanToSameCluster | 45 - .../gridmix/submissionScripts/webdataSortHod | 28 - .../webdataSortToSameCluster | 29 - .../gridmix/webdatascan/webdata_scan.large | 25 - .../gridmix/webdatascan/webdata_scan.medium | 25 - .../gridmix/webdatascan/webdata_scan.small | 25 - .../gridmix/webdatasort/webdata_sort.large | 27 - .../gridmix/webdatasort/webdata_sort.medium | 27 - .../gridmix/webdatasort/webdata_sort.small | 27 - .../src/benchmarks/gridmix2/README.gridmix2 | 148 - .../src/benchmarks/gridmix2/build.xml | 100 - .../gridmix2/generateGridmix2data.sh | 94 - .../src/benchmarks/gridmix2/gridmix-env-2 | 35 - .../benchmarks/gridmix2/gridmix_config.xml | 567 - .../src/benchmarks/gridmix2/rungridmix_2 | 37 - .../hadoop/mapreduce/CombinerJobCreator.java | 85 - .../mapreduce/GenericMRLoadJobCreator.java | 100 - .../hadoop/mapreduce/GridMixRunner.java | 680 - .../src/c++/librecordio/Makefile | 73 - .../src/c++/librecordio/archive.hh | 122 - .../src/c++/librecordio/binarchive.cc | 330 - .../src/c++/librecordio/binarchive.hh | 81 - .../src/c++/librecordio/csvarchive.cc | 368 - .../src/c++/librecordio/csvarchive.hh | 128 - .../src/c++/librecordio/exception.cc | 152 - .../src/c++/librecordio/exception.hh | 129 - .../src/c++/librecordio/fieldTypeInfo.cc | 64 - .../src/c++/librecordio/fieldTypeInfo.hh | 59 - .../src/c++/librecordio/filestream.cc | 98 - .../src/c++/librecordio/filestream.hh | 55 - .../src/c++/librecordio/recordTypeInfo.cc | 143 - .../src/c++/librecordio/recordTypeInfo.hh | 68 - .../src/c++/librecordio/recordio.cc | 75 - .../src/c++/librecordio/recordio.hh | 82 - .../src/c++/librecordio/test/Makefile | 51 - .../src/c++/librecordio/test/test.cc | 309 - .../src/c++/librecordio/test/test.hh | 26 - .../src/c++/librecordio/test/test.jr | 63 - .../src/c++/librecordio/test/testFromJava.cc | 71 - .../src/c++/librecordio/test/testFromJava.hh | 26 - .../src/c++/librecordio/typeIDs.cc | 274 - .../src/c++/librecordio/typeIDs.hh | 169 - .../src/c++/librecordio/typeInfo.cc | 69 - .../src/c++/librecordio/typeInfo.hh | 56 - .../src/c++/librecordio/utils.cc | 109 - .../src/c++/librecordio/utils.hh | 50 - .../src/c++/librecordio/xmlarchive.cc | 431 - .../src/c++/librecordio/xmlarchive.hh | 265 - .../src/contrib/block_forensics/README | 25 - .../src/contrib/block_forensics/build.xml | 66 - .../client/BlockForensics.java | 186 - .../src/contrib/block_forensics/ivy.xml | 52 - .../block_forensics/ivy/libraries.properties | 21 - .../hadoop/blockforensics/BlockSearch.java | 136 - .../src/contrib/build-contrib.xml | 531 - .../src/contrib/build.xml | 100 - .../src/contrib/data_join/build.xml | 45 - .../src/contrib/data_join/ivy.xml | 68 - .../data_join/ivy/libraries.properties | 17 - .../src/contrib/eclipse-plugin/.classpath | 26 - .../src/contrib/eclipse-plugin/.project | 45 - .../.settings/org.eclipse.jdt.core.prefs | 274 - .../.settings/org.eclipse.jdt.ui.prefs | 17 - .../org.eclipse.wst.validation.prefs | 17 - .../eclipse-plugin/META-INF/MANIFEST.MF | 28 - .../contrib/eclipse-plugin/build.properties | 19 - .../src/contrib/eclipse-plugin/build.xml | 82 - .../src/contrib/eclipse-plugin/ivy.xml | 58 - .../eclipse-plugin/ivy/libraries.properties | 17 - .../src/contrib/eclipse-plugin/plugin.xml | 304 - .../resources/Components/Conf.png | Bin 1006 -> 0 bytes .../resources/Components/Export.png | Bin 2500 -> 0 bytes .../resources/Components/Import.png | Bin 2288 -> 0 bytes .../resources/Components/New.png | Bin 359 -> 0 bytes .../resources/Components/Reload.png | Bin 1455 -> 0 bytes .../resources/Components/Tool.png | Bin 853 -> 0 bytes .../resources/Components/Tools.png | Bin 1202 -> 0 bytes .../eclipse-plugin/resources/ConnectDFS.xml | 49 - .../eclipse-plugin/resources/CreateProj.xml | 79 - .../resources/Elephant-16x16.png | Bin 866 -> 0 bytes .../resources/Elephant-24x24.png | Bin 1671 -> 0 bytes .../resources/Elephant-32x32.png | Bin 2513 -> 0 bytes .../resources/Elephant-64x64.png | Bin 31779 -> 0 bytes .../resources/Elephant-small-16x16.png | Bin 809 -> 0 bytes .../eclipse-plugin/resources/Elephant.jpg | Bin 4105 -> 0 bytes .../resources/Elephant100x100.gif | Bin 7553 -> 0 bytes .../resources/Elephant16x16.gif | Bin 1080 -> 0 bytes .../resources/Elephant2-136x136.png | Bin 23414 -> 0 bytes .../resources/Elephant2-16x16.png | Bin 790 -> 0 bytes .../resources/Elephant2-24x24.png | Bin 1416 -> 0 bytes .../resources/Elephant2-32x32.png | Bin 2165 -> 0 bytes .../resources/Elephant2-64x64.png | Bin 6662 -> 0 bytes .../eclipse-plugin/resources/Elephant2.jpg | Bin 3562 -> 0 bytes .../resources/Elephant3-122x122.png | Bin 16171 -> 0 bytes .../resources/Elephant3-16x16.png | Bin 777 -> 0 bytes .../resources/Elephant3-24x24.png | Bin 1324 -> 0 bytes .../eclipse-plugin/resources/HelloWorld.xml | 138 - .../eclipse-plugin/resources/MAP100x100.gif | Bin 6978 -> 0 bytes .../eclipse-plugin/resources/MAP16x15.gif | Bin 1065 -> 0 bytes .../eclipse-plugin/resources/RunProj.xml | 41 - .../resources/SetHadoopPath.xml | 42 - .../eclipse-plugin/resources/Setup.xml | 35 - .../eclipse-plugin/resources/download.png | Bin 661 -> 0 bytes .../eclipse-plugin/resources/drive100x100.gif | Bin 5903 -> 0 bytes .../eclipse-plugin/resources/drive16x16.gif | Bin 1071 -> 0 bytes .../eclipse-plugin/resources/driver.png | Bin 820 -> 0 bytes .../eclipse-plugin/resources/driverwiz.png | Bin 4431 -> 0 bytes .../resources/elephantblue16x16.gif | Bin 1053 -> 0 bytes .../eclipse-plugin/resources/files.gif | Bin 339 -> 0 bytes .../resources/hadoop-logo-16x16.png | Bin 808 -> 0 bytes .../resources/hadoop-logo-24x24.png | Bin 1422 -> 0 bytes .../resources/hadoop-logo-85x85.png | Bin 10725 -> 0 bytes .../eclipse-plugin/resources/hadoop-logo.jpg | Bin 9443 -> 0 bytes .../eclipse-plugin/resources/hadoop.gif | Bin 1465 -> 0 bytes .../eclipse-plugin/resources/hadoop_small.gif | Bin 1072 -> 0 bytes .../contrib/eclipse-plugin/resources/job.gif | Bin 59 -> 0 bytes .../resources/location-edit-16x16.png | Bin 930 -> 0 bytes .../resources/location-new-16x16.png | Bin 888 -> 0 bytes .../eclipse-plugin/resources/map16x16.gif | Bin 1032 -> 0 bytes .../eclipse-plugin/resources/mapper16.png | Bin 851 -> 0 bytes .../eclipse-plugin/resources/mapwiz.png | Bin 4378 -> 0 bytes .../eclipse-plugin/resources/new-folder.png | Bin 456 -> 0 bytes .../eclipse-plugin/resources/projwiz.png | Bin 6867 -> 0 bytes .../resources/reduce100x100.gif | Bin 7166 -> 0 bytes .../eclipse-plugin/resources/reduce16x16.gif | Bin 1077 -> 0 bytes .../resources/reducer-16x16.gif | Bin 988 -> 0 bytes .../eclipse-plugin/resources/reducer16.png | Bin 853 -> 0 bytes .../eclipse-plugin/resources/reducewiz.png | Bin 4470 -> 0 bytes .../eclipse-plugin/resources/refresh.png | Bin 986 -> 0 bytes .../resources/spite_overcloud.png | Bin 838 -> 0 bytes .../eclipse-plugin/resources/spitesmall.gif | Bin 162 -> 0 bytes .../eclipse-plugin/resources/spitesmall.png | Bin 395 -> 0 bytes .../eclipse-plugin/resources/upload.png | Bin 745 -> 0 bytes .../org/apache/hadoop/eclipse/Activator.java | 77 - .../hadoop/eclipse/ErrorMessageDialog.java | 45 - .../eclipse/HadoopPerspectiveFactory.java | 95 - .../apache/hadoop/eclipse/ImageLibrary.java | 252 - .../hadoop/eclipse/MapReduceNature.java | 146 - .../hadoop/eclipse/NewDriverWizard.java | 99 - .../hadoop/eclipse/NewDriverWizardPage.java | 263 - .../eclipse/NewMapReduceProjectWizard.java | 411 - .../hadoop/eclipse/NewMapperWizard.java | 181 - .../hadoop/eclipse/NewReducerWizard.java | 184 - .../apache/hadoop/eclipse/PropertyTester.java | 43 - .../hadoop/eclipse/actions/DFSActionImpl.java | 478 - .../eclipse/actions/EditLocationAction.java | 73 - .../eclipse/actions/NewLocationAction.java | 64 - .../actions/OpenNewMRClassWizardAction.java | 76 - .../actions/OpenNewMRProjectAction.java | 48 - .../hadoop/eclipse/dfs/ActionProvider.java | 193 - .../apache/hadoop/eclipse/dfs/DFSActions.java | 44 - .../apache/hadoop/eclipse/dfs/DFSContent.java | 32 - .../eclipse/dfs/DFSContentProvider.java | 244 - .../apache/hadoop/eclipse/dfs/DFSFile.java | 350 - .../apache/hadoop/eclipse/dfs/DFSFolder.java | 213 - .../hadoop/eclipse/dfs/DFSLocation.java | 108 - .../hadoop/eclipse/dfs/DFSLocationsRoot.java | 150 - .../apache/hadoop/eclipse/dfs/DFSMessage.java | 57 - .../apache/hadoop/eclipse/dfs/DFSPath.java | 160 - .../HadoopApplicationLaunchShortcut.java | 145 - .../launch/LocalMapReduceLaunchTabGroup.java | 182 - .../hadoop/eclipse/launch/MutexRule.java | 37 - .../launch/StartHadoopLaunchTabGroup.java | 47 - .../preferences/MapReducePreferencePage.java | 64 - .../preferences/PreferenceConstants.java | 34 - .../preferences/PreferenceInitializer.java | 33 - .../hadoop/eclipse/server/ConfProp.java | 147 - .../hadoop/eclipse/server/HadoopJob.java | 349 - .../hadoop/eclipse/server/HadoopPathPage.java | 124 - .../hadoop/eclipse/server/HadoopServer.java | 517 - .../hadoop/eclipse/server/IJobListener.java | 36 - .../hadoop/eclipse/server/JarModule.java | 146 - .../eclipse/servers/HadoopLocationWizard.java | 972 -- ...oopServerSelectionListContentProvider.java | 76 - .../servers/IHadoopServerListener.java | 28 - .../eclipse/servers/RunOnHadoopWizard.java | 383 - .../eclipse/servers/ServerRegistry.java | 203 - .../eclipse/view/servers/ServerView.java | 460 - .../src/contrib/gridmix/README | 22 - .../src/contrib/gridmix/build.xml | 32 - .../src/contrib/gridmix/ivy.xml | 153 - .../contrib/gridmix/ivy/libraries.properties | 20 - .../gridmix/TestDistCacheEmulation.java | 515 - .../mapred/gridmix/TestGridmixExitCodes.java | 176 - .../mapred/gridmix/TestGridmixStatistics.java | 217 - .../mapred/gridmix/TestGridmixSubmission.java | 606 - .../hadoop/mapred/gridmix/TestSleepJob.java | 280 - .../src/contrib/index/README | 43 - .../src/contrib/index/build.xml | 80 - .../index/conf/index-config.xml.template | 48 - .../src/contrib/index/ivy.xml | 69 - .../contrib/index/ivy/libraries.properties | 16 - .../src/contrib/index/sample/data.txt | 10 - .../src/contrib/index/sample/data2.txt | 10 - .../example/HashingDistributionPolicy.java | 56 - .../index/example/IdentityLocalAnalysis.java | 57 - .../index/example/LineDocInputFormat.java | 46 - .../index/example/LineDocLocalAnalysis.java | 80 - .../index/example/LineDocRecordReader.java | 231 - .../index/example/LineDocTextAndOp.java | 92 - .../example/RoundRobinDistributionPolicy.java | 58 - .../index/lucene/FileSystemDirectory.java | 349 - .../lucene/LuceneIndexFileNameFilter.java | 55 - .../contrib/index/lucene/LuceneUtil.java | 112 - .../index/lucene/MixedDeletionPolicy.java | 49 - .../contrib/index/lucene/MixedDirectory.java | 185 - .../index/lucene/RAMDirectoryUtil.java | 119 - .../contrib/index/lucene/ShardWriter.java | 233 - .../contrib/index/main/UpdateIndex.java | 276 - .../contrib/index/mapred/DocumentAndOp.java | 208 - .../contrib/index/mapred/DocumentID.java | 89 - .../index/mapred/IDistributionPolicy.java | 50 - .../contrib/index/mapred/IIndexUpdater.java | 46 - .../contrib/index/mapred/ILocalAnalysis.java | 32 - .../index/mapred/IndexUpdateCombiner.java | 111 - .../mapred/IndexUpdateConfiguration.java | 256 - .../index/mapred/IndexUpdateMapper.java | 199 - .../index/mapred/IndexUpdateOutputFormat.java | 65 - .../index/mapred/IndexUpdatePartitioner.java | 60 - .../index/mapred/IndexUpdateReducer.java | 143 - .../contrib/index/mapred/IndexUpdater.java | 150 - .../index/mapred/IntermediateForm.java | 252 - .../hadoop/contrib/index/mapred/Shard.java | 240 - .../index/lucene/TestMixedDirectory.java | 105 - .../index/mapred/TestDistributionPolicy.java | 234 - .../index/mapred/TestIndexUpdater.java | 258 - .../src/contrib/vaidya/build.xml | 68 - .../src/contrib/vaidya/ivy.xml | 60 - .../contrib/vaidya/ivy/libraries.properties | 17 - .../apache/hadoop/vaidya/DiagnosticTest.java | 370 - .../apache/hadoop/vaidya/JobDiagnoser.java | 85 - .../PostExPerformanceDiagnoser.java | 277 - .../tests/BalancedReducePartitioning.java | 119 - .../tests/MapSideDiskSpill.java | 121 - .../tests/MapsReExecutionImpact.java | 87 - .../tests/ReadingHDFSFilesAsSideEffect.java | 112 - .../tests/ReducesReExecutionImpact.java | 94 - .../tests/postex_diagnosis_tests.xml | 105 - .../vaidya/statistics/job/JobStatistics.java | 609 - .../job/JobStatisticsInterface.java | 125 - .../statistics/job/MapTaskStatistics.java | 25 - .../statistics/job/ReduceTaskStatistics.java | 25 - .../vaidya/statistics/job/TaskStatistics.java | 103 - .../apache/hadoop/vaidya/util/XMLUtils.java | 236 - .../java/org/apache/hadoop/vaidya/vaidya.sh | 108 - .../src/contrib/vertica/build.xml | 104 - .../src/contrib/vertica/ivy.xml | 81 - .../contrib/vertica/ivy/libraries.properties | 21 - .../hadoop/vertica/VerticaConfiguration.java | 460 - .../hadoop/vertica/VerticaInputFormat.java | 160 - .../hadoop/vertica/VerticaInputSplit.java | 189 - .../hadoop/vertica/VerticaOutputFormat.java | 244 - .../apache/hadoop/vertica/VerticaRecord.java | 660 - .../hadoop/vertica/VerticaRecordReader.java | 116 - .../hadoop/vertica/VerticaRecordWriter.java | 132 - .../hadoop/vertica/VerticaStreamingInput.java | 47 - .../vertica/VerticaStreamingOutput.java | 81 - .../vertica/VerticaStreamingRecordReader.java | 110 - .../vertica/VerticaStreamingRecordWriter.java | 101 - .../apache/hadoop/vertica/VerticaUtil.java | 247 - .../org/apache/hadoop/vertica/AllTests.java | 151 - .../apache/hadoop/vertica/TestExample.java | 168 - .../apache/hadoop/vertica/TestVertica.java | 383 - .../hadoop/vertica/VerticaTestCase.java | 31 - .../contrib/vertica/testdata/vertica_test.sql | 47 - .../src/docs/changes/ChangesFancyStyle.css | 170 - .../src/docs/changes/ChangesSimpleStyle.css | 49 - .../src/docs/changes/changes2html.pl | 286 - .../src/docs/forrest.properties | 112 - .../src/docs/releasenotes.html | 1 - .../src/docs/src/documentation/README.txt | 7 - .../classes/CatalogManager.properties | 40 - .../src/docs/src/documentation/conf/cli.xconf | 327 - .../documentation/content/xdocs/distcp.xml | 402 - .../documentation/content/xdocs/gridmix.xml | 801 -- .../content/xdocs/hadoop_archives.xml | 197 - .../src/documentation/content/xdocs/index.xml | 48 - .../content/xdocs/mapred_tutorial.xml | 2948 ---- .../src/documentation/content/xdocs/rumen.xml | 625 - .../src/documentation/content/xdocs/site.xml | 391 - .../documentation/content/xdocs/streaming.xml | 953 -- .../src/documentation/content/xdocs/tabs.xml | 37 - .../documentation/content/xdocs/vaidya.xml | 172 - .../resources/images/architecture.gif | Bin 15461 -> 0 bytes .../resources/images/core-logo.gif | Bin 6665 -> 0 bytes .../resources/images/favicon.ico | Bin 766 -> 0 bytes .../resources/images/hadoop-logo-big.jpg | Bin 127869 -> 0 bytes .../resources/images/hadoop-logo.jpg | Bin 9443 -> 0 bytes .../resources/images/hdfsarchitecture.gif | Bin 17653 -> 0 bytes .../resources/images/hdfsarchitecture.odg | Bin 41298 -> 0 bytes .../resources/images/hdfsarchitecture.png | Bin 40571 -> 0 bytes .../resources/images/hdfsdatanodes.gif | Bin 16060 -> 0 bytes .../resources/images/hdfsdatanodes.odg | Bin 37296 -> 0 bytes .../resources/images/hdfsdatanodes.png | Bin 30012 -> 0 bytes .../resources/images/mapreduce-logo.jpg | Bin 6672 -> 0 bytes .../src/docs/src/documentation/skinconf.xml | 366 - hadoop-mapreduce-project/src/docs/status.xml | 75 - .../src/examples/python/WordCount.py | 70 - .../src/examples/python/compile | 33 - .../python/pyAbacus/JyAbacusWCPlugIN.py | 36 - .../examples/python/pyAbacus/JythonAbacus.py | 82 - .../src/examples/python/pyAbacus/compile | 37 - .../python/pyAbacus/wordcountaggregator.spec | 32 - ....mapreduce.protocol.ClientProtocolProvider | 14 - .../src/java/mapred-default.xml | 1128 -- .../src/java/mapred-queues-default.xml | 29 - .../org/apache/hadoop/mapred/ACLsManager.java | 242 - .../mapred/AdminOperationsProtocol.java | 57 - .../java/org/apache/hadoop/mapred/Child.java | 287 - .../hadoop/mapred/CommitTaskAction.java | 55 - .../mapred/CompletedJobStatusStore.java | 372 - .../CompositeTaskTrackerInstrumentation.java | 85 - .../hadoop/mapred/DefaultTaskController.java | 229 - .../DisallowedTaskTrackerException.java | 37 - .../EagerTaskInitializationListener.java | 181 - .../hadoop/mapred/HeartbeatResponse.java | 112 - .../hadoop/mapred/InterTrackerProtocol.java | 159 - .../org/apache/hadoop/mapred/JSPUtil.java | 610 - .../apache/hadoop/mapred/JobChangeEvent.java | 37 - .../apache/hadoop/mapred/JobInProgress.java | 3777 ----- .../hadoop/mapred/JobInProgressListener.java | 47 - .../mapred/JobQueueJobInProgressListener.java | 145 - .../hadoop/mapred/JobQueueTaskScheduler.java | 312 - .../hadoop/mapred/JobStatusChangeEvent.java | 74 - .../org/apache/hadoop/mapred/JobTracker.java | 4865 ------- .../JobTrackerClientProtocolProvider.java | 71 - .../mapred/JobTrackerInstrumentation.java | 177 - .../hadoop/mapred/JobTrackerMetricsInst.java | 447 - .../hadoop/mapred/JobTrackerStatistics.java | 97 - .../org/apache/hadoop/mapred/JvmManager.java | 596 - .../apache/hadoop/mapred/KillJobAction.java | 59 - .../apache/hadoop/mapred/KillTaskAction.java | 57 - .../hadoop/mapred/LaunchTaskAction.java | 61 - .../mapred/LimitTasksPerJobTaskScheduler.java | 197 - .../mapred/MapReducePolicyProvider.java | 62 - .../apache/hadoop/mapred/MapTaskRunner.java | 48 - .../mapred/NodeHealthCheckerService.java | 376 - .../hadoop/mapred/ReduceTaskRunner.java | 57 - .../hadoop/mapred/ReinitTrackerAction.java | 40 - .../hadoop/mapred/ResourceEstimator.java | 110 - .../apache/hadoop/mapred/TaskController.java | 448 - .../hadoop/mapred/TaskGraphServlet.java | 248 - .../apache/hadoop/mapred/TaskInProgress.java | 1431 -- .../apache/hadoop/mapred/TaskLogServlet.java | 304 - .../mapred/TaskMemoryManagerThread.java | 548 - .../org/apache/hadoop/mapred/TaskRunner.java | 745 - .../apache/hadoop/mapred/TaskScheduler.java | 117 - .../org/apache/hadoop/mapred/TaskTracker.java | 4233 ------ .../hadoop/mapred/TaskTrackerAction.java | 117 - .../mapred/TaskTrackerInstrumentation.java | 71 - .../hadoop/mapred/TaskTrackerManager.java | 115 - .../hadoop/mapred/TaskTrackerMetricsInst.java | 86 - .../hadoop/mapred/TaskTrackerStatus.java | 684 - .../apache/hadoop/mapred/UserLogCleaner.java | 202 - .../hadoop/mapred/jobcontrol/package.html | 25 - .../apache/hadoop/mapred/join/package.html | 105 - .../hadoop/mapred/lib/aggregate/package.html | 186 - .../apache/hadoop/mapred/lib/db/package.html | 44 - .../org/apache/hadoop/mapred/lib/package.html | 25 - .../org/apache/hadoop/mapred/package.html | 230 - .../apache/hadoop/mapred/pipes/package.html | 127 - .../apache/hadoop/mapred/tools/MRAdmin.java | 357 - .../TaskDistributedCacheManager.java | 266 - .../TrackerDistributedCacheManager.java | 867 -- .../mapreduce/jobhistory/JobHistory.java | 589 - .../mapreduce/lib/aggregate/package.html | 199 - .../hadoop/mapreduce/lib/db/package.html | 44 - .../mapreduce/lib/jobcontrol/package.html | 25 - .../hadoop/mapreduce/lib/join/package.html | 105 - .../jobtracker/JobTrackerJspHelper.java | 110 - .../mapreduce/server/jobtracker/State.java | 33 - .../server/jobtracker/TaskTracker.java | 205 - .../server/tasktracker/Localizer.java | 273 - .../server/tasktracker/package-info.java | 23 - .../hadoop/mapreduce/tools/package-info.java | 23 - .../util/LinuxMemoryCalculatorPlugin.java | 37 - .../hadoop/util/MemoryCalculatorPlugin.java | 37 - .../org/apache/hadoop/util/ProcessTree.java | 34 - .../hadoop/util/ProcfsBasedProcessTree.java | 54 - .../org/apache/hadoop/util/package-info.java | 21 - .../src/java/overview.html | 292 - ....apache.hadoop.security.token.TokenRenewer | 1 - hadoop-mapreduce-project/src/test/all-tests | 1 - .../src/test/checkstyle-noframes-sorted.xsl | 195 - .../src/test/checkstyle.xml | 187 - .../src/test/commit-tests | 45 - hadoop-mapreduce-project/src/test/empty-file | 0 hadoop-mapreduce-project/src/test/fi-site.xml | 31 - .../src/test/findbugsExcludeFile.xml | 391 - .../src/test/hadoop-policy.xml | 114 - hadoop-mapreduce-project/src/test/krb5.conf | 28 - .../src/test/log4j.properties | 19 - .../src/test/mapred-site.xml | 55 - .../org/apache/hadoop/cli/TestMRCLI.java | 148 - .../hadoop/mapred/ControlledMapReduceJob.java | 578 - .../mapred/DummyMemoryCalculatorPlugin.java | 49 - .../DummyTaskTrackerInstrumentation.java | 69 - .../hadoop/mapred/FakeObjectUtilities.java | 622 - .../apache/hadoop/mapred/HadoopTestCase.java | 214 - .../apache/hadoop/mapred/MiniMRCluster.java | 778 -- .../hadoop/mapred/NotificationTestCase.java | 224 - .../hadoop/mapred/QueueManagerTestUtils.java | 358 - .../apache/hadoop/mapred/SortValidator.java | 597 - ...tionsProtocolWithServiceAuthorization.java | 56 - .../hadoop/mapred/TestClusterStatus.java | 270 - ...stCompositeTaskTrackerInstrumentation.java | 98 - .../mapred/TestCompressedEmptyMapOutputs.java | 123 - .../mapred/TestControlledMapReduceJob.java | 82 - .../apache/hadoop/mapred/TestDebugScript.java | 188 - .../apache/hadoop/mapred/TestEmptyJob.java | 255 - .../hadoop/mapred/TestJobDirCleanup.java | 113 - .../apache/hadoop/mapred/TestJobHistory.java | 1047 -- .../hadoop/mapred/TestJobHistoryParsing.java | 138 - .../hadoop/mapred/TestJobInProgress.java | 369 - .../mapred/TestJobInProgressListener.java | 464 - .../hadoop/mapred/TestJobKillAndFail.java | 195 - .../hadoop/mapred/TestJobQueueClient.java | 110 - .../mapred/TestJobQueueInformation.java | 144 - .../mapred/TestJobQueueTaskScheduler.java | 343 - .../apache/hadoop/mapred/TestJobRetire.java | 414 - .../mapred/TestJobStatusPersistency.java | 185 - .../mapred/TestJobTrackerInstrumentation.java | 394 - .../hadoop/mapred/TestJobTrackerStart.java | 62 - .../hadoop/mapred/TestJobTrackerXmlJsp.java | 55 - .../apache/hadoop/mapred/TestJvmManager.java | 327 - .../apache/hadoop/mapred/TestJvmReuse.java | 172 - .../hadoop/mapred/TestKillCompletedJob.java | 66 - .../hadoop/mapred/TestKillSubProcesses.java | 542 - .../TestLimitTasksPerJobTaskScheduler.java | 125 - .../apache/hadoop/mapred/TestLostTracker.java | 231 - .../hadoop/mapred/TestMRServerPorts.java | 210 - .../hadoop/mapred/TestMapredHeartbeat.java | 111 - .../hadoop/mapred/TestMapredSystemDir.java | 114 - .../hadoop/mapred/TestMiniMRDFSSort.java | 237 - .../hadoop/mapred/TestMiniMRLocalFS.java | 348 - .../hadoop/mapred/TestMiniMRWithDFS.java | 341 - .../hadoop/mapred/TestNodeHealthService.java | 160 - .../apache/hadoop/mapred/TestNodeRefresh.java | 490 - .../mapred/TestParallelInitialization.java | 253 - .../mapred/TestQueueAclsForCurrentUser.java | 172 - .../hadoop/mapred/TestQueueManager.java | 603 - .../mapred/TestQueueManagerRefresh.java | 341 - .../TestQueueManagerWithDeprecatedConf.java | 156 - .../TestQueueManagerWithJobTracker.java | 414 - .../mapred/TestRackAwareTaskPlacement.java | 183 - .../hadoop/mapred/TestRecoveryManager.java | 330 - .../hadoop/mapred/TestResourceEstimation.java | 108 - .../mapred/TestSetupAndCleanupFailure.java | 263 - .../mapred/TestSetupTaskScheduling.java | 334 - .../hadoop/mapred/TestSetupWorkDir.java | 235 - .../org/apache/hadoop/mapred/TestSeveral.java | 443 - .../mapred/TestShuffleExceptionCount.java | 245 - .../hadoop/mapred/TestShuffleJobToken.java | 154 - .../mapred/TestSpeculativeExecution.java | 333 - .../apache/hadoop/mapred/TestSubmitJob.java | 263 - .../mapred/TestTaskChildOptsParsing.java | 64 - .../apache/hadoop/mapred/TestTaskFail.java | 288 - .../hadoop/mapred/TestTaskLauncher.java | 123 - .../mapred/TestTaskLauncherThreaded.java | 165 - .../apache/hadoop/mapred/TestTaskLimits.java | 68 - .../hadoop/mapred/TestTaskLogServlet.java | 160 - .../hadoop/mapred/TestTaskOutputSize.java | 85 - .../mapred/TestTaskTrackerBlacklisting.java | 504 - .../TestTaskTrackerInstrumentation.java | 121 - .../mapred/TestTaskTrackerLocalization.java | 1075 -- .../mapred/TestTaskTrackerMemoryManager.java | 606 - .../mapred/TestTaskTrackerSlotManagement.java | 115 - .../TestTrackerBlacklistAcrossJobs.java | 98 - .../hadoop/mapred/TestTrackerReservation.java | 282 - .../hadoop/mapred/TestUserLogCleanup.java | 288 - .../apache/hadoop/mapred/UtilsForTests.java | 792 -- .../org/apache/hadoop/mapred/WordCount.java | 159 - .../org/apache/hadoop/mapred/concat.bz2 | Bin 208 -> 0 bytes .../mapred/org/apache/hadoop/mapred/concat.gz | Bin 148 -> 0 bytes .../jobcontrol/JobControlTestUtils.java | 154 - .../mapred/org/apache/hadoop/mapred/test.jar | Bin 518 -> 0 bytes .../mapred/org/apache/hadoop/mapred/test.tar | Bin 10240 -> 0 bytes .../org/apache/hadoop/mapred/test.tar.gz | Bin 189 -> 0 bytes .../mapred/org/apache/hadoop/mapred/test.tgz | Bin 180 -> 0 bytes .../mapred/org/apache/hadoop/mapred/test.txt | 1 - .../mapred/org/apache/hadoop/mapred/test.zip | Bin 213 -> 0 bytes .../mapred/testCompressThenConcat.txt.bz2 | Bin 3056 -> 0 bytes .../mapred/testCompressThenConcat.txt.gz | Bin 3413 -> 0 bytes .../mapred/testConcatThenCompress.txt.bz2 | Bin 2567 -> 0 bytes .../mapred/testConcatThenCompress.txt.gz | Bin 2734 -> 0 bytes .../org/apache/hadoop/mapred/testscript.txt | 2 - .../apache/hadoop/mapreduce/TestJobACLs.java | 463 - .../hadoop/mapreduce/TestMapReduceLocal.java | 196 - .../TestTrackerDistributedCacheManager.java | 1019 -- .../jobhistory/TestJobHistoryEvents.java | 163 - .../security/TestTokenCacheOldApi.java | 296 - .../apache/hadoop/record/TestRecordMR.java | 467 - .../hadoop/record/TestRecordWritable.java | 114 - .../TestMapredGroupMappingServiceRefresh.java | 271 - .../authorize/HadoopPolicyProvider.java | 39 - .../TestServiceLevelAuthorization.java | 190 - .../hadoop/tools/TestHarFileSystem.java | 427 - hadoop-mapreduce-project/src/test/smoke-tests | 13 - .../rumen/histogram-tests/gold-minimal.json | 15 - .../gold-one-value-many-repeats.json | 15 - .../histogram-tests/gold-only-one-value.json | 15 - .../histogram-tests/gold-three-values.json | 15 - .../rumen/histogram-tests/input-minimal.json | 17 - .../input-one-value-many-repeats.json | 76 - .../histogram-tests/input-only-one-value.json | 13 - .../histogram-tests/input-three-values.json | 15 - ...5616107882_job_201002080801_40864_conf.xml | 3 - ...0801_40864_job_name-DAILY%2F20100210%5D.gz | Bin 2509 -> 0 bytes ...5616107882_job_201002080801_50510_conf.xml | 3 - ...0801_50510_job_name-DAILY%2F20100208%5D.gz | Bin 2502 -> 0 bytes .../counters-test-trace.json.gz | Bin 3008 -> 0 bytes .../dispatch-sample-v20-jt-log.gz | Bin 29932 -> 0 bytes .../dispatch-topology-output.json.gz | Bin 2290 -> 0 bytes .../dispatch-trace-output.json.gz | Bin 28321 -> 0 bytes .../folder-input-trace.json.gz | Bin 110363 -> 0 bytes .../small-trace-test/goldFoldedTrace.json.gz | Bin 159847 -> 0 bytes .../job-tracker-logs-topology-output | 1768 --- .../job-tracker-logs-trace-output.gz | Bin 24960 -> 0 bytes .../small-trace-test/sample-conf.file.new.xml | 36 - .../small-trace-test/sample-conf.file.xml | 35 - .../sample-job-tracker-logs.gz | Bin 29907 -> 0 bytes .../truncated-job-tracker-log | 110 - .../truncated-topology-output | 343 - .../small-trace-test/truncated-trace-output | 1410 -- .../v20-resource-usage-log.gz | Bin 845 -> 0 bytes .../small-trace-test/v20-single-input-log.gz | Bin 3257 -> 0 bytes .../data/rumen/zombie/input-topology.json | 1693 --- .../tools/data/rumen/zombie/input-trace.json | 11364 ---------------- .../hadoop/mapred/TestJobTrackerPlugins.java | 90 - .../hadoop/mapred/TestLostTaskTracker.java | 113 - .../mapred/TestTaskTrackerDirectories.java | 140 - .../org/apache/hadoop/fs/package-info.java | 22 - .../src/webapps/job/analysejobhistory.jsp | 254 - .../src/webapps/job/index.html | 35 - .../webapps/job/job_authorization_error.jsp | 53 - .../webapps/job/jobblacklistedtrackers.jsp | 92 - .../src/webapps/job/jobconf.jsp | 95 - .../src/webapps/job/jobconf_history.jsp | 93 - .../src/webapps/job/jobdetails.jsp | 493 - .../src/webapps/job/jobdetailshistory.jsp | 291 - .../src/webapps/job/jobfailures.jsp | 202 - .../src/webapps/job/jobhistory.jsp | 318 - .../src/webapps/job/jobqueue_details.jsp | 107 - .../src/webapps/job/jobtable.jsp | 73 - .../src/webapps/job/jobtasks.jsp | 173 - .../src/webapps/job/jobtaskshistory.jsp | 95 - .../src/webapps/job/jobtracker.jsp | 185 - .../src/webapps/job/jobtracker.jspx | 91 - .../src/webapps/job/machines.jsp | 186 - .../src/webapps/job/queueinfo.jsp | 145 - .../src/webapps/job/queuetable.jsp | 78 - .../src/webapps/job/taskdetails.jsp | 374 - .../src/webapps/job/taskdetailshistory.jsp | 145 - .../src/webapps/job/taskstats.jsp | 135 - .../src/webapps/job/taskstatshistory.jsp | 122 - .../src/webapps/static/hadoop-logo.jpg | Bin 9443 -> 0 bytes .../src/webapps/static/hadoop.css | 134 - .../src/webapps/static/jobconf.xsl | 35 - .../src/webapps/static/jobtracker.js | 151 - .../src/webapps/task/index.html | 17 - .../src/webapps/task/tasktracker.jsp | 116 - 599 files changed, 2 insertions(+), 119174 deletions(-) delete mode 100644 hadoop-mapreduce-project/build-utils.xml delete mode 100644 hadoop-mapreduce-project/build.xml delete mode 100644 hadoop-mapreduce-project/ivy.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-examples-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-instrumented-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-instrumented-test-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-test-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/hadoop-mapred-tools-template.xml delete mode 100644 hadoop-mapreduce-project/ivy/ivysettings.xml delete mode 100644 hadoop-mapreduce-project/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/README delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/generateData.sh delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/gridmix-env delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/javasort/text-sort.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/javasort/text-sort.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/javasort/text-sort.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/maxent/maxent.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/monsterQuery/monster_query.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/monsterQuery/monster_query.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/monsterQuery/monster_query.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/pipesort/text-sort.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/pipesort/text-sort.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/pipesort/text-sort.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/streamsort/text-sort.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/streamsort/text-sort.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/streamsort/text-sort.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/allThroughHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/allToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/maxentHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/maxentToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/monsterQueriesHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/monsterQueriesToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/sleep_if_too_busy delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/textSortHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/textSortToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/webdataScanHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/webdataScanToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/webdataSortHod delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/submissionScripts/webdataSortToSameCluster delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatascan/webdata_scan.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatascan/webdata_scan.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatascan/webdata_scan.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatasort/webdata_sort.large delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatasort/webdata_sort.medium delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix/webdatasort/webdata_sort.small delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/README.gridmix2 delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/build.xml delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/generateGridmix2data.sh delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/gridmix-env-2 delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/gridmix_config.xml delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/rungridmix_2 delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/src/java/org/apache/hadoop/mapreduce/CombinerJobCreator.java delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/src/java/org/apache/hadoop/mapreduce/GenericMRLoadJobCreator.java delete mode 100644 hadoop-mapreduce-project/src/benchmarks/gridmix2/src/java/org/apache/hadoop/mapreduce/GridMixRunner.java delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/Makefile delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/archive.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/binarchive.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/binarchive.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/csvarchive.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/csvarchive.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/exception.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/exception.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/fieldTypeInfo.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/fieldTypeInfo.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/filestream.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/filestream.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/recordTypeInfo.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/recordTypeInfo.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/recordio.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/recordio.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/Makefile delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/test.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/test.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/test.jr delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/testFromJava.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/test/testFromJava.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/typeIDs.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/typeIDs.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/typeInfo.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/typeInfo.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/utils.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/utils.hh delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/xmlarchive.cc delete mode 100644 hadoop-mapreduce-project/src/c++/librecordio/xmlarchive.hh delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/README delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/client/BlockForensics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/block_forensics/src/java/org/apache/hadoop/blockforensics/BlockSearch.java delete mode 100644 hadoop-mapreduce-project/src/contrib/build-contrib.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/data_join/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/data_join/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/data_join/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/.classpath delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/.project delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/.settings/org.eclipse.jdt.core.prefs delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/.settings/org.eclipse.jdt.ui.prefs delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/.settings/org.eclipse.wst.validation.prefs delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/META-INF/MANIFEST.MF delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/build.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/plugin.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Conf.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Export.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Import.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/New.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Reload.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Tool.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Components/Tools.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/ConnectDFS.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/CreateProj.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant-24x24.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant-32x32.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant-64x64.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant-small-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant.jpg delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant100x100.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2-136x136.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2-24x24.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2-32x32.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2-64x64.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant2.jpg delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant3-122x122.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant3-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Elephant3-24x24.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/HelloWorld.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/MAP100x100.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/MAP16x15.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/RunProj.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/SetHadoopPath.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/Setup.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/download.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/drive100x100.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/drive16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/driver.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/driverwiz.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/elephantblue16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/files.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop-logo-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop-logo-24x24.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop-logo-85x85.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop-logo.jpg delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/hadoop_small.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/job.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/location-edit-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/location-new-16x16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/map16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/mapper16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/mapwiz.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/new-folder.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/projwiz.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/reduce100x100.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/reduce16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/reducer-16x16.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/reducer16.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/reducewiz.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/refresh.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/spite_overcloud.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/spitesmall.gif delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/spitesmall.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/resources/upload.png delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/Activator.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/ErrorMessageDialog.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/HadoopPerspectiveFactory.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/ImageLibrary.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/MapReduceNature.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/NewDriverWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/NewDriverWizardPage.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/NewMapReduceProjectWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/NewMapperWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/NewReducerWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/PropertyTester.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/actions/DFSActionImpl.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/actions/EditLocationAction.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/actions/NewLocationAction.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/actions/OpenNewMRClassWizardAction.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/actions/OpenNewMRProjectAction.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/ActionProvider.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSActions.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSContent.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSContentProvider.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSFile.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSFolder.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSLocation.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSLocationsRoot.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSMessage.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/dfs/DFSPath.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/launch/HadoopApplicationLaunchShortcut.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/launch/LocalMapReduceLaunchTabGroup.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/launch/MutexRule.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/launch/StartHadoopLaunchTabGroup.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/preferences/MapReducePreferencePage.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/preferences/PreferenceConstants.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/preferences/PreferenceInitializer.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/ConfProp.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/HadoopJob.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/HadoopPathPage.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/HadoopServer.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/IJobListener.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/server/JarModule.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/servers/HadoopLocationWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/servers/HadoopServerSelectionListContentProvider.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/servers/IHadoopServerListener.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/servers/RunOnHadoopWizard.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/servers/ServerRegistry.java delete mode 100644 hadoop-mapreduce-project/src/contrib/eclipse-plugin/src/java/org/apache/hadoop/eclipse/view/servers/ServerView.java delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/README delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixExitCodes.java delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixStatistics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java delete mode 100644 hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestSleepJob.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/README delete mode 100755 hadoop-mapreduce-project/src/contrib/index/build.xml delete mode 100755 hadoop-mapreduce-project/src/contrib/index/conf/index-config.xml.template delete mode 100644 hadoop-mapreduce-project/src/contrib/index/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/index/ivy/libraries.properties delete mode 100755 hadoop-mapreduce-project/src/contrib/index/sample/data.txt delete mode 100755 hadoop-mapreduce-project/src/contrib/index/sample/data2.txt delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/HashingDistributionPolicy.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/IdentityLocalAnalysis.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocInputFormat.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocLocalAnalysis.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocRecordReader.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocTextAndOp.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/RoundRobinDistributionPolicy.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/FileSystemDirectory.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/LuceneIndexFileNameFilter.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/LuceneUtil.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/MixedDeletionPolicy.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/MixedDirectory.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/RAMDirectoryUtil.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/ShardWriter.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/main/UpdateIndex.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/DocumentAndOp.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/DocumentID.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IDistributionPolicy.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IIndexUpdater.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/ILocalAnalysis.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateCombiner.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateConfiguration.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateMapper.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateOutputFormat.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdatePartitioner.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateReducer.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdater.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IntermediateForm.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/Shard.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/lucene/TestMixedDirectory.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/mapred/TestDistributionPolicy.java delete mode 100755 hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/mapred/TestIndexUpdater.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/DiagnosticTest.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/JobDiagnoser.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/PostExPerformanceDiagnoser.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/BalancedReducePartitioning.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/MapSideDiskSpill.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/MapsReExecutionImpact.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/ReadingHDFSFilesAsSideEffect.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/ReducesReExecutionImpact.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/postexdiagnosis/tests/postex_diagnosis_tests.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatisticsInterface.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/MapTaskStatistics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/ReduceTaskStatistics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/TaskStatistics.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/util/XMLUtils.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/vaidya.sh delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/build.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/ivy.xml delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/ivy/libraries.properties delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaConfiguration.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputFormat.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaInputSplit.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaOutputFormat.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecord.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordReader.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaRecordWriter.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingInput.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingOutput.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordReader.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaStreamingRecordWriter.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/java/org/apache/hadoop/vertica/VerticaUtil.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/test/org/apache/hadoop/vertica/AllTests.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/test/org/apache/hadoop/vertica/TestExample.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/test/org/apache/hadoop/vertica/TestVertica.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/src/test/org/apache/hadoop/vertica/VerticaTestCase.java delete mode 100644 hadoop-mapreduce-project/src/contrib/vertica/testdata/vertica_test.sql delete mode 100644 hadoop-mapreduce-project/src/docs/changes/ChangesFancyStyle.css delete mode 100644 hadoop-mapreduce-project/src/docs/changes/ChangesSimpleStyle.css delete mode 100755 hadoop-mapreduce-project/src/docs/changes/changes2html.pl delete mode 100644 hadoop-mapreduce-project/src/docs/forrest.properties delete mode 100644 hadoop-mapreduce-project/src/docs/releasenotes.html delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/README.txt delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/classes/CatalogManager.properties delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/conf/cli.xconf delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/distcp.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/gridmix.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/hadoop_archives.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/index.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/rumen.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/streaming.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/tabs.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/vaidya.xml delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/architecture.gif delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/core-logo.gif delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/favicon.ico delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hadoop-logo-big.jpg delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hadoop-logo.jpg delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsarchitecture.gif delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsarchitecture.odg delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsarchitecture.png delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsdatanodes.gif delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsdatanodes.odg delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/hdfsdatanodes.png delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/resources/images/mapreduce-logo.jpg delete mode 100644 hadoop-mapreduce-project/src/docs/src/documentation/skinconf.xml delete mode 100644 hadoop-mapreduce-project/src/docs/status.xml delete mode 100644 hadoop-mapreduce-project/src/examples/python/WordCount.py delete mode 100644 hadoop-mapreduce-project/src/examples/python/compile delete mode 100644 hadoop-mapreduce-project/src/examples/python/pyAbacus/JyAbacusWCPlugIN.py delete mode 100644 hadoop-mapreduce-project/src/examples/python/pyAbacus/JythonAbacus.py delete mode 100644 hadoop-mapreduce-project/src/examples/python/pyAbacus/compile delete mode 100644 hadoop-mapreduce-project/src/examples/python/pyAbacus/wordcountaggregator.spec delete mode 100644 hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider delete mode 100644 hadoop-mapreduce-project/src/java/mapred-default.xml delete mode 100644 hadoop-mapreduce-project/src/java/mapred-queues-default.xml delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ACLsManager.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/AdminOperationsProtocol.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/Child.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/CommitTaskAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/DefaultTaskController.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/DisallowedTaskTrackerException.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/EagerTaskInitializationListener.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JSPUtil.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobChangeEvent.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgressListener.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobQueueJobInProgressListener.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobQueueTaskScheduler.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobStatusChangeEvent.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTrackerClientProtocolProvider.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTrackerStatistics.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JvmManager.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/KillJobAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/KillTaskAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LaunchTaskAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/LimitTasksPerJobTaskScheduler.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/NodeHealthCheckerService.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReinitTrackerAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ResourceEstimator.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskController.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskGraphServlet.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskInProgress.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskLogServlet.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskScheduler.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTracker.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTrackerAction.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTrackerManager.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTrackerMetricsInst.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/UserLogCleaner.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/jobcontrol/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/join/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/lib/aggregate/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/lib/db/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/lib/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/pipes/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/lib/aggregate/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/lib/db/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/lib/jobcontrol/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/lib/join/package.html delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JobTrackerJspHelper.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/server/jobtracker/State.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/server/tasktracker/package-info.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/tools/package-info.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/util/LinuxMemoryCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/util/MemoryCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/util/ProcessTree.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/util/ProcfsBasedProcessTree.java delete mode 100644 hadoop-mapreduce-project/src/java/org/apache/hadoop/util/package-info.java delete mode 100644 hadoop-mapreduce-project/src/java/overview.html delete mode 100644 hadoop-mapreduce-project/src/test/META-INF/services/org.apache.hadoop.security.token.TokenRenewer delete mode 100644 hadoop-mapreduce-project/src/test/all-tests delete mode 100644 hadoop-mapreduce-project/src/test/checkstyle-noframes-sorted.xsl delete mode 100644 hadoop-mapreduce-project/src/test/checkstyle.xml delete mode 100644 hadoop-mapreduce-project/src/test/commit-tests delete mode 100644 hadoop-mapreduce-project/src/test/empty-file delete mode 100644 hadoop-mapreduce-project/src/test/fi-site.xml delete mode 100644 hadoop-mapreduce-project/src/test/findbugsExcludeFile.xml delete mode 100644 hadoop-mapreduce-project/src/test/hadoop-policy.xml delete mode 100644 hadoop-mapreduce-project/src/test/krb5.conf delete mode 100644 hadoop-mapreduce-project/src/test/log4j.properties delete mode 100644 hadoop-mapreduce-project/src/test/mapred-site.xml delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyMemoryCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/QueueManagerTestUtils.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAdminOperationsProtocolWithServiceAuthorization.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompressedEmptyMapOutputs.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestControlledMapReduceJob.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgressListener.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobKillAndFail.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueClient.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobStatusPersistency.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillCompletedJob.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillSubProcesses.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLimitTasksPerJobTaskScheduler.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLostTracker.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMRServerPorts.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRLocalFS.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeHealthService.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueAclsForCurrentUser.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRackAwareTaskPlacement.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupWorkDir.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleExceptionCount.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleJobToken.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskChildOptsParsing.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncher.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLimits.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskOutputSize.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerSlotManagement.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerBlacklistAcrossJobs.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/WordCount.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.bz2 delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.gz delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.jar delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.tar delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.tar.gz delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.tgz delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.txt delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.zip delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.bz2 delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.gz delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.bz2 delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.gz delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testscript.txt delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordMR.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordWritable.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/HadoopPolicyProvider.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/TestHarFileSystem.java delete mode 100644 hadoop-mapreduce-project/src/test/smoke-tests delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-minimal.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-one-value-many-repeats.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-only-one-value.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-three-values.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-minimal.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-one-value-many-repeats.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-only-one-value.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-three-values.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-sample-v20-jt-log.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-topology-output.json.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/folder-input-trace.json.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/goldFoldedTrace.json.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-topology-output delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-job-tracker-log delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-topology-output delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-trace-output delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/v20-resource-usage-log.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/v20-single-input-log.gz delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/zombie/input-topology.json delete mode 100644 hadoop-mapreduce-project/src/test/tools/data/rumen/zombie/input-trace.json delete mode 100644 hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestJobTrackerPlugins.java delete mode 100644 hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java delete mode 100644 hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java delete mode 100644 hadoop-mapreduce-project/src/tools/org/apache/hadoop/fs/package-info.java delete mode 100644 hadoop-mapreduce-project/src/webapps/job/analysejobhistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/index.html delete mode 100644 hadoop-mapreduce-project/src/webapps/job/job_authorization_error.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobblacklistedtrackers.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobconf.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobconf_history.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobdetails.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobdetailshistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobfailures.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobhistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobqueue_details.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobtable.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobtasks.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobtaskshistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobtracker.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/jobtracker.jspx delete mode 100644 hadoop-mapreduce-project/src/webapps/job/machines.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/queueinfo.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/queuetable.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/taskdetails.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/taskdetailshistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/taskstats.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/job/taskstatshistory.jsp delete mode 100644 hadoop-mapreduce-project/src/webapps/static/hadoop-logo.jpg delete mode 100644 hadoop-mapreduce-project/src/webapps/static/hadoop.css delete mode 100644 hadoop-mapreduce-project/src/webapps/static/jobconf.xsl delete mode 100644 hadoop-mapreduce-project/src/webapps/static/jobtracker.js delete mode 100644 hadoop-mapreduce-project/src/webapps/task/index.html delete mode 100644 hadoop-mapreduce-project/src/webapps/task/tasktracker.jsp diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 78fdfb99e2e..dfc9f9d73df 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -587,6 +587,8 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert Evans via tgraves) + MAPREDUCE-4266. remove Ant remnants from MR (tgraves via bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/build-utils.xml b/hadoop-mapreduce-project/build-utils.xml deleted file mode 100644 index 9f19d761d32..00000000000 --- a/hadoop-mapreduce-project/build-utils.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/hadoop-mapreduce-project/build.xml b/hadoop-mapreduce-project/build.xml deleted file mode 100644 index 286af16530b..00000000000 --- a/hadoop-mapreduce-project/build.xml +++ /dev/null @@ -1,1912 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Tests failed! - - - - - - - - - - - - - - - - Tests failed! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -

CheckStyle Audit

Designed for use with CheckStyle and Ant.
-
- - - -
- - - -
- - - - -
- - - - - - - - - -

Files

- - - - - - - - - - - - - - -
NameErrors
- - - - - -

File

- - - - - - - - - - - - - - -
Error DescriptionLine
- Back to top - - - - -

Summary

- - - - - - - - - - - - -
FilesErrors
- - - - - a - b - - - - - diff --git a/hadoop-mapreduce-project/src/test/checkstyle.xml b/hadoop-mapreduce-project/src/test/checkstyle.xml deleted file mode 100644 index 5fb0c472707..00000000000 --- a/hadoop-mapreduce-project/src/test/checkstyle.xml +++ /dev/null @@ -1,187 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hadoop-mapreduce-project/src/test/commit-tests b/hadoop-mapreduce-project/src/test/commit-tests deleted file mode 100644 index 277cb7d1779..00000000000 --- a/hadoop-mapreduce-project/src/test/commit-tests +++ /dev/null @@ -1,45 +0,0 @@ -**/TestCollect.java -**/TestCommandLineJobSubmission.java -**/TestComparators.java -**/TestCounters.java -**/TestFileInputFormat.java -**/TestFileInputFormatPathFilter.java -**/TestFileOutputCommitter.java -**/TestFileOutputFormat.java -**/TestGetSplitHosts.java -**/TestIFileStreams.java -**/TestIndexCache.java -**/TestJavaSerialization.java -**/TestJobHistoryParsing.java -**/TestJobHistoryVersion.java -**/TestJobInProgress.java -**/TestJobQueueTaskScheduler.java -**/TestKillCompletedJob.java -**/TestLostTracker.java -**/TestMapCollection.java -**/TestMapOutputType.java -**/TestMapRed.java -**/TestMapReduceLocal.java -**/TestMiniMRDFSCaching.java -**/TestQueueAclsForCurrentUser.java -**/TestRackAwareTaskPlacement.java -**/TestReduceFetchFromPartialMem.java -**/TestReduceTask.java -**/TestSequenceFileAsBinaryInputFormat.java -**/TestSequenceFileAsBinaryOutputFormat.java -**/TestSequenceFileInputFormat.java -**/TestSeveral.java -**/TestSpilledRecordsCounter.java -**/TestSpeculativeExecution.java -**/TestTaskLimits.java -**/TestTextInputFormat.java -**/TestTextOutputFormat.java -**/TestTrackerBlacklistAcrossJobs.java -**/TestTaskTrackerBlacklisting.java -**/TestTaskTrackerLocalization -**/TestTrackerDistributedCacheManager -**/TestQueueManager -**/TestContainerQueue -**/TestCapacityScheduler -**/TestRefreshOfQueues -**/TestQueueManagerRefresh \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/empty-file b/hadoop-mapreduce-project/src/test/empty-file deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hadoop-mapreduce-project/src/test/fi-site.xml b/hadoop-mapreduce-project/src/test/fi-site.xml deleted file mode 100644 index 42bae52195e..00000000000 --- a/hadoop-mapreduce-project/src/test/fi-site.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - fi.* - 0.00 - - Default probability level for all injected faults specified - as a floating number between 0 and 1.00 - - - diff --git a/hadoop-mapreduce-project/src/test/findbugsExcludeFile.xml b/hadoop-mapreduce-project/src/test/findbugsExcludeFile.xml deleted file mode 100644 index 28edfbfe363..00000000000 --- a/hadoop-mapreduce-project/src/test/findbugsExcludeFile.xml +++ /dev/null @@ -1,391 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hadoop-mapreduce-project/src/test/hadoop-policy.xml b/hadoop-mapreduce-project/src/test/hadoop-policy.xml deleted file mode 100644 index 09352749b73..00000000000 --- a/hadoop-mapreduce-project/src/test/hadoop-policy.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - - - - security.client.protocol.acl - * - ACL for ClientProtocol, which is used by user code - via the DistributedFileSystem. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.client.datanode.protocol.acl - * - ACL for ClientDatanodeProtocol, the client-to-datanode protocol - for block recovery. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.datanode.protocol.acl - * - ACL for DatanodeProtocol, which is used by datanodes to - communicate with the namenode. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.inter.datanode.protocol.acl - * - ACL for InterDatanodeProtocol, the inter-datanode protocol - for updating generation timestamp. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.namenode.protocol.acl - * - ACL for NamenodeProtocol, the protocol used by the secondary - namenode to communicate with the namenode. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.inter.tracker.protocol.acl - * - ACL for InterTrackerProtocol, used by the tasktrackers to - communicate with the jobtracker. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.job.submission.protocol.acl - * - ACL for JobSubmissionProtocol, used by job clients to - communciate with the jobtracker for job submission, querying job status etc. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.task.umbilical.protocol.acl - * - ACL for TaskUmbilicalProtocol, used by the map and reduce - tasks to communicate with the parent tasktracker. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.refresh.policy.protocol.acl - ${user.name} - ACL for RefreshAuthorizationPolicyProtocol, used by the - dfsadmin and mradmin commands to refresh the security policy in-effect. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - diff --git a/hadoop-mapreduce-project/src/test/krb5.conf b/hadoop-mapreduce-project/src/test/krb5.conf deleted file mode 100644 index 9f4b9ad7cd2..00000000000 --- a/hadoop-mapreduce-project/src/test/krb5.conf +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[libdefaults] - default_realm = APACHE.ORG - udp_preference_limit = 1 - extra_addresses = 127.0.0.1 -[realms] - APACHE.ORG = { - admin_server = localhost:88 - kdc = localhost:88 - } -[domain_realm] - localhost = APACHE.ORG diff --git a/hadoop-mapreduce-project/src/test/log4j.properties b/hadoop-mapreduce-project/src/test/log4j.properties deleted file mode 100644 index 531b68b5a9f..00000000000 --- a/hadoop-mapreduce-project/src/test/log4j.properties +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# log4j configuration used during build and unit tests - -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-mapreduce-project/src/test/mapred-site.xml b/hadoop-mapreduce-project/src/test/mapred-site.xml deleted file mode 100644 index 4874e61be54..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred-site.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - mapreduce.task.io.sort.mb - 10 - - - hadoop.security.authentication - simple - - - mapreduce.jobtracker.hosts.exclude.filename - hosts.exclude - - - - mapreduce.jobtracker.retirejobs - false - - - - mapred.child.java.opts - -Xmx200m -Djava.net.preferIPv4Stack=true - - - mapreduce.jobtracker.persist.jobstatus.active - false - - - mapreduce.task.local.output.class - org.apache.hadoop.mapred.MROutputFiles - - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java deleted file mode 100644 index 2d443821566..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cli; - -import org.apache.hadoop.cli.util.*; -import org.apache.hadoop.cli.util.CommandExecutor.Result; -import org.apache.hadoop.tools.HadoopArchives; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.tools.MRAdmin; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.authorize.HadoopPolicyProvider; -import org.apache.hadoop.security.authorize.PolicyProvider; -import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.xml.sax.SAXException; - -public class TestMRCLI extends TestHDFSCLI { - - protected MiniMRCluster mrCluster = null; - protected String jobtracker = null; - private JobConf mrConf; - - @Before - public void setUp() throws Exception { - super.setUp(); - conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, - HadoopPolicyProvider.class, PolicyProvider.class); - mrConf = new JobConf(conf); - mrCluster = new MiniMRCluster(1, dfsCluster.getFileSystem().getUri().toString(), 1, - null, null, mrConf); - jobtracker = mrCluster.createJobConf().get(JTConfig.JT_IPC_ADDRESS, "local"); - } - - @After - public void tearDown() throws Exception { - mrCluster.shutdown(); - super.tearDown(); - } - - @Override - protected TestConfigFileParser getConfigParser() { - return new TestConfigFileParserMR(); - } - - protected String getTestFile() { - return "testMRConf.xml"; - } - - @Override - protected String expandCommand(final String cmd) { - String expCmd = cmd; - expCmd = expCmd.replaceAll("JOBTRACKER", jobtracker); - expCmd = super.expandCommand(expCmd); - return expCmd; - } - - @Override - protected Result execute(CLICommand cmd) throws Exception { - if (cmd.getType() instanceof CLICommandMRAdmin) - return new TestMRCLI.MRCmdExecutor(jobtracker).executeCommand(cmd.getCmd()); - else if (cmd.getType() instanceof CLICommandArchive) - return new TestMRCLI.ArchiveCmdExecutor(namenode, mrConf).executeCommand(cmd.getCmd()); - else - return super.execute(cmd); - } - - public static class MRCmdExecutor extends CommandExecutor { - private String jobtracker = null; - public MRCmdExecutor(String jobtracker) { - this.jobtracker = jobtracker; - } - @Override - protected void execute(final String cmd) throws Exception{ - MRAdmin mradmin = new MRAdmin(); - String[] args = getCommandAsArgs(cmd, "JOBTRACKER", jobtracker); - ToolRunner.run(mradmin, args); - } - - } - - public static class ArchiveCmdExecutor extends CommandExecutor { - private String namenode = null; - private JobConf jobConf = null; - public ArchiveCmdExecutor(String namenode, JobConf jobConf) { - this.namenode = namenode; - this.jobConf = jobConf; - } - @Override - protected void execute(final String cmd) throws Exception { - HadoopArchives archive = new HadoopArchives(jobConf); - String[] args = getCommandAsArgs(cmd, "NAMENODE", namenode); - ToolRunner.run(archive, args); - } - } - - @Test - @Ignore - @Override - public void testAll () { - super.testAll(); - } - - class TestConfigFileParserMR extends CLITestHelper.TestConfigFileParser { - @Override - public void endElement(String uri, String localName, String qName) - throws SAXException { - if (qName.equals("mr-admin-command")) { - if (testCommands != null) { - testCommands.add(new CLITestCmdMR(charString, - new CLICommandMRAdmin())); - } else if (cleanupCommands != null) { - cleanupCommands.add(new CLITestCmdMR(charString, - new CLICommandMRAdmin())); - } - } else if (qName.equals("archive-command")) { - if (testCommands != null) { - testCommands.add(new CLITestCmdMR(charString, - new CLICommandArchive())); - } else if (cleanupCommands != null) { - cleanupCommands.add(new CLITestCmdMR(charString, - new CLICommandArchive())); - } - } else { - super.endElement(uri, localName, qName); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java deleted file mode 100644 index 13f831c3364..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/ControlledMapReduceJob.java +++ /dev/null @@ -1,578 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.mapred.lib.NullOutputFormat; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - -/** - * A Controlled Map/Reduce Job. The tasks are controlled by the presence of - * particularly named files in the directory signalFileDir on the file-system - * that the job is configured to work with. Tasks get scheduled by the - * scheduler, occupy the slots on the TaskTrackers and keep running till the - * user gives a signal via files whose names are of the form MAPS_[0-9]* and - * REDUCES_[0-9]*. For e.g., whenever the map tasks see that a file name MAPS_5 - * is created in the singalFileDir, all the maps whose TaskAttemptIDs are below - * 4 get finished. At any time, there should be only one MAPS_[0-9]* file and - * only one REDUCES_[0-9]* file in the singnalFileDir. In the beginning MAPS_0 - * and REDUCE_0 files are present, and further signals are given by renaming - * these files. - * - */ -class ControlledMapReduceJob extends Configured implements Tool, - Mapper, - Reducer, - Partitioner, - InputFormat { - - static final Log LOG = LogFactory.getLog(ControlledMapReduceJob.class); - - private FileSystem fs = null; - private int taskNumber; - - private static ArrayList signalFileDirCache = new ArrayList(); - - private Path signalFileDir; - { - Random random = new Random(); - signalFileDir = new Path("signalFileDir-" + random.nextLong()); - while (signalFileDirCache.contains(signalFileDir)) { - signalFileDir = new Path("signalFileDir-" + random.nextLong()); - } - signalFileDirCache.add(signalFileDir); - } - - private long mapsFinished = 0; - private long reducesFinished = 0; - - private RunningJob rJob = null; - - private int numMappers; - private int numReducers; - - private final String MAP_SIGFILE_PREFIX = "MAPS_"; - private final String REDUCE_SIGFILE_PREFIX = "REDUCES_"; - - private void initialize() - throws IOException { - fs = FileSystem.get(getConf()); - fs.mkdirs(signalFileDir); - writeFile(new Path(signalFileDir, MAP_SIGFILE_PREFIX + mapsFinished)); - writeFile(new Path(signalFileDir, REDUCE_SIGFILE_PREFIX + reducesFinished)); - } - - /** - * Finish N number of maps/reduces. - * - * @param isMap - * @param noOfTasksToFinish - * @throws IOException - */ - public void finishNTasks(boolean isMap, int noOfTasksToFinish) - throws IOException { - if (noOfTasksToFinish < 0) { - throw new IOException( - "Negative values for noOfTasksToFinish not acceptable"); - } - - if (noOfTasksToFinish == 0) { - return; - } - - LOG.info("Going to finish off " + noOfTasksToFinish); - String PREFIX = isMap ? MAP_SIGFILE_PREFIX : REDUCE_SIGFILE_PREFIX; - long tasksFinished = isMap ? mapsFinished : reducesFinished; - Path oldSignalFile = - new Path(signalFileDir, PREFIX + String.valueOf(tasksFinished)); - Path newSignalFile = - new Path(signalFileDir, PREFIX - + String.valueOf(tasksFinished + noOfTasksToFinish)); - fs.rename(oldSignalFile, newSignalFile); - if (isMap) { - mapsFinished += noOfTasksToFinish; - } else { - reducesFinished += noOfTasksToFinish; - } - LOG.info("Successfully sent signal to finish off " + noOfTasksToFinish); - } - - /** - * Finished all tasks of type determined by isMap - * - * @param isMap - * @throws IOException - */ - public void finishAllTasks(boolean isMap) - throws IOException { - finishNTasks(isMap, (isMap ? numMappers : numReducers)); - } - - /** - * Finish the job - * - * @throws IOException - */ - public void finishJob() - throws IOException { - finishAllTasks(true); - finishAllTasks(false); - } - - /** - * Wait till noOfTasksToBeRunning number of tasks of type specified by isMap - * started running. This currently uses a jip object and directly uses its api - * to determine the number of tasks running. - * - *

- * - * TODO: It should eventually use a JobID and then get the information from - * the JT to check the number of running tasks. - * - * @param jip - * @param isMap - * @param noOfTasksToBeRunning - */ - static void waitTillNTasksStartRunning(JobInProgress jip, boolean isMap, - int noOfTasksToBeRunning) - throws InterruptedException { - int numTasks = 0; - while (numTasks != noOfTasksToBeRunning) { - Thread.sleep(1000); - numTasks = isMap ? jip.runningMaps() : jip.runningReduces(); - LOG.info("Waiting till " + noOfTasksToBeRunning - + (isMap ? " map" : " reduce") + " tasks of the job " - + jip.getJobID() + " start running. " + numTasks - + " tasks already started running."); - } - } - - /** - * Make sure that the number of tasks of type specified by isMap running in - * the given job is the same as noOfTasksToBeRunning - * - *

- * - * TODO: It should eventually use a JobID and then get the information from - * the JT to check the number of running tasks. - * - * @param jip - * @param isMap - * @param noOfTasksToBeRunning - */ - static void assertNumTasksRunning(JobInProgress jip, boolean isMap, - int noOfTasksToBeRunning) - throws Exception { - if ((isMap ? jip.runningMaps() : jip.runningReduces()) != noOfTasksToBeRunning) { - throw new Exception("Number of tasks running is not " - + noOfTasksToBeRunning); - } - } - - /** - * Wait till noOfTasksToFinish number of tasks of type specified by isMap - * are finished. This currently uses a jip object and directly uses its api to - * determine the number of tasks finished. - * - *

- * - * TODO: It should eventually use a JobID and then get the information from - * the JT to check the number of finished tasks. - * - * @param jip - * @param isMap - * @param noOfTasksToFinish - * @throws InterruptedException - */ - static void waitTillNTotalTasksFinish(JobInProgress jip, boolean isMap, - int noOfTasksToFinish) - throws InterruptedException { - int noOfTasksAlreadyFinished = 0; - while (noOfTasksAlreadyFinished < noOfTasksToFinish) { - Thread.sleep(1000); - noOfTasksAlreadyFinished = - (isMap ? jip.finishedMaps() : jip.finishedReduces()); - LOG.info("Waiting till " + noOfTasksToFinish - + (isMap ? " map" : " reduce") + " tasks of the job " - + jip.getJobID() + " finish. " + noOfTasksAlreadyFinished - + " tasks already got finished."); - } - } - - /** - * Have all the tasks of type specified by isMap finished in this job? - * - * @param jip - * @param isMap - * @return true if finished, false otherwise - */ - static boolean haveAllTasksFinished(JobInProgress jip, boolean isMap) { - return ((isMap ? jip.runningMaps() : jip.runningReduces()) == 0); - } - - private void writeFile(Path name) - throws IOException { - Configuration conf = new Configuration(false); - SequenceFile.Writer writer = - SequenceFile.createWriter(fs, conf, name, BytesWritable.class, - BytesWritable.class, CompressionType.NONE); - writer.append(new BytesWritable(), new BytesWritable()); - writer.close(); - } - - @Override - public void configure(JobConf conf) { - try { - signalFileDir = new Path(conf.get("signal.dir.path")); - numReducers = conf.getNumReduceTasks(); - fs = FileSystem.get(conf); - String taskAttemptId = conf.get(JobContext.TASK_ATTEMPT_ID); - if (taskAttemptId != null) { - TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptId); - taskNumber = taskAttemptID.getTaskID().getId(); - } - } catch (IOException ioe) { - LOG.warn("Caught exception " + ioe); - } - } - - private FileStatus[] listSignalFiles(FileSystem fileSys, final boolean isMap) - throws IOException { - return fileSys.globStatus(new Path(signalFileDir.toString() + "/*"), - new PathFilter() { - @Override - public boolean accept(Path path) { - if (isMap && path.getName().startsWith(MAP_SIGFILE_PREFIX)) { - LOG.debug("Found signal file : " + path.getName()); - return true; - } else if (!isMap - && path.getName().startsWith(REDUCE_SIGFILE_PREFIX)) { - LOG.debug("Found signal file : " + path.getName()); - return true; - } - LOG.info("Didn't find any relevant signal files."); - return false; - } - }); - } - - @Override - public void map(NullWritable key, NullWritable value, - OutputCollector output, Reporter reporter) - throws IOException { - LOG.info(taskNumber + " has started."); - FileStatus[] files = listSignalFiles(fs, true); - String[] sigFileComps = files[0].getPath().getName().split("_"); - String signalType = sigFileComps[0]; - int noOfTasks = Integer.parseInt(sigFileComps[1]); - - while (!signalType.equals("MAPS") || taskNumber + 1 > noOfTasks) { - LOG.info("Signal type found : " + signalType - + " .Number of tasks to be finished by this signal : " + noOfTasks - + " . My id : " + taskNumber); - LOG.info(taskNumber + " is still alive."); - try { - reporter.progress(); - Thread.sleep(1000); - } catch (InterruptedException ie) { - LOG.info(taskNumber + " is still alive."); - break; - } - files = listSignalFiles(fs, true); - sigFileComps = files[0].getPath().getName().split("_"); - signalType = sigFileComps[0]; - noOfTasks = Integer.parseInt(sigFileComps[1]); - } - LOG.info("Signal type found : " + signalType - + " .Number of tasks to be finished by this signal : " + noOfTasks - + " . My id : " + taskNumber); - // output numReduce number of random values, so that - // each reducer will get one key each. - for (int i = 0; i < numReducers; i++) { - output.collect(new IntWritable(i), NullWritable.get()); - } - - LOG.info(taskNumber + " is finished."); - } - - @Override - public void reduce(IntWritable key, Iterator values, - OutputCollector output, Reporter reporter) - throws IOException { - LOG.info(taskNumber + " has started."); - FileStatus[] files = listSignalFiles(fs, false); - String[] sigFileComps = files[0].getPath().getName().split("_"); - String signalType = sigFileComps[0]; - int noOfTasks = Integer.parseInt(sigFileComps[1]); - - while (!signalType.equals("REDUCES") || taskNumber + 1 > noOfTasks) { - LOG.info("Signal type found : " + signalType - + " .Number of tasks to be finished by this signal : " + noOfTasks - + " . My id : " + taskNumber); - LOG.info(taskNumber + " is still alive."); - try { - reporter.progress(); - Thread.sleep(1000); - } catch (InterruptedException ie) { - LOG.info(taskNumber + " is still alive."); - break; - } - files = listSignalFiles(fs, false); - sigFileComps = files[0].getPath().getName().split("_"); - signalType = sigFileComps[0]; - noOfTasks = Integer.parseInt(sigFileComps[1]); - } - LOG.info("Signal type found : " + signalType - + " .Number of tasks to be finished by this signal : " + noOfTasks - + " . My id : " + taskNumber); - LOG.info(taskNumber + " is finished."); - } - - @Override - public void close() - throws IOException { - // nothing - } - - public JobID getJobId() { - if (rJob == null) { - return null; - } - return rJob.getID(); - } - - public int run(int numMapper, int numReducer) - throws IOException { - JobConf conf = - getControlledMapReduceJobConf(getConf(), numMapper, numReducer); - JobClient client = new JobClient(conf); - rJob = client.submitJob(conf); - while (!rJob.isComplete()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ie) { - break; - } - } - if (rJob.isSuccessful()) { - return 0; - } - return 1; - } - - private JobConf getControlledMapReduceJobConf(Configuration clusterConf, - int numMapper, int numReducer) - throws IOException { - setConf(clusterConf); - initialize(); - JobConf conf = new JobConf(getConf(), ControlledMapReduceJob.class); - conf.setJobName("ControlledJob"); - conf.set("signal.dir.path", signalFileDir.toString()); - conf.setNumMapTasks(numMapper); - conf.setNumReduceTasks(numReducer); - conf.setMapperClass(ControlledMapReduceJob.class); - conf.setMapOutputKeyClass(IntWritable.class); - conf.setMapOutputValueClass(NullWritable.class); - conf.setReducerClass(ControlledMapReduceJob.class); - conf.setOutputKeyClass(NullWritable.class); - conf.setOutputValueClass(NullWritable.class); - conf.setInputFormat(ControlledMapReduceJob.class); - FileInputFormat.addInputPath(conf, new Path("ignored")); - conf.setOutputFormat(NullOutputFormat.class); - conf.setMapSpeculativeExecution(false); - conf.setReduceSpeculativeExecution(false); - - // Set the following for reduce tasks to be able to be started running - // immediately along with maps. - conf.set(JobContext.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, String.valueOf(0)); - - return conf; - } - - @Override - public int run(String[] args) - throws Exception { - numMappers = Integer.parseInt(args[0]); - numReducers = Integer.parseInt(args[1]); - return run(numMappers, numReducers); - } - - @Override - public int getPartition(IntWritable k, NullWritable v, int numPartitions) { - return k.get() % numPartitions; - } - - @Override - public RecordReader getRecordReader( - InputSplit split, JobConf job, Reporter reporter) { - LOG.debug("Inside RecordReader.getRecordReader"); - return new RecordReader() { - private int pos = 0; - - public void close() { - // nothing - } - - public NullWritable createKey() { - return NullWritable.get(); - } - - public NullWritable createValue() { - return NullWritable.get(); - } - - public long getPos() { - return pos; - } - - public float getProgress() { - return pos * 100; - } - - public boolean next(NullWritable key, NullWritable value) { - if (pos++ == 0) { - LOG.debug("Returning the next record"); - return true; - } - LOG.debug("No more records. Returning none."); - return false; - } - }; - } - - @Override - public InputSplit[] getSplits(JobConf job, int numSplits) { - LOG.debug("Inside InputSplit.getSplits"); - InputSplit[] ret = new InputSplit[numSplits]; - for (int i = 0; i < numSplits; ++i) { - ret[i] = new EmptySplit(); - } - return ret; - } - - public static class EmptySplit implements InputSplit { - public void write(DataOutput out) - throws IOException { - } - - public void readFields(DataInput in) - throws IOException { - } - - public long getLength() { - return 0L; - } - - public String[] getLocations() { - return new String[0]; - } - } - - static class ControlledMapReduceJobRunner extends Thread { - private JobConf conf; - private ControlledMapReduceJob job; - private JobID jobID; - - private int numMappers; - private int numReducers; - - public ControlledMapReduceJobRunner() { - this(new JobConf(), 5, 5); - } - - public ControlledMapReduceJobRunner(JobConf cnf, int numMap, int numRed) { - this.conf = cnf; - this.numMappers = numMap; - this.numReducers = numRed; - } - - public ControlledMapReduceJob getJob() { - while (job == null) { - try { - Thread.sleep(1000); - } catch (InterruptedException ie) { - LOG.info(ControlledMapReduceJobRunner.class.getName() - + " is interrupted."); - break; - } - } - return job; - } - - public JobID getJobID() - throws IOException { - ControlledMapReduceJob job = getJob(); - JobID id = job.getJobId(); - while (id == null) { - id = job.getJobId(); - try { - Thread.sleep(1000); - } catch (InterruptedException ie) { - LOG.info(ControlledMapReduceJobRunner.class.getName() - + " is interrupted."); - break; - } - } - return id; - } - - @Override - public void run() { - if (job != null) { - LOG.warn("Job is already running."); - return; - } - try { - job = new ControlledMapReduceJob(); - int ret = - ToolRunner.run(this.conf, job, new String[] { - String.valueOf(numMappers), String.valueOf(numReducers) }); - LOG.info("Return value for the job : " + ret); - } catch (Exception e) { - LOG.warn("Caught exception : " + StringUtils.stringifyException(e)); - } - } - - static ControlledMapReduceJobRunner getControlledMapReduceJobRunner( - JobConf conf, int numMappers, int numReducers) { - return new ControlledMapReduceJobRunner(conf, numMappers, numReducers); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyMemoryCalculatorPlugin.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyMemoryCalculatorPlugin.java deleted file mode 100644 index 6c16e564faa..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyMemoryCalculatorPlugin.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import org.apache.hadoop.util.MemoryCalculatorPlugin; - -/** - * Plugin class to test virtual and physical memories reported by TT. Use - * configuration items {@link #MAXVMEM_TESTING_PROPERTY} and - * {@link #MAXPMEM_TESTING_PROPERTY} to tell TT the total vmem and the total - * pmem. - */ -public class DummyMemoryCalculatorPlugin extends MemoryCalculatorPlugin { - - /** max vmem on the TT */ - public static final String MAXVMEM_TESTING_PROPERTY = - "mapred.tasktracker.maxvmem.testing"; - /** max pmem on the TT */ - public static final String MAXPMEM_TESTING_PROPERTY = - "mapred.tasktracker.maxpmem.testing"; - - /** {@inheritDoc} */ - @Override - public long getVirtualMemorySize() { - return getConf().getLong(MAXVMEM_TESTING_PROPERTY, -1); - } - - /** {@inheritDoc} */ - @Override - public long getPhysicalMemorySize() { - return getConf().getLong(MAXPMEM_TESTING_PROPERTY, -1); - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java deleted file mode 100644 index 82d4b83a34c..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; - -/** - * Mock instrumentation class used in TaskTrackerInstrumentation tests. - * This class just records whether each instrumentation method was called. - */ -public class DummyTaskTrackerInstrumentation - extends TaskTrackerInstrumentation -{ - boolean completeTaskCalled = false; - boolean timedoutTaskCalled = false; - boolean taskFailedPingCalled = false; - boolean reportTaskLaunchCalled = false; - boolean reportTaskEndCalled = false; - boolean statusUpdateCalled = false; - - public DummyTaskTrackerInstrumentation(TaskTracker tt) { - super(tt); - } - - @Override - public void completeTask(TaskAttemptID t) { - completeTaskCalled = true; - } - - @Override - public void timedoutTask(TaskAttemptID t) { - timedoutTaskCalled = true; - } - - @Override - public void taskFailedPing(TaskAttemptID t) { - taskFailedPingCalled = true; - } - - @Override - public void reportTaskLaunch(TaskAttemptID t, File stdout, File stderr) { - reportTaskLaunchCalled = true; - } - - @Override - public void reportTaskEnd(TaskAttemptID t) { - reportTaskEndCalled = true; - } - - @Override - public void statusUpdate(Task t, TaskStatus s) { - statusUpdateCalled = true; - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java deleted file mode 100644 index 545482c32f8..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java +++ /dev/null @@ -1,622 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; - -import javax.security.auth.login.LoginException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.TaskStatus.Phase; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent; -import org.apache.hadoop.mapreduce.jobhistory.JobHistory; -import org.apache.hadoop.mapreduce.split.JobSplit; -import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; - -/** - * Utilities used in unit test. - * - */ -public class FakeObjectUtilities { - - static final Log LOG = LogFactory.getLog(FakeObjectUtilities.class); - - private static String jtIdentifier = "test"; - private static int jobCounter; - - /** - * A Fake JobTracker class for use in Unit Tests - */ - static class FakeJobTracker extends JobTracker { - - int totalSlots; - private String[] trackers; - - FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException, - InterruptedException, LoginException { - super(conf, clock); - this.trackers = tts; - //initialize max{Map/Reduce} task capacities to twice the clustersize - totalSlots = trackers.length * 4; - } - @Override - public ClusterStatus getClusterStatus(boolean detailed) { - return new ClusterStatus( - taskTrackers().size() - getBlacklistedTrackerCount(), - getBlacklistedTrackerCount(), 0, 0, 0, totalSlots/2, totalSlots/2, - JobTrackerStatus.RUNNING, 0); - } - - public void setNumSlots(int totalSlots) { - this.totalSlots = totalSlots; - } - } - - static class FakeJobInProgress extends JobInProgress { - @SuppressWarnings("deprecation") - FakeJobInProgress(JobConf jobConf, JobTracker tracker) throws IOException { - super(new JobID(jtIdentifier, ++jobCounter), jobConf, tracker); - Path jobFile = new Path("Dummy"); - this.profile = new JobProfile(jobConf.getUser(), getJobID(), - jobFile.toString(), null, jobConf.getJobName(), - jobConf.getQueueName()); - this.jobHistory = new FakeJobHistory(); - } - - @Override - public synchronized void initTasks() throws IOException { - - TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(jobId); - numMapTasks = taskSplitMetaInfo.length; - createMapTasks(null, taskSplitMetaInfo); - nonRunningMapCache = createCache(taskSplitMetaInfo, maxLevel); - createReduceTasks(null); - tasksInited.set(true); - this.status.setRunState(JobStatus.RUNNING); - } - - @Override - TaskSplitMetaInfo [] createSplits(org.apache.hadoop.mapreduce.JobID jobId){ - TaskSplitMetaInfo[] splits = - new TaskSplitMetaInfo[numMapTasks]; - for (int i = 0; i < numMapTasks; i++) { - splits[i] = JobSplit.EMPTY_TASK_SPLIT; - } - return splits; - } - - @Override - protected void createMapTasks(String ignored, TaskSplitMetaInfo[] splits) { - maps = new TaskInProgress[numMapTasks]; - for (int i = 0; i < numMapTasks; i++) { - maps[i] = new TaskInProgress(getJobID(), "test", - splits[i], jobtracker, getJobConf(), this, i, 1); - } - } - - @Override - protected void createReduceTasks(String ignored) { - reduces = new TaskInProgress[numReduceTasks]; - for (int i = 0; i < numReduceTasks; i++) { - reduces[i] = new TaskInProgress(getJobID(), "test", - numMapTasks, i, - jobtracker, getJobConf(), this, 1); - nonRunningReduces.add(reduces[i]); - } - } - - private TaskAttemptID findTask(String trackerName, String trackerHost, - Collection nonRunningTasks, - Collection runningTasks, TaskType taskType) - throws IOException { - TaskInProgress tip = null; - Iterator iter = nonRunningTasks.iterator(); - //look for a non-running task first - while (iter.hasNext()) { - TaskInProgress t = iter.next(); - if (t.isRunnable() && !t.isRunning()) { - runningTasks.add(t); - iter.remove(); - tip = t; - break; - } - } - if (tip == null) { - if (getJobConf().getSpeculativeExecution()) { - tip = findSpeculativeTask(runningTasks, trackerName, trackerHost, - taskType); - } - } - if (tip != null) { - TaskAttemptID tId = tip.getTaskToRun(trackerName).getTaskID(); - if (tip.isMapTask()) { - scheduleMap(tip); - } else { - scheduleReduce(tip); - } - //Set it to RUNNING - makeRunning(tId, tip, trackerName); - return tId; - } - return null; - } - - public TaskAttemptID findMapTask(String trackerName) - throws IOException { - return findTask(trackerName, - JobInProgress.convertTrackerNameToHostName(trackerName), - nonLocalMaps, nonLocalRunningMaps, TaskType.MAP); - } - - public TaskAttemptID findReduceTask(String trackerName) - throws IOException { - return findTask(trackerName, - JobInProgress.convertTrackerNameToHostName(trackerName), - nonRunningReduces, runningReduces, TaskType.REDUCE); - } - - public void finishTask(TaskAttemptID taskId) { - TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId); - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - 1.0f, 1, TaskStatus.State.SUCCEEDED, "", "", - tip.machineWhereTaskRan(taskId), - tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - private void makeRunning(TaskAttemptID taskId, TaskInProgress tip, - String taskTracker) { - addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker, - JobInProgress.convertTrackerNameToHostName(taskTracker)), true); - - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - 0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker, - tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - public void progressMade(TaskAttemptID taskId, float progress) { - TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId); - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - progress, 1, TaskStatus.State.RUNNING, "", "", - tip.machineWhereTaskRan(taskId), - tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - public void failTask(TaskAttemptID taskId) { - TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId); - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - 1.0f, 1, TaskStatus.State.FAILED, "", "", tip - .machineWhereTaskRan(taskId), tip.isMapTask() ? Phase.MAP - : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - public void killTask(TaskAttemptID taskId) { - TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId); - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - 1.0f, 1, TaskStatus.State.KILLED, "", "", tip - .machineWhereTaskRan(taskId), tip.isMapTask() ? Phase.MAP - : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - public void cleanUpMetrics() { - } - - public void setClusterSize(int clusterSize) { - super.setClusterSize(clusterSize); - } - } - - static short sendHeartBeat(JobTracker jt, TaskTrackerStatus status, - boolean initialContact, boolean acceptNewTasks, - String tracker, short responseId) - throws IOException { - if (status == null) { - status = new TaskTrackerStatus(tracker, - JobInProgress.convertTrackerNameToHostName(tracker)); - - } - jt.heartbeat(status, false, initialContact, acceptNewTasks, responseId); - return ++responseId ; - } - - static void establishFirstContact(JobTracker jt, String tracker) - throws IOException { - sendHeartBeat(jt, null, true, false, tracker, (short) 0); - } - - static class FakeTaskInProgress extends TaskInProgress { - - public FakeTaskInProgress(JobID jobId, String jobFile, int numMaps, - int partition, JobTracker jobTracker, JobConf conf, JobInProgress job, - int numSlotsRequired) { - super(jobId, jobFile, numMaps, partition, jobTracker, conf, job, - numSlotsRequired); - } - - public FakeTaskInProgress(JobID jobId, String jobFile, TaskSplitMetaInfo emptySplit, - JobTracker jobTracker, JobConf jobConf, - JobInProgress job, int partition, int numSlotsRequired) { - super(jobId, jobFile, emptySplit, jobTracker, jobConf, job, - partition, numSlotsRequired); - } - - @Override - synchronized boolean updateStatus(TaskStatus status) { - TaskAttemptID taskid = status.getTaskID(); - taskStatuses.put(taskid, status); - return false; - } - } - - static class FakeJobHistory extends JobHistory { - @Override - public void init(JobTracker jt, - JobConf conf, - String hostname, - long jobTrackerStartTime) throws IOException { } - - @Override - public void initDone(JobConf conf, FileSystem fs) throws IOException { } - - @Override - public void markCompleted(org.apache.hadoop.mapreduce.JobID id) - throws IOException { } - - @Override - public void shutDown() { } - - @Override - public void - logEvent(HistoryEvent event, org.apache.hadoop.mapreduce.JobID id) { } - - @Override - public void closeWriter(org.apache.hadoop.mapreduce.JobID id) { } - } - - static class FakeJobTrackerMetricsInst extends JobTrackerInstrumentation { - public FakeJobTrackerMetricsInst(JobTracker tracker, JobConf conf) { - super(tracker, conf); - } - - int numMapTasksLaunched = 0; - int numMapTasksCompleted = 0; - int numMapTasksFailed = 0; - int numReduceTasksLaunched = 0; - int numReduceTasksCompleted = 0; - int numReduceTasksFailed = 0; - int numJobsSubmitted = 0; - int numJobsCompleted = 0; - int numWaitingMaps = 0; - int numWaitingReduces = 0; - int numSpeculativeMaps = 0; - int numSpeculativeReduces = 0; - int numDataLocalMaps = 0; - int numRackLocalMaps = 0; - - //Cluster status fields. - volatile int numMapSlots = 0; - volatile int numReduceSlots = 0; - int numBlackListedMapSlots = 0; - int numBlackListedReduceSlots = 0; - - int numReservedMapSlots = 0; - int numReservedReduceSlots = 0; - int numOccupiedMapSlots = 0; - int numOccupiedReduceSlots = 0; - - int numJobsFailed = 0; - int numJobsKilled = 0; - - int numJobsPreparing = 0; - int numJobsRunning = 0; - - int numRunningMaps = 0; - int numRunningReduces = 0; - - int numMapTasksKilled = 0; - int numReduceTasksKilled = 0; - - int numTrackers = 0; - int numTrackersBlackListed = 0; - - int numTrackersDecommissioned = 0; - - long numHeartbeats = 0; - - @Override - public synchronized void launchMap(TaskAttemptID taskAttemptID) { - ++numMapTasksLaunched; - decWaitingMaps(taskAttemptID.getJobID(), 1); - } - - @Override - public synchronized void completeMap(TaskAttemptID taskAttemptID) { - ++numMapTasksCompleted; - } - - @Override - public synchronized void failedMap(TaskAttemptID taskAttemptID) { - ++numMapTasksFailed; - addWaitingMaps(taskAttemptID.getJobID(), 1); - } - - @Override - public synchronized void launchReduce(TaskAttemptID taskAttemptID) { - ++numReduceTasksLaunched; - decWaitingReduces(taskAttemptID.getJobID(), 1); - } - - @Override - public synchronized void completeReduce(TaskAttemptID taskAttemptID) { - ++numReduceTasksCompleted; - } - - @Override - public synchronized void failedReduce(TaskAttemptID taskAttemptID) { - ++numReduceTasksFailed; - addWaitingReduces(taskAttemptID.getJobID(), 1); - } - - @Override - public synchronized void submitJob(JobConf conf, JobID id) { - ++numJobsSubmitted; - } - - @Override - public synchronized void completeJob(JobConf conf, JobID id) { - ++numJobsCompleted; - } - - @Override - public synchronized void addWaitingMaps(JobID id, int task) { - numWaitingMaps += task; - } - - @Override - public synchronized void decWaitingMaps(JobID id, int task) { - numWaitingMaps -= task; - } - - @Override - public synchronized void addWaitingReduces(JobID id, int task) { - numWaitingReduces += task; - } - - @Override - public synchronized void decWaitingReduces(JobID id, int task){ - numWaitingReduces -= task; - } - - @Override - public void setMapSlots(int slots) { - numMapSlots = slots; - } - - @Override - public void setReduceSlots(int slots) { - numReduceSlots = slots; - } - - @Override - public synchronized void addBlackListedMapSlots(int slots){ - numBlackListedMapSlots += slots; - } - - @Override - public synchronized void decBlackListedMapSlots(int slots){ - numBlackListedMapSlots -= slots; - } - - @Override - public synchronized void addBlackListedReduceSlots(int slots){ - numBlackListedReduceSlots += slots; - } - - @Override - public synchronized void decBlackListedReduceSlots(int slots){ - numBlackListedReduceSlots -= slots; - } - - @Override - public synchronized void addReservedMapSlots(int slots) - { - numReservedMapSlots += slots; - } - - @Override - public synchronized void decReservedMapSlots(int slots) - { - numReservedMapSlots -= slots; - } - - @Override - public synchronized void addReservedReduceSlots(int slots) - { - numReservedReduceSlots += slots; - } - - @Override - public synchronized void decReservedReduceSlots(int slots) - { - numReservedReduceSlots -= slots; - } - - @Override - public synchronized void addOccupiedMapSlots(int slots) - { - numOccupiedMapSlots += slots; - } - - @Override - public synchronized void decOccupiedMapSlots(int slots) - { - numOccupiedMapSlots -= slots; - } - - @Override - public synchronized void addOccupiedReduceSlots(int slots) - { - numOccupiedReduceSlots += slots; - } - - @Override - public synchronized void decOccupiedReduceSlots(int slots) - { - numOccupiedReduceSlots -= slots; - } - - @Override - public synchronized void failedJob(JobConf conf, JobID id) - { - numJobsFailed++; - } - - @Override - public synchronized void killedJob(JobConf conf, JobID id) - { - numJobsKilled++; - } - - @Override - public synchronized void addPrepJob(JobConf conf, JobID id) - { - numJobsPreparing++; - } - - @Override - public synchronized void decPrepJob(JobConf conf, JobID id) - { - numJobsPreparing--; - } - - @Override - public synchronized void addRunningJob(JobConf conf, JobID id) - { - numJobsRunning++; - } - - @Override - public synchronized void decRunningJob(JobConf conf, JobID id) - { - numJobsRunning--; - } - - @Override - public synchronized void addRunningMaps(int task) - { - numRunningMaps += task; - } - - @Override - public synchronized void decRunningMaps(int task) - { - numRunningMaps -= task; - } - - @Override - public synchronized void addRunningReduces(int task) - { - numRunningReduces += task; - } - - @Override - public synchronized void decRunningReduces(int task) - { - numRunningReduces -= task; - } - - @Override - public synchronized void killedMap(TaskAttemptID taskAttemptID) - { - numMapTasksKilled++; - } - - @Override - public synchronized void killedReduce(TaskAttemptID taskAttemptID) - { - numReduceTasksKilled++; - } - - @Override - public synchronized void addTrackers(int trackers) - { - numTrackers += trackers; - } - - @Override - public synchronized void decTrackers(int trackers) - { - numTrackers -= trackers; - } - - @Override - public synchronized void addBlackListedTrackers(int trackers) - { - numTrackersBlackListed += trackers; - } - - @Override - public synchronized void decBlackListedTrackers(int trackers) - { - numTrackersBlackListed -= trackers; - } - - @Override - public synchronized void setDecommissionedTrackers(int trackers) - { - numTrackersDecommissioned = trackers; - } - - @Override - public synchronized void heartbeat() { - ++numHeartbeats; - } - - @Override - public synchronized void speculateReduce(TaskAttemptID taskAttemptID) { - ++numSpeculativeReduces; - } - - @Override - public synchronized void speculateMap(TaskAttemptID taskAttemptID) { - ++numSpeculativeMaps; - } - - @Override - public synchronized void launchDataLocalMap(TaskAttemptID taskAttemptID) { - ++numDataLocalMaps; - } - - @Override - public synchronized void launchRackLocalMap(TaskAttemptID taskAttemptID) { - ++numRackLocalMaps; - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java deleted file mode 100644 index c102e8f8626..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import junit.framework.TestCase; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.MRConfig; - -import java.io.File; -import java.io.IOException; - -/** - * Abstract Test case class to run MR in local or cluster mode and in local FS - * or DFS. - * - * The Hadoop instance is started and stopped on each test method. - * - * If using DFS the filesystem is reformated at each start (test method). - * - * Job Configurations should be created using a configuration returned by the - * 'createJobConf()' method. - */ -public abstract class HadoopTestCase extends TestCase { - public static final int LOCAL_MR = 1; - public static final int CLUSTER_MR = 2; - public static final int LOCAL_FS = 4; - public static final int DFS_FS = 8; - - private boolean localMR; - private boolean localFS; - - private int taskTrackers; - private int dataNodes; - - /** - * Creates a testcase for local or cluster MR using DFS. - * - * The DFS will be formatted regardless if there was one or not before in the - * given location. - * - * @param mrMode indicates if the MR should be local (LOCAL_MR) or cluster - * (CLUSTER_MR) - * @param fsMode indicates if the FS should be local (LOCAL_FS) or DFS (DFS_FS) - * - * local FS when using relative PATHs) - * - * @param taskTrackers number of task trackers to start when using cluster - * - * @param dataNodes number of data nodes to start when using DFS - * - * @throws IOException thrown if the base directory cannot be set. - */ - public HadoopTestCase(int mrMode, int fsMode, int taskTrackers, int dataNodes) - throws IOException { - if (mrMode != LOCAL_MR && mrMode != CLUSTER_MR) { - throw new IllegalArgumentException( - "Invalid MapRed mode, must be LOCAL_MR or CLUSTER_MR"); - } - if (fsMode != LOCAL_FS && fsMode != DFS_FS) { - throw new IllegalArgumentException( - "Invalid FileSystem mode, must be LOCAL_FS or DFS_FS"); - } - if (taskTrackers < 1) { - throw new IllegalArgumentException( - "Invalid taskTrackers value, must be greater than 0"); - } - if (dataNodes < 1) { - throw new IllegalArgumentException( - "Invalid dataNodes value, must be greater than 0"); - } - localMR = (mrMode == LOCAL_MR); - localFS = (fsMode == LOCAL_FS); - /* - JobConf conf = new JobConf(); - fsRoot = conf.get("hadoop.tmp.dir"); - - if (fsRoot == null) { - throw new IllegalArgumentException( - "hadoop.tmp.dir is not defined"); - } - - fsRoot = fsRoot.replace(' ', '+') + "/fs"; - - File file = new File(fsRoot); - if (!file.exists()) { - if (!file.mkdirs()) { - throw new RuntimeException("Could not create FS base path: " + file); - } - } - */ - this.taskTrackers = taskTrackers; - this.dataNodes = dataNodes; - } - - /** - * Indicates if the MR is running in local or cluster mode. - * - * @return returns TRUE if the MR is running locally, FALSE if running in - * cluster mode. - */ - public boolean isLocalMR() { - return localMR; - } - - /** - * Indicates if the filesystem is local or DFS. - * - * @return returns TRUE if the filesystem is local, FALSE if it is DFS. - */ - public boolean isLocalFS() { - return localFS; - } - - - private MiniDFSCluster dfsCluster = null; - private MiniMRCluster mrCluster = null; - private FileSystem fileSystem = null; - - /** - * Creates Hadoop instance based on constructor configuration before - * a test case is run. - * - * @throws Exception - */ - protected void setUp() throws Exception { - super.setUp(); - if (localFS) { - fileSystem = FileSystem.getLocal(new JobConf()); - } - else { - dfsCluster = new MiniDFSCluster(new JobConf(), dataNodes, true, null); - fileSystem = dfsCluster.getFileSystem(); - } - if (localMR) { - } - else { - //noinspection deprecation - mrCluster = new MiniMRCluster(taskTrackers, fileSystem.getUri().toString(), 1); - } - } - - /** - * Destroys Hadoop instance based on constructor configuration after - * a test case is run. - * - * @throws Exception - */ - protected void tearDown() throws Exception { - try { - if (mrCluster != null) { - mrCluster.shutdown(); - } - } - catch (Exception ex) { - System.out.println(ex); - } - try { - if (dfsCluster != null) { - dfsCluster.shutdown(); - } - } - catch (Exception ex) { - System.out.println(ex); - } - super.tearDown(); - } - - /** - * Returns the Filesystem in use. - * - * TestCases should use this Filesystem as it - * is properly configured with the workingDir for relative PATHs. - * - * @return the filesystem used by Hadoop. - */ - protected FileSystem getFileSystem() { - return fileSystem; - } - - /** - * Returns a job configuration preconfigured to run against the Hadoop - * managed by the testcase. - * @return configuration that works on the testcase Hadoop instance - */ - protected JobConf createJobConf() { - if (localMR) { - JobConf conf = new JobConf(); - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); - return conf; - } - else { - return mrCluster.createJobConf(); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java deleted file mode 100644 index 86980bb73d6..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java +++ /dev/null @@ -1,778 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.net.StaticMapping; -import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.UserGroupInformation; - -/** - * This class creates a single-process Map-Reduce cluster for junit testing. - * One thread is created for each server. - */ -public class MiniMRCluster { - private static final Log LOG = LogFactory.getLog(MiniMRCluster.class); - - private Thread jobTrackerThread; - private JobTrackerRunner jobTracker; - - private int jobTrackerPort = 0; - private int taskTrackerPort = 0; - private int jobTrackerInfoPort = 0; - private int numTaskTrackers; - - private List taskTrackerList = new ArrayList(); - private List taskTrackerThreadList = new ArrayList(); - - private String namenode; - private UserGroupInformation ugi = null; - private JobConf conf; - private int numTrackerToExclude; - - private JobConf job; - private Clock clock; - - /** - * An inner class that runs a job tracker. - */ - public class JobTrackerRunner implements Runnable { - private JobTracker tracker = null; - private volatile boolean isActive = true; - - JobConf jc = null; - Clock clock = JobTracker.DEFAULT_CLOCK; - - public JobTrackerRunner(JobConf conf) { - jc = conf; - } - - public JobTrackerRunner(JobConf conf, Clock clock) { - jc = conf; - this.clock = clock; - } - - public boolean isUp() { - return (tracker != null); - } - - public boolean isActive() { - return isActive; - } - - public int getJobTrackerPort() { - return tracker.getTrackerPort(); - } - - public int getJobTrackerInfoPort() { - return tracker.getInfoPort(); - } - - public JobTracker getJobTracker() { - return tracker; - } - - /** - * Create the job tracker and run it. - */ - public void run() { - try { - jc = (jc == null) ? createJobConf() : createJobConf(jc); - File f = new File("build/test/mapred/local").getAbsoluteFile(); - jc.set(MRConfig.LOCAL_DIR, f.getAbsolutePath()); - jc.setClass("topology.node.switch.mapping.impl", - StaticMapping.class, DNSToSwitchMapping.class); - final String id = - new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date()); - if (ugi == null) { - ugi = UserGroupInformation.getLoginUser(); - } - tracker = ugi.doAs(new PrivilegedExceptionAction() { - public JobTracker run() throws InterruptedException, IOException { - return JobTracker.startTracker(jc, clock, id); - } - }); - tracker.offerService(); - } catch (Throwable e) { - LOG.error("Job tracker crashed", e); - isActive = false; - } - } - - /** - * Shutdown the job tracker and wait for it to finish. - */ - public void shutdown() { - try { - if (tracker != null) { - tracker.stopTracker(); - } - } catch (Throwable e) { - LOG.error("Problem shutting down job tracker", e); - } - isActive = false; - } - } - - /** - * An inner class to run the task tracker. - */ - class TaskTrackerRunner implements Runnable { - volatile TaskTracker tt; - int trackerId; - // the localDirs for this taskTracker - String[] localDirs; - volatile boolean isInitialized = false; - volatile boolean isDead = false; - int numDir; - - TaskTrackerRunner(int trackerId, int numDir, String hostname, - JobConf cfg) - throws IOException { - this.trackerId = trackerId; - this.numDir = numDir; - localDirs = new String[numDir]; - final JobConf conf; - if (cfg == null) { - conf = createJobConf(); - } else { - conf = createJobConf(cfg); - } - if (hostname != null) { - conf.set(TTConfig.TT_HOST_NAME, hostname); - } - conf.set(TTConfig.TT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.set(TTConfig.TT_REPORT_ADDRESS, - "127.0.0.1:" + taskTrackerPort); - File localDirBase = - new File(conf.get(MRConfig.LOCAL_DIR)).getAbsoluteFile(); - localDirBase.mkdirs(); - StringBuffer localPath = new StringBuffer(); - for(int i=0; i < numDir; ++i) { - File ttDir = new File(localDirBase, - Integer.toString(trackerId) + "_" + i); - if (!ttDir.mkdirs()) { - if (!ttDir.isDirectory()) { - throw new IOException("Mkdirs failed to create " + ttDir); - } - } - localDirs[i] = ttDir.toString(); - if (i != 0) { - localPath.append(","); - } - localPath.append(localDirs[i]); - } - conf.set(MRConfig.LOCAL_DIR, localPath.toString()); - LOG.info(MRConfig.LOCAL_DIR + " is " + localPath); - try { - tt = ugi.doAs(new PrivilegedExceptionAction() { - public TaskTracker run() throws InterruptedException, IOException { - return createTaskTracker(conf); - } - }); - isInitialized = true; - } catch (Throwable e) { - isDead = true; - tt = null; - LOG.error("task tracker " + trackerId + " crashed", e); - } - } - - /** - * Creates a default {@link TaskTracker} using the conf passed. - */ - TaskTracker createTaskTracker(JobConf conf) - throws IOException, InterruptedException { - return new TaskTracker(conf); - } - - /** - * Create and run the task tracker. - */ - public void run() { - try { - if (tt != null) { - tt.run(); - } - } catch (Throwable e) { - isDead = true; - tt = null; - LOG.error("task tracker " + trackerId + " crashed", e); - } - } - - /** - * Get the local dir for this TaskTracker. - * This is there so that we do not break - * previous tests. - * @return the absolute pathname - */ - public String getLocalDir() { - return localDirs[0]; - } - - public String[] getLocalDirs(){ - return localDirs; - } - - public TaskTracker getTaskTracker() { - return tt; - } - - /** - * Shut down the server and wait for it to finish. - */ - public void shutdown() { - if (tt != null) { - try { - tt.shutdown(); - } catch (Throwable e) { - LOG.error("task tracker " + trackerId + " could not shut down", - e); - } - } - } - } - - /** - * Get the local directory for the Nth task tracker - * @param taskTracker the index of the task tracker to check - * @return the absolute pathname of the local dir - */ - public String getTaskTrackerLocalDir(int taskTracker) { - return (taskTrackerList.get(taskTracker)).getLocalDir(); - } - - /** - * Get all the local directories for the Nth task tracker - * @param taskTracker the index of the task tracker to check - * @return array of local dirs - */ - public String[] getTaskTrackerLocalDirs(int taskTracker) { - return (taskTrackerList.get(taskTracker)).getLocalDirs(); - } - - public JobTrackerRunner getJobTrackerRunner() { - return jobTracker; - } - - TaskTrackerRunner getTaskTrackerRunner(int id) { - return taskTrackerList.get(id); - } - /** - * Get the number of task trackers in the cluster - */ - public int getNumTaskTrackers() { - return taskTrackerList.size(); - } - - /** - * Sets inline cleanup threads to all task trackers sothat deletion of - * temporary files/dirs happen inline - */ - public void setInlineCleanupThreads() { - for (int i = 0; i < getNumTaskTrackers(); i++) { - getTaskTrackerRunner(i).getTaskTracker().setCleanupThread( - new UtilsForTests.InlineCleanupQueue()); - } - } - - /** - * Wait until the system is idle. - */ - public void waitUntilIdle() { - waitTaskTrackers(); - - JobClient client; - try { - client = new JobClient(job); - ClusterStatus status = client.getClusterStatus(); - while(status.getTaskTrackers() + numTrackerToExclude - < taskTrackerList.size()) { - for(TaskTrackerRunner runner : taskTrackerList) { - if(runner.isDead) { - throw new RuntimeException("TaskTracker is dead"); - } - } - Thread.sleep(1000); - status = client.getClusterStatus(); - } - } - catch (IOException ex) { - throw new RuntimeException(ex); - } - catch (InterruptedException ex) { - throw new RuntimeException(ex); - } - - } - - private void waitTaskTrackers() { - for(Iterator itr= taskTrackerList.iterator(); itr.hasNext();) { - TaskTrackerRunner runner = itr.next(); - while (!runner.isDead && (!runner.isInitialized || !runner.tt.isIdle())) { - if (!runner.isInitialized) { - LOG.info("Waiting for task tracker to start."); - } else { - LOG.info("Waiting for task tracker " + runner.tt.getName() + - " to be idle."); - } - try { - Thread.sleep(1000); - } catch (InterruptedException ie) {} - } - } - } - - /** - * Get the actual rpc port used. - */ - public int getJobTrackerPort() { - return jobTrackerPort; - } - - public JobConf createJobConf() { - return createJobConf(new JobConf()); - } - - public JobConf createJobConf(JobConf conf) { - if(conf == null) { - conf = new JobConf(); - } - return configureJobConf(conf, namenode, jobTrackerPort, jobTrackerInfoPort, - ugi); - } - - static JobConf configureJobConf(JobConf conf, String namenode, - int jobTrackerPort, int jobTrackerInfoPort, - UserGroupInformation ugi) { - JobConf result = new JobConf(conf); - FileSystem.setDefaultUri(result, namenode); - result.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); - result.set(JTConfig.JT_IPC_ADDRESS, "localhost:"+jobTrackerPort); - result.set(JTConfig.JT_HTTP_ADDRESS, - "127.0.0.1:" + jobTrackerInfoPort); - // for debugging have all task output sent to the test output - JobClient.setTaskOutputFilter(result, JobClient.TaskStatusFilter.ALL); - return result; - } - - /** - * Create the config and the cluster. - * @param numTaskTrackers no. of tasktrackers in the cluster - * @param namenode the namenode - * @param numDir no. of directories - * @throws IOException - */ - public MiniMRCluster(int numTaskTrackers, String namenode, int numDir, - String[] racks, String[] hosts) throws IOException { - this(0, 0, numTaskTrackers, namenode, numDir, racks, hosts); - } - - /** - * Create the config and the cluster. - * @param numTaskTrackers no. of tasktrackers in the cluster - * @param namenode the namenode - * @param numDir no. of directories - * @param racks Array of racks - * @param hosts Array of hosts in the corresponding racks - * @param conf Default conf for the jobtracker - * @throws IOException - */ - public MiniMRCluster(int numTaskTrackers, String namenode, int numDir, - String[] racks, String[] hosts, JobConf conf) - throws IOException { - this(0, 0, numTaskTrackers, namenode, numDir, racks, hosts, null, conf); - } - - /** - * Create the config and the cluster. - * @param numTaskTrackers no. of tasktrackers in the cluster - * @param namenode the namenode - * @param numDir no. of directories - * @throws IOException - */ - public MiniMRCluster(int numTaskTrackers, String namenode, int numDir) - throws IOException { - this(0, 0, numTaskTrackers, namenode, numDir); - } - - public MiniMRCluster(int jobTrackerPort, - int taskTrackerPort, - int numTaskTrackers, - String namenode, - int numDir) - throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, - numDir, null); - } - - public MiniMRCluster(int jobTrackerPort, - int taskTrackerPort, - int numTaskTrackers, - String namenode, - int numDir, - String[] racks) throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, - numDir, racks, null); - } - - public MiniMRCluster(int jobTrackerPort, - int taskTrackerPort, - int numTaskTrackers, - String namenode, - int numDir, - String[] racks, String[] hosts) throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, - numDir, racks, hosts, null); - } - - public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, - int numTaskTrackers, String namenode, - int numDir, String[] racks, String[] hosts, UserGroupInformation ugi - ) throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, - numDir, racks, hosts, ugi, null); - } - - public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, - int numTaskTrackers, String namenode, - int numDir, String[] racks, String[] hosts, UserGroupInformation ugi, - JobConf conf) throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, numDir, - racks, hosts, ugi, conf, 0); - } - - public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, - int numTaskTrackers, String namenode, - int numDir, String[] racks, String[] hosts, UserGroupInformation ugi, - JobConf conf, int numTrackerToExclude) throws IOException { - this(jobTrackerPort, taskTrackerPort, numTaskTrackers, namenode, numDir, - racks, hosts, ugi, conf, numTrackerToExclude, new Clock()); - } - - public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, - int numTaskTrackers, String namenode, - int numDir, String[] racks, String[] hosts, UserGroupInformation ugi, - JobConf conf, int numTrackerToExclude, Clock clock) throws IOException { - if (racks != null && racks.length < numTaskTrackers) { - LOG.error("Invalid number of racks specified. It should be at least " + - "equal to the number of tasktrackers"); - shutdown(); - } - if (hosts != null && numTaskTrackers > hosts.length ) { - throw new IllegalArgumentException( "The length of hosts [" + hosts.length - + "] is less than the number of tasktrackers [" + numTaskTrackers + "]."); - } - - //Generate rack names if required - if (racks == null) { - System.out.println("Generating rack names for tasktrackers"); - racks = new String[numTaskTrackers]; - for (int i=0; i < racks.length; ++i) { - racks[i] = NetworkTopology.DEFAULT_RACK; - } - } - - //Generate some hostnames if required - if (hosts == null) { - System.out.println("Generating host names for tasktrackers"); - hosts = new String[numTaskTrackers]; - for (int i = 0; i < numTaskTrackers; i++) { - hosts[i] = "host" + i + ".foo.com"; - } - } - this.jobTrackerPort = jobTrackerPort; - this.taskTrackerPort = taskTrackerPort; - this.jobTrackerInfoPort = 0; - this.numTaskTrackers = 0; - this.namenode = namenode; - this.ugi = ugi; - this.conf = conf; // this is the conf the mr starts with - this.numTrackerToExclude = numTrackerToExclude; - this.clock = clock; - - // start the jobtracker - startJobTracker(); - - // Create the TaskTrackers - for (int idx = 0; idx < numTaskTrackers; idx++) { - String rack = null; - String host = null; - if (racks != null) { - rack = racks[idx]; - } - if (hosts != null) { - host = hosts[idx]; - } - - startTaskTracker(host, rack, idx, numDir); - } - - this.job = createJobConf(conf); - waitUntilIdle(); - } - - public UserGroupInformation getUgi() { - return ugi; - } - - /** - * Get the task completion events - */ - public TaskCompletionEvent[] getTaskCompletionEvents(JobID id, int from, - int max) - throws IOException { - return jobTracker.getJobTracker().getTaskCompletionEvents(id, from, max); - } - - /** - * Change the job's priority - * - * @throws IOException - * @throws AccessControlException - */ - public void setJobPriority(JobID jobId, JobPriority priority) - throws AccessControlException, IOException { - jobTracker.getJobTracker().setJobPriority(jobId, priority); - } - - /** - * Get the job's priority - */ - public JobPriority getJobPriority(JobID jobId) { - return jobTracker.getJobTracker().getJob(jobId).getPriority(); - } - - /** - * Get the job finish time - */ - public long getJobFinishTime(JobID jobId) { - return jobTracker.getJobTracker().getJob(jobId).getFinishTime(); - } - - /** - * Init the job - */ - public void initializeJob(JobID jobId) throws IOException { - JobInProgress job = jobTracker.getJobTracker().getJob(jobId); - jobTracker.getJobTracker().initJob(job); - } - - /** - * Get the events list at the tasktracker - */ - public MapTaskCompletionEventsUpdate - getMapTaskCompletionEventsUpdates(int index, JobID jobId, int max) - throws IOException { - String jtId = jobTracker.getJobTracker().getTrackerIdentifier(); - TaskAttemptID dummy = - new TaskAttemptID(jtId, jobId.getId(), TaskType.REDUCE, 0, 0); - return taskTrackerList.get(index).getTaskTracker() - .getMapCompletionEvents(jobId, 0, max, - dummy); - } - - /** - * Get jobtracker conf - */ - public JobConf getJobTrackerConf() { - return this.conf; - } - - - public int getFaultCount(String hostName) { - return jobTracker.getJobTracker().getFaultCount(hostName); - } - - /** - * Start the jobtracker. - */ - public void startJobTracker() { - startJobTracker(true); - } - - public void startJobTracker(boolean wait) { - // Create the JobTracker - jobTracker = new JobTrackerRunner(conf, clock); - jobTrackerThread = new Thread(jobTracker); - - jobTrackerThread.start(); - - if (!wait) { - return; - } - - while (jobTracker.isActive() && !jobTracker.isUp()) { - try { // let daemons get started - Thread.sleep(1000); - } catch(InterruptedException e) { - } - } - - // is the jobtracker has started then wait for it to init - ClusterStatus status = null; - if (jobTracker.isUp()) { - status = jobTracker.getJobTracker().getClusterStatus(false); - while (jobTracker.isActive() && status.getJobTrackerStatus() - == JobTrackerStatus.INITIALIZING) { - try { - LOG.info("JobTracker still initializing. Waiting."); - Thread.sleep(1000); - } catch(InterruptedException e) {} - status = jobTracker.getJobTracker().getClusterStatus(false); - } - } - - if (!jobTracker.isActive()) { - // return if jobtracker has crashed - return; - } - - // Set the configuration for the task-trackers - this.jobTrackerPort = jobTracker.getJobTrackerPort(); - this.jobTrackerInfoPort = jobTracker.getJobTrackerInfoPort(); - } - - /** - * Kill the jobtracker. - */ - public void stopJobTracker() { - //jobTracker.exit(-1); - jobTracker.shutdown(); - - jobTrackerThread.interrupt(); - try { - jobTrackerThread.join(); - } catch (InterruptedException ex) { - LOG.error("Problem waiting for job tracker to finish", ex); - } - } - - /** - * Kill the tasktracker. - */ - public void stopTaskTracker(int id) { - TaskTrackerRunner tracker = taskTrackerList.remove(id); - tracker.shutdown(); - - Thread thread = taskTrackerThreadList.remove(id); - - try { - thread.join(); - // This will break the wait until idle loop - tracker.isDead = true; - --numTaskTrackers; - } catch (InterruptedException ex) { - LOG.error("Problem waiting for task tracker to finish", ex); - } - } - - /** - * Start the tasktracker. - */ - public void startTaskTracker(String host, String rack, int idx, int numDir) - throws IOException { - if (rack != null) { - StaticMapping.addNodeToRack(host, rack); - } - if (host != null) { - NetUtils.addStaticResolution(host, "localhost"); - } - TaskTrackerRunner taskTracker; - taskTracker = new TaskTrackerRunner(idx, numDir, host, conf); - - addTaskTracker(taskTracker); - } - - /** - * Add a task-tracker to the Mini-MR cluster. - */ - void addTaskTracker(TaskTrackerRunner taskTracker) { - Thread taskTrackerThread = new Thread(taskTracker); - taskTrackerList.add(taskTracker); - taskTrackerThreadList.add(taskTrackerThread); - taskTrackerThread.start(); - ++numTaskTrackers; - } - - /** - * Get the tasktrackerID in MiniMRCluster with given trackerName. - */ - int getTaskTrackerID(String trackerName) { - for (int id=0; id < numTaskTrackers; id++) { - if (taskTrackerList.get(id).getTaskTracker().getName().equals( - trackerName)) { - return id; - } - } - return -1; - } - - /** - * Shut down the servers. - */ - public void shutdown() { - try { - waitTaskTrackers(); - for (int idx = 0; idx < numTaskTrackers; idx++) { - TaskTrackerRunner taskTracker = taskTrackerList.get(idx); - Thread taskTrackerThread = taskTrackerThreadList.get(idx); - taskTracker.shutdown(); - try { - taskTrackerThread.join(); - } catch (InterruptedException ex) { - LOG.error("Problem shutting down task tracker", ex); - } - } - stopJobTracker(); - } finally { - File configDir = new File("build", "minimr"); - File siteFile = new File(configDir, "mapred-site.xml"); - siteFile.delete(); - } - } - - public static void main(String[] args) throws IOException { - LOG.info("Bringing up Jobtracker and tasktrackers."); - MiniMRCluster mr = new MiniMRCluster(4, "file:///", 1); - LOG.info("JobTracker and TaskTrackers are up."); - mr.shutdown(); - LOG.info("JobTracker and TaskTrackers brought down."); - } -} - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java deleted file mode 100644 index 026edfbddb0..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/NotificationTestCase.java +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import org.mortbay.jetty.Server; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpServlet; -import javax.servlet.ServletException; -import java.io.IOException; -import java.io.DataOutputStream; - -/** - * Base class to test Job end notification in local and cluster mode. - * - * Starts up hadoop on Local or Cluster mode (by extending of the - * HadoopTestCase class) and it starts a servlet engine that hosts - * a servlet that will receive the notification of job finalization. - * - * The notification servlet returns a HTTP 400 the first time is called - * and a HTTP 200 the second time, thus testing retry. - * - * In both cases local file system is used (this is irrelevant for - * the tested functionality) - * - * - */ -public abstract class NotificationTestCase extends HadoopTestCase { - - protected NotificationTestCase(int mode) throws IOException { - super(mode, HadoopTestCase.LOCAL_FS, 1, 1); - } - - private int port; - private String contextPath = "/notification"; - private String servletPath = "/mapred"; - private Server webServer; - - private void startHttpServer() throws Exception { - - // Create the webServer - if (webServer != null) { - webServer.stop(); - webServer = null; - } - webServer = new Server(0); - - Context context = new Context(webServer, contextPath); - - // create servlet handler - context.addServlet(new ServletHolder(new NotificationServlet()), - servletPath); - - // Start webServer - webServer.start(); - port = webServer.getConnectors()[0].getLocalPort(); - - } - - private void stopHttpServer() throws Exception { - if (webServer != null) { - webServer.stop(); - webServer.destroy(); - webServer = null; - } - } - - public static class NotificationServlet extends HttpServlet { - public static int counter = 0; - private static final long serialVersionUID = 1L; - - protected void doGet(HttpServletRequest req, HttpServletResponse res) - throws ServletException, IOException { - switch (counter) { - case 0: - { - assertTrue(req.getQueryString().contains("SUCCEEDED")); - } - break; - case 2: - { - assertTrue(req.getQueryString().contains("KILLED")); - } - break; - case 4: - { - assertTrue(req.getQueryString().contains("FAILED")); - } - break; - } - if (counter % 2 == 0) { - res.sendError(HttpServletResponse.SC_BAD_REQUEST, "forcing error"); - } - else { - res.setStatus(HttpServletResponse.SC_OK); - } - counter++; - } - } - - private String getNotificationUrlTemplate() { - return "http://localhost:" + port + contextPath + servletPath + - "?jobId=$jobId&jobStatus=$jobStatus"; - } - - protected JobConf createJobConf() { - JobConf conf = super.createJobConf(); - conf.setJobEndNotificationURI(getNotificationUrlTemplate()); - conf.setInt(JobContext.MR_JOB_END_RETRY_ATTEMPTS, 3); - conf.setInt(JobContext.MR_JOB_END_RETRY_INTERVAL, 200); - return conf; - } - - - protected void setUp() throws Exception { - super.setUp(); - startHttpServer(); - } - - protected void tearDown() throws Exception { - stopHttpServer(); - super.tearDown(); - } - - public void testMR() throws Exception { - System.out.println(launchWordCount(this.createJobConf(), - "a b c d e f g h", 1, 1)); - Thread.sleep(2000); - assertEquals(2, NotificationServlet.counter); - - Path inDir = new Path("notificationjob/input"); - Path outDir = new Path("notificationjob/output"); - - // Hack for local FS that does not have the concept of a 'mounting point' - if (isLocalFS()) { - String localPathRoot = System.getProperty("test.build.data","/tmp") - .toString().replace(' ', '+');; - inDir = new Path(localPathRoot, inDir); - outDir = new Path(localPathRoot, outDir); - } - - // run a job with KILLED status - System.out.println(UtilsForTests.runJobKill(this.createJobConf(), inDir, - outDir).getID()); - Thread.sleep(2000); - assertEquals(4, NotificationServlet.counter); - - // run a job with FAILED status - System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir, - outDir).getID()); - Thread.sleep(2000); - assertEquals(6, NotificationServlet.counter); - } - - private String launchWordCount(JobConf conf, - String input, - int numMaps, - int numReduces) throws IOException { - Path inDir = new Path("testing/wc/input"); - Path outDir = new Path("testing/wc/output"); - - // Hack for local FS that does not have the concept of a 'mounting point' - if (isLocalFS()) { - String localPathRoot = System.getProperty("test.build.data","/tmp") - .toString().replace(' ', '+');; - inDir = new Path(localPathRoot, inDir); - outDir = new Path(localPathRoot, outDir); - } - - FileSystem fs = FileSystem.get(conf); - fs.delete(outDir, true); - if (!fs.mkdirs(inDir)) { - throw new IOException("Mkdirs failed to create " + inDir.toString()); - } - { - DataOutputStream file = fs.create(new Path(inDir, "part-0")); - file.writeBytes(input); - file.close(); - } - conf.setJobName("wordcount"); - conf.setInputFormat(TextInputFormat.class); - - // the keys are words (strings) - conf.setOutputKeyClass(Text.class); - // the values are counts (ints) - conf.setOutputValueClass(IntWritable.class); - - conf.setMapperClass(WordCount.MapClass.class); - conf.setCombinerClass(WordCount.Reduce.class); - conf.setReducerClass(WordCount.Reduce.class); - - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReduces); - JobClient.runJob(conf); - return MapReduceTestUtil.readOutput(outDir, conf); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/QueueManagerTestUtils.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/QueueManagerTestUtils.java deleted file mode 100644 index 4cb0fee616c..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/QueueManagerTestUtils.java +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -//import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.UserGroupInformation; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.TransformerException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.dom.DOMSource; - -import java.security.PrivilegedExceptionAction; -import java.util.Properties; -import java.util.Set; -import java.io.File; -import java.io.IOException; - -//@Private -public class QueueManagerTestUtils { - /** - * Queue-configuration file for tests that start a cluster and wish to modify - * the queue configuration. This file is always in the unit tests classpath, - * so QueueManager started through JobTracker will automatically pick this up. - */ - public static final String QUEUES_CONFIG_FILE_PATH = new File(System - .getProperty("test.build.extraconf", "build/test/extraconf"), - QueueManager.QUEUE_CONF_FILE_NAME).getAbsolutePath(); - - private static final Log LOG = LogFactory.getLog(QueueManagerTestUtils.class); - - /** - * Create and return a new instance of a DOM Document object to build a queue - * tree with. - * - * @return the created {@link Document} - * @throws Exception - */ - public static Document createDocument() throws Exception { - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() - .newDocument(); - return doc; - } - - public static void createSimpleDocument(Document doc) throws Exception { - Element queues = createQueuesNode(doc); - - // Create parent level queue q1. - Element q1 = createQueue(doc, "q1"); - Properties props = new Properties(); - props.setProperty("capacity", "10"); - props.setProperty("maxCapacity", "35"); - q1.appendChild(createProperties(doc, props)); - queues.appendChild(q1); - - // Create another parent level p1 - Element p1 = createQueue(doc, "p1"); - - // append child p11 to p1 - p1.appendChild(createQueue(doc, "p11")); - - Element p12 = createQueue(doc, "p12"); - - p12.appendChild(createState(doc, QueueState.STOPPED.getStateName())); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "u1")); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "u2")); - - // append p12 to p1. - p1.appendChild(p12); - - queues.appendChild(p1); - } - - static void createSimpleDocumentWithAcls(Document doc) { - Element queues = createQueuesNode(doc); - - // Create parent level queue q1. - Element q1 = createQueue(doc, "q1"); - Properties props = new Properties(); - props.setProperty("capacity", "10"); - props.setProperty("maxCapacity", "35"); - q1.appendChild(createProperties(doc, props)); - queues.appendChild(q1); - - // Create another parent level p1 - Element p1 = createQueue(doc, "p1"); - - // append child p11 to p1 - Element p11 = createQueue(doc, "p11"); - p11.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "u1")); - p11.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "u2")); - p1.appendChild(p11); - - // append child p12 to p1 - Element p12 = createQueue(doc, "p12"); - p12.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "*")); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "*")); - p1.appendChild(p12); - - // append child p13 to p1 - Element p13 = createQueue(doc, "p13"); - p13.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - p1.appendChild(p13); - - // append child p14 to p1 - Element p14 = createQueue(doc, "p14"); - p14.appendChild(createState(doc, QueueState.STOPPED.getStateName())); - p1.appendChild(p14); - - queues.appendChild(p1); - } - - /** - * Creates all given queues as 1st level queues(no nesting) - * @param doc the queues config document - * @param queueNames the queues to be added to the queues config document - * @param submitAcls acl-submit-job acls for each of the queues - * @param adminsAcls acl-administer-jobs acls for each of the queues - * @throws Exception - */ - public static void createSimpleDocument(Document doc, String[] queueNames, - String[] submitAcls, String[] adminsAcls) throws Exception { - - Element queues = createQueuesNode(doc); - - // Create all queues as 1st level queues(no nesting) - for (int i = 0; i < queueNames.length; i++) { - Element q = createQueue(doc, queueNames[i]); - - q.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - q.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, submitAcls[i])); - q.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, adminsAcls[i])); - queues.appendChild(q); - } - } - - /** - * Creates queues configuration file with given queues at 1st level(i.e. - * no nesting of queues) and with the given queue acls. - * @param queueNames queue names which are to be configured - * @param submitAclStrings acl-submit-job acls for each of the queues - * @param adminsAclStrings acl-administer-jobs acls for each of the queues - * @return Configuration the queues configuration - * @throws Exception - */ - public static void createQueuesConfigFile(String[] queueNames, - String[] submitAclStrings, String[] adminsAclStrings) - throws Exception { - if (queueNames.length > submitAclStrings.length || - queueNames.length > adminsAclStrings.length) { - LOG.error("Number of queues is more than acls given."); - return; - } - Document doc = createDocument(); - createSimpleDocument(doc, queueNames, submitAclStrings, adminsAclStrings); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - } - - public static void refreshSimpleDocument(Document doc) throws Exception { - Element queues = createQueuesNode(doc); - - // Create parent level queue q1. - Element q1 = createQueue(doc, "q1"); - Properties props = new Properties(); - props.setProperty("capacity", "70"); - props.setProperty("maxCapacity", "35"); - q1.appendChild(createProperties(doc, props)); - queues.appendChild(q1); - - // Create another parent level p1 - Element p1 = createQueue(doc, "p1"); - - // append child p11 to p1 - Element p11 = createQueue(doc, "p11"); - p11.appendChild(createState(doc, QueueState.STOPPED.getStateName())); - p1.appendChild(p11); - - Element p12 = createQueue(doc, "p12"); - - p12.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - p12.appendChild(createAcls(doc, "acl-submit-job", "u3")); - p12.appendChild(createAcls(doc, "acl-administer-jobs", "u4")); - - // append p12 to p1. - p1.appendChild(p12); - - queues.appendChild(p1); - } - - /** - * Create the root element along with the - * element. - * - * @param doc - * @param enable - * @return the created element. - */ - public static Element createQueuesNode(Document doc) { - Element queues = doc.createElement("queues"); - doc.appendChild(queues); - return queues; - } - - public static void writeToFile(Document doc, String filePath) - throws TransformerException { - Transformer trans = TransformerFactory.newInstance().newTransformer(); - trans.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - trans.setOutputProperty(OutputKeys.INDENT, "yes"); - DOMSource source = new DOMSource(doc); - trans.transform(source, new StreamResult(new File(filePath))); - } - - public static Element createQueue(Document doc, String name) { - Element queue = doc.createElement("queue"); - Element nameNode = doc.createElement("name"); - nameNode.setTextContent(name); - queue.appendChild(nameNode); - return queue; - } - - public static Element createAcls(Document doc, String aclName, - String listNames) { - Element acls = doc.createElement(aclName); - acls.setTextContent(listNames); - return acls; - } - - public static Element createState(Document doc, String state) { - Element stateElement = doc.createElement("state"); - stateElement.setTextContent(state); - return stateElement; - } - - public static Element createProperties(Document doc, Properties props) { - Element propsElement = doc.createElement("properties"); - if (props != null) { - Set propList = props.stringPropertyNames(); - for (String prop : propList) { - Element property = doc.createElement("property"); - property.setAttribute("key", prop); - property.setAttribute("value", (String) props.get(prop)); - propsElement.appendChild(property); - } - } - return propsElement; - } - - /** - * Delete queues configuration file if exists - */ - public static void deleteQueuesConfigFile() { - if (new File(QUEUES_CONFIG_FILE_PATH).exists()) { - new File(QUEUES_CONFIG_FILE_PATH).delete(); - } - } - - /** - * Write the given queueHierarchy to the given file. - * - * @param filePath - * - * @param rootQueues - * @throws Exception - */ - public static void writeQueueConfigurationFile(String filePath, - JobQueueInfo[] rootQueues) throws Exception { - Document doc = createDocument(); - Element queueElements = createQueuesNode(doc); - for (JobQueueInfo rootQ : rootQueues) { - queueElements.appendChild(QueueConfigurationParser.getQueueElement(doc, - rootQ)); - } - writeToFile(doc, filePath); - } - - static Job submitSleepJob(final int numMappers, final int numReducers, final long mapSleepTime, - final long reduceSleepTime, boolean shouldComplete, String userInfo, - String queueName, Configuration clientConf) throws IOException, - InterruptedException, ClassNotFoundException { - clientConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); - clientConf.set(JTConfig.JT_IPC_ADDRESS, "localhost:" - + miniMRCluster.getJobTrackerPort()); - UserGroupInformation ugi; - if (userInfo != null) { - String[] splits = userInfo.split(","); - String[] groups = new String[splits.length - 1]; - System.arraycopy(splits, 1, groups, 0, splits.length - 1); - ugi = UserGroupInformation.createUserForTesting(splits[0], groups); - } else { - ugi = UserGroupInformation.getCurrentUser(); - } - if (queueName != null) { - clientConf.set(JobContext.QUEUE_NAME, queueName); - } - final SleepJob sleep = new SleepJob(); - sleep.setConf(clientConf); - - Job job = ugi.doAs(new PrivilegedExceptionAction() { - public Job run() throws IOException { - return sleep.createJob(numMappers, numReducers, mapSleepTime, - (int) mapSleepTime, reduceSleepTime, (int) reduceSleepTime); - }}); - if (shouldComplete) { - job.waitForCompletion(false); - } else { - job.submit(); - // miniMRCluster.getJobTrackerRunner().getJobTracker().jobsToComplete()[] - Cluster cluster = new Cluster(miniMRCluster.createJobConf()); - JobStatus[] status = miniMRCluster.getJobTrackerRunner().getJobTracker() - .jobsToComplete(); - JobID id = status[status.length -1].getJobID(); - Job newJob = cluster.getJob(id); - cluster.close(); - return newJob; - } - return job; - } - - static MiniMRCluster miniMRCluster; -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java deleted file mode 100644 index 6cfcabc9b71..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java +++ /dev/null @@ -1,597 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.*; -import java.net.URI; -import java.util.*; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.WritableComparator; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.mapred.lib.HashPartitioner; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.fs.*; - -/** - * A set of utilities to validate the sort of the map-reduce framework. - * This utility program has 2 main parts: - * 1. Checking the records' statistics - * a) Validates the no. of bytes and records in sort's input & output. - * b) Validates the xor of the md5's of each key/value pair. - * c) Ensures same key/value is present in both input and output. - * 2. Check individual records to ensure each record is present in both - * the input and the output of the sort (expensive on large data-sets). - * - * To run: bin/hadoop jar build/hadoop-examples.jar sortvalidate - * [-m maps] [-r reduces] [-deep] - * -sortInput sort-in-dir -sortOutput sort-out-dir - */ -public class SortValidator extends Configured implements Tool { - - static private final IntWritable sortInput = new IntWritable(1); - static private final IntWritable sortOutput = new IntWritable(2); - static public String SORT_REDUCES = - "mapreduce.sortvalidator.sort.reduce.tasks"; - static public String MAPS_PER_HOST = "mapreduce.sortvalidator.mapsperhost"; - static public String REDUCES_PER_HOST = - "mapreduce.sortvalidator.reducesperhost"; - static void printUsage() { - System.err.println("sortvalidate [-m ] [-r ] [-deep] " + - "-sortInput -sortOutput "); - System.exit(1); - } - - static private IntWritable deduceInputFile(JobConf job) { - Path[] inputPaths = FileInputFormat.getInputPaths(job); - Path inputFile = new Path(job.get(JobContext.MAP_INPUT_FILE)); - - // value == one for sort-input; value == two for sort-output - return (inputFile.getParent().equals(inputPaths[0])) ? - sortInput : sortOutput; - } - - static private byte[] pair(BytesWritable a, BytesWritable b) { - byte[] pairData = new byte[a.getLength()+ b.getLength()]; - System.arraycopy(a.getBytes(), 0, pairData, 0, a.getLength()); - System.arraycopy(b.getBytes(), 0, pairData, a.getLength(), b.getLength()); - return pairData; - } - - private static final PathFilter sortPathsFilter = new PathFilter() { - public boolean accept(Path path) { - return (path.getName().startsWith("part-")); - } - }; - - /** - * A simple map-reduce job which checks consistency of the - * MapReduce framework's sort by checking: - * a) Records are sorted correctly - * b) Keys are partitioned correctly - * c) The input and output have same no. of bytes and records. - * d) The input and output have the correct 'checksum' by xor'ing - * the md5 of each record. - * - */ - public static class RecordStatsChecker { - - /** - * Generic way to get raw data from a {@link Writable}. - */ - static class Raw { - /** - * Get raw data bytes from a {@link Writable} - * @param writable {@link Writable} object from whom to get the raw data - * @return raw data of the writable - */ - public byte[] getRawBytes(Writable writable) { - return writable.toString().getBytes(); - } - - /** - * Get number of raw data bytes of the {@link Writable} - * @param writable {@link Writable} object from whom to get the raw data - * length - * @return number of raw data bytes - */ - public int getRawBytesLength(Writable writable) { - return writable.toString().getBytes().length; - } - } - - /** - * Specialization of {@link Raw} for {@link BytesWritable}. - */ - static class RawBytesWritable extends Raw { - public byte[] getRawBytes(Writable bw) { - return ((BytesWritable)bw).getBytes(); - } - public int getRawBytesLength(Writable bw) { - return ((BytesWritable)bw).getLength(); - } - } - - /** - * Specialization of {@link Raw} for {@link Text}. - */ - static class RawText extends Raw { - public byte[] getRawBytes(Writable text) { - return ((Text)text).getBytes(); - } - public int getRawBytesLength(Writable text) { - return ((Text)text).getLength(); - } - } - - private static Raw createRaw(Class rawClass) { - if (rawClass == Text.class) { - return new RawText(); - } else if (rawClass == BytesWritable.class) { - System.err.println("Returning " + RawBytesWritable.class); - return new RawBytesWritable(); - } - return new Raw(); - } - - public static class RecordStatsWritable implements Writable { - private long bytes = 0; - private long records = 0; - private int checksum = 0; - - public RecordStatsWritable() {} - - public RecordStatsWritable(long bytes, long records, int checksum) { - this.bytes = bytes; - this.records = records; - this.checksum = checksum; - } - - public void write(DataOutput out) throws IOException { - WritableUtils.writeVLong(out, bytes); - WritableUtils.writeVLong(out, records); - WritableUtils.writeVInt(out, checksum); - } - - public void readFields(DataInput in) throws IOException { - bytes = WritableUtils.readVLong(in); - records = WritableUtils.readVLong(in); - checksum = WritableUtils.readVInt(in); - } - - public long getBytes() { return bytes; } - public long getRecords() { return records; } - public int getChecksum() { return checksum; } - } - - public static class Map extends MapReduceBase - implements Mapper { - - private IntWritable key = null; - private WritableComparable prevKey = null; - private Class keyClass; - private Partitioner partitioner = null; - private int partition = -1; - private int noSortReducers = -1; - private long recordId = -1; - - private Raw rawKey; - private Raw rawValue; - - public void configure(JobConf job) { - // 'key' == sortInput for sort-input; key == sortOutput for sort-output - key = deduceInputFile(job); - - if (key == sortOutput) { - partitioner = new HashPartitioner(); - - // Figure the 'current' partition and no. of reduces of the 'sort' - try { - URI inputURI = new URI(job.get(JobContext.MAP_INPUT_FILE)); - String inputFile = inputURI.getPath(); - // part file is of the form part-r-xxxxx - partition = Integer.valueOf(inputFile.substring( - inputFile.lastIndexOf("part") + 7)).intValue(); - noSortReducers = job.getInt(SORT_REDUCES, -1); - } catch (Exception e) { - System.err.println("Caught: " + e); - System.exit(-1); - } - } - } - - @SuppressWarnings("unchecked") - public void map(WritableComparable key, Writable value, - OutputCollector output, - Reporter reporter) throws IOException { - // Set up rawKey and rawValue on the first call to 'map' - if (recordId == -1) { - rawKey = createRaw(key.getClass()); - rawValue = createRaw(value.getClass()); - } - ++recordId; - - if (this.key == sortOutput) { - // Check if keys are 'sorted' if this - // record is from sort's output - if (prevKey == null) { - prevKey = key; - keyClass = prevKey.getClass(); - } else { - // Sanity check - if (keyClass != key.getClass()) { - throw new IOException("Type mismatch in key: expected " + - keyClass.getName() + ", received " + - key.getClass().getName()); - } - - // Check if they were sorted correctly - if (prevKey.compareTo(key) > 0) { - throw new IOException("The 'map-reduce' framework wrongly" + - " classifed (" + prevKey + ") > (" + - key + ") "+ "for record# " + recordId); - } - prevKey = key; - } - - // Check if the sorted output is 'partitioned' right - int keyPartition = - partitioner.getPartition(key, value, noSortReducers); - if (partition != keyPartition) { - throw new IOException("Partitions do not match for record# " + - recordId + " ! - '" + partition + "' v/s '" + - keyPartition + "'"); - } - } - - // Construct the record-stats and output (this.key, record-stats) - byte[] keyBytes = rawKey.getRawBytes(key); - int keyBytesLen = rawKey.getRawBytesLength(key); - byte[] valueBytes = rawValue.getRawBytes(value); - int valueBytesLen = rawValue.getRawBytesLength(value); - - int keyValueChecksum = - (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^ - WritableComparator.hashBytes(valueBytes, valueBytesLen)); - - output.collect(this.key, - new RecordStatsWritable((keyBytesLen+valueBytesLen), - 1, keyValueChecksum) - ); - } - - } - - public static class Reduce extends MapReduceBase - implements Reducer { - - public void reduce(IntWritable key, Iterator values, - OutputCollector output, - Reporter reporter) throws IOException { - long bytes = 0; - long records = 0; - int xor = 0; - while (values.hasNext()) { - RecordStatsWritable stats = values.next(); - bytes += stats.getBytes(); - records += stats.getRecords(); - xor ^= stats.getChecksum(); - } - - output.collect(key, new RecordStatsWritable(bytes, records, xor)); - } - } - - public static class NonSplitableSequenceFileInputFormat - extends SequenceFileInputFormat { - protected boolean isSplitable(FileSystem fs, Path filename) { - return false; - } - } - - static void checkRecords(Configuration defaults, - Path sortInput, Path sortOutput) throws IOException { - FileSystem inputfs = sortInput.getFileSystem(defaults); - FileSystem outputfs = sortOutput.getFileSystem(defaults); - FileSystem defaultfs = FileSystem.get(defaults); - JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class); - jobConf.setJobName("sortvalidate-recordstats-checker"); - - int noSortReduceTasks = - outputfs.listStatus(sortOutput, sortPathsFilter).length; - jobConf.setInt(SORT_REDUCES, noSortReduceTasks); - int noSortInputpaths = inputfs.listStatus(sortInput).length; - - jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); - jobConf.setOutputFormat(SequenceFileOutputFormat.class); - - jobConf.setOutputKeyClass(IntWritable.class); - jobConf.setOutputValueClass(RecordStatsChecker.RecordStatsWritable.class); - - jobConf.setMapperClass(Map.class); - jobConf.setCombinerClass(Reduce.class); - jobConf.setReducerClass(Reduce.class); - - jobConf.setNumMapTasks(noSortReduceTasks); - jobConf.setNumReduceTasks(1); - - FileInputFormat.setInputPaths(jobConf, sortInput); - FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path("/tmp", - "sortvalidate"), UUID.randomUUID().toString()); - if (defaultfs.exists(outputPath)) { - defaultfs.delete(outputPath, true); - } - FileOutputFormat.setOutputPath(jobConf, outputPath); - - // Uncomment to run locally in a single process - //job_conf.set(JTConfig.JT, "local"); - Path[] inputPaths = FileInputFormat.getInputPaths(jobConf); - System.out.println("\nSortValidator.RecordStatsChecker: Validate sort " + - "from " + inputPaths[0] + " (" + - noSortInputpaths + " files), " + - inputPaths[1] + " (" + - noSortReduceTasks + - " files) into " + - FileOutputFormat.getOutputPath(jobConf) + - " with 1 reducer."); - Date startTime = new Date(); - System.out.println("Job started: " + startTime); - JobClient.runJob(jobConf); - try { - Date end_time = new Date(); - System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); - - // Check to ensure that the statistics of the - // framework's sort-input and sort-output match - SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs, - new Path(outputPath, "part-00000"), defaults); - try { - IntWritable k1 = new IntWritable(); - IntWritable k2 = new IntWritable(); - RecordStatsWritable v1 = new RecordStatsWritable(); - RecordStatsWritable v2 = new RecordStatsWritable(); - if (!stats.next(k1, v1)) { - throw new IOException( - "Failed to read record #1 from reduce's output"); - } - if (!stats.next(k2, v2)) { - throw new IOException( - "Failed to read record #2 from reduce's output"); - } - - if ((v1.getBytes() != v2.getBytes()) || - (v1.getRecords() != v2.getRecords()) || - v1.getChecksum() != v2.getChecksum()) { - throw new IOException("(" + - v1.getBytes() + ", " + v1.getRecords() + ", " + v1.getChecksum() - + ") v/s (" + - v2.getBytes() + ", " + v2.getRecords() + ", " + v2.getChecksum() - + ")"); - } - } finally { - stats.close(); - } - } finally { - defaultfs.delete(outputPath, true); - } - } - - } - - /** - * A simple map-reduce task to check if the input and the output - * of the framework's sort is consistent by ensuring each record - * is present in both the input and the output. - * - */ - public static class RecordChecker { - - public static class Map extends MapReduceBase - implements Mapper { - - private IntWritable value = null; - - public void configure(JobConf job) { - // value == one for sort-input; value == two for sort-output - value = deduceInputFile(job); - } - - public void map(BytesWritable key, - BytesWritable value, - OutputCollector output, - Reporter reporter) throws IOException { - // newKey = (key, value) - BytesWritable keyValue = new BytesWritable(pair(key, value)); - - // output (newKey, value) - output.collect(keyValue, this.value); - } - } - - public static class Reduce extends MapReduceBase - implements Reducer { - - public void reduce(BytesWritable key, Iterator values, - OutputCollector output, - Reporter reporter) throws IOException { - int ones = 0; - int twos = 0; - while (values.hasNext()) { - IntWritable count = values.next(); - if (count.equals(sortInput)) { - ++ones; - } else if (count.equals(sortOutput)) { - ++twos; - } else { - throw new IOException("Invalid 'value' of " + count.get() + - " for (key,value): " + key.toString()); - } - } - - // Check to ensure there are equal no. of ones and twos - if (ones != twos) { - throw new IOException("Illegal ('one', 'two'): (" + ones + ", " + twos + - ") for (key, value): " + key.toString()); - } - } - } - - static void checkRecords(Configuration defaults, int noMaps, int noReduces, - Path sortInput, Path sortOutput) throws IOException { - JobConf jobConf = new JobConf(defaults, RecordChecker.class); - jobConf.setJobName("sortvalidate-record-checker"); - - jobConf.setInputFormat(SequenceFileInputFormat.class); - jobConf.setOutputFormat(SequenceFileOutputFormat.class); - - jobConf.setOutputKeyClass(BytesWritable.class); - jobConf.setOutputValueClass(IntWritable.class); - - jobConf.setMapperClass(Map.class); - jobConf.setReducerClass(Reduce.class); - - JobClient client = new JobClient(jobConf); - ClusterStatus cluster = client.getClusterStatus(); - if (noMaps == -1) { - noMaps = cluster.getTaskTrackers() * - jobConf.getInt(MAPS_PER_HOST, 10); - } - if (noReduces == -1) { - noReduces = (int) (cluster.getMaxReduceTasks() * 0.9); - String sortReduces = jobConf.get(REDUCES_PER_HOST); - if (sortReduces != null) { - noReduces = cluster.getTaskTrackers() * - Integer.parseInt(sortReduces); - } - } - jobConf.setNumMapTasks(noMaps); - jobConf.setNumReduceTasks(noReduces); - - FileInputFormat.setInputPaths(jobConf, sortInput); - FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path("/tmp/sortvalidate/recordchecker"); - FileSystem fs = FileSystem.get(defaults); - if (fs.exists(outputPath)) { - fs.delete(outputPath, true); - } - FileOutputFormat.setOutputPath(jobConf, outputPath); - - // Uncomment to run locally in a single process - //job_conf.set(JTConfig.JT, "local"); - Path[] inputPaths = FileInputFormat.getInputPaths(jobConf); - System.out.println("\nSortValidator.RecordChecker: Running on " + - cluster.getTaskTrackers() + - " nodes to validate sort from " + - inputPaths[0] + ", " + - inputPaths[1] + " into " + - FileOutputFormat.getOutputPath(jobConf) + - " with " + noReduces + " reduces."); - Date startTime = new Date(); - System.out.println("Job started: " + startTime); - JobClient.runJob(jobConf); - Date end_time = new Date(); - System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); - } - } - - - /** - * The main driver for sort-validator program. - * Invoke this method to submit the map/reduce job. - * @throws IOException When there is communication problems with the - * job tracker. - */ - public int run(String[] args) throws Exception { - Configuration defaults = getConf(); - - int noMaps = -1, noReduces = -1; - Path sortInput = null, sortOutput = null; - boolean deepTest = false; - for(int i=0; i < args.length; ++i) { - try { - if ("-m".equals(args[i])) { - noMaps = Integer.parseInt(args[++i]); - } else if ("-r".equals(args[i])) { - noReduces = Integer.parseInt(args[++i]); - } else if ("-sortInput".equals(args[i])){ - sortInput = new Path(args[++i]); - } else if ("-sortOutput".equals(args[i])){ - sortOutput = new Path(args[++i]); - } else if ("-deep".equals(args[i])) { - deepTest = true; - } else { - printUsage(); - return -1; - } - } catch (NumberFormatException except) { - System.err.println("ERROR: Integer expected instead of " + args[i]); - printUsage(); - return -1; - } catch (ArrayIndexOutOfBoundsException except) { - System.err.println("ERROR: Required parameter missing from " + - args[i-1]); - printUsage(); - return -1; - } - } - - // Sanity check - if (sortInput == null || sortOutput == null) { - printUsage(); - return -2; - } - - // Check if the records are consistent and sorted correctly - RecordStatsChecker.checkRecords(defaults, sortInput, sortOutput); - - // Check if the same records are present in sort's inputs & outputs - if (deepTest) { - RecordChecker.checkRecords(defaults, noMaps, noReduces, sortInput, - sortOutput); - } - - System.out.println("\nSUCCESS! Validated the MapReduce framework's 'sort'" + - " successfully."); - - return 0; - } - - public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new SortValidator(), args); - System.exit(res); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAdminOperationsProtocolWithServiceAuthorization.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAdminOperationsProtocolWithServiceAuthorization.java deleted file mode 100644 index 92ce67bd3dd..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestAdminOperationsProtocolWithServiceAuthorization.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import org.apache.hadoop.mapred.tools.MRAdmin; -import org.apache.hadoop.security.authorize.PolicyProvider; -import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; - -import junit.framework.TestCase; - -/** - * Test case to check if {@link AdminOperationsProtocol#refreshNodes()} and - * {@link AdminOperationsProtocol#refreshQueueAcls()} works with service-level - * authorization enabled i.e 'hadoop.security.authorization' set to true. - */ -public class TestAdminOperationsProtocolWithServiceAuthorization -extends TestCase { - public void testServiceLevelAuthorization() throws Exception { - MiniMRCluster mr = null; - try { - // Turn on service-level authorization - final JobConf conf = new JobConf(); - conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, - MapReducePolicyProvider.class, PolicyProvider.class); - conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, - true); - - // Start the mini mr cluster - mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); - - // Invoke MRAdmin commands - MRAdmin mrAdmin = new MRAdmin(mr.createJobConf()); - assertEquals(0, mrAdmin.run(new String[] { "-refreshQueues" })); - assertEquals(0, mrAdmin.run(new String[] { "-refreshNodes" })); - } finally { - if (mr != null) { - mr.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java deleted file mode 100644 index eb0f77c586a..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.ClusterMetrics; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -public class TestClusterStatus extends TestCase { - - private static String[] trackers = new String[] { "tracker_tracker1:1000", - "tracker_tracker2:1000", "tracker_tracker3:1000" }; - private static JobTracker jobTracker; - private static int mapSlotsPerTracker = 4; - private static int reduceSlotsPerTracker = 2; - private static MiniMRCluster mr; - private static FakeJobInProgress fakeJob = null; - private static Cluster cluster; - // heartbeat responseId. increment this after sending a heartbeat - private static short responseId = 1; - - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestClusterStatus.class)) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - conf.setClass(JTConfig.JT_TASK_SCHEDULER, FakeTaskScheduler.class, - TaskScheduler.class); - mr = new MiniMRCluster(0, "file:///", 1, null, null, new JobConf(conf)); - jobTracker = mr.getJobTrackerRunner().getJobTracker(); - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - cluster = new Cluster(mr.createJobConf()); - } - - protected void tearDown() throws Exception { - cluster.close(); - mr.shutdown(); - } - }; - return setup; - } - - /** - * Fake scheduler to test reservations. - * - * The reservations are updated incrementally in each - * heartbeat to pass through the re-reservation logic. - */ - static class FakeTaskScheduler extends JobQueueTaskScheduler { - - private Map reservedCounts - = new HashMap(); - - public FakeTaskScheduler() { - super(); - } - - public List assignTasks(TaskTracker tt) { - int currCount = 1; - if (reservedCounts.containsKey(tt)) { - currCount = reservedCounts.get(tt) + 1; - } - reservedCounts.put(tt, currCount); - tt.reserveSlots(TaskType.MAP, fakeJob, currCount); - tt.reserveSlots(TaskType.REDUCE, fakeJob, currCount); - return new ArrayList(); - } - } - - private TaskTrackerStatus getTTStatus(String trackerName, - List taskStatuses) { - return new TaskTrackerStatus(trackerName, - JobInProgress.convertTrackerNameToHostName(trackerName), 0, - taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker); - } - - public void testClusterMetrics() throws IOException, InterruptedException { - assertEquals("tasktracker count doesn't match", trackers.length, - cluster.getClusterStatus().getTaskTrackerCount()); - - List list = new ArrayList(); - - // create a map task status, which uses 2 slots. - int mapSlotsPerTask = 2; - addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.RUNNING); - - // create a reduce task status, which uses 1 slot. - int reduceSlotsPerTask = 1; - addReduceTaskAttemptToList(list, - reduceSlotsPerTask, TaskStatus.State.RUNNING); - - // create TaskTrackerStatus and send heartbeats - sendHeartbeats(list); - - // assert ClusterMetrics - ClusterMetrics metrics = cluster.getClusterStatus(); - assertEquals("occupied map slots do not match", mapSlotsPerTask, - metrics.getOccupiedMapSlots()); - assertEquals("occupied reduce slots do not match", reduceSlotsPerTask, - metrics.getOccupiedReduceSlots()); - assertEquals("map slot capacities do not match", - mapSlotsPerTracker * trackers.length, - metrics.getMapSlotCapacity()); - assertEquals("reduce slot capacities do not match", - reduceSlotsPerTracker * trackers.length, - metrics.getReduceSlotCapacity()); - assertEquals("running map tasks do not match", 1, - metrics.getRunningMaps()); - assertEquals("running reduce tasks do not match", 1, - metrics.getRunningReduces()); - - // assert the values in ClusterStatus also - assertEquals("running map tasks do not match", 1, - jobTracker.getClusterStatus().getMapTasks()); - assertEquals("running reduce tasks do not match", 1, - jobTracker.getClusterStatus().getReduceTasks()); - assertEquals("map slot capacities do not match", - mapSlotsPerTracker * trackers.length, - jobTracker.getClusterStatus().getMaxMapTasks()); - assertEquals("reduce slot capacities do not match", - reduceSlotsPerTracker * trackers.length, - jobTracker.getClusterStatus().getMaxReduceTasks()); - - // send a heartbeat finishing only a map and check - // counts are updated. - list.clear(); - addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.SUCCEEDED); - addReduceTaskAttemptToList(list, - reduceSlotsPerTask, TaskStatus.State.RUNNING); - sendHeartbeats(list); - metrics = jobTracker.getClusterMetrics(); - assertEquals(0, metrics.getOccupiedMapSlots()); - assertEquals(reduceSlotsPerTask, metrics.getOccupiedReduceSlots()); - - // send a heartbeat finishing the reduce task also. - list.clear(); - addReduceTaskAttemptToList(list, - reduceSlotsPerTask, TaskStatus.State.SUCCEEDED); - sendHeartbeats(list); - metrics = jobTracker.getClusterMetrics(); - assertEquals(0, metrics.getOccupiedReduceSlots()); - } - - private void sendHeartbeats(List list) throws IOException { - TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length]; - status[0] = getTTStatus(trackers[0], list); - status[1] = getTTStatus(trackers[1], new ArrayList()); - status[2] = getTTStatus(trackers[2], new ArrayList()); - for (int i = 0; i< trackers.length; i++) { - FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false, false, - trackers[i], responseId); - } - responseId++; - } - - private void addReduceTaskAttemptToList(List list, - int reduceSlotsPerTask, TaskStatus.State state) { - TaskStatus ts = TaskStatus.createTaskStatus(false, - new TaskAttemptID("jt", 1, TaskType.REDUCE, 0, 0), 0.0f, - reduceSlotsPerTask, - state, "", "", trackers[0], - TaskStatus.Phase.REDUCE, null); - list.add(ts); - } - - private void addMapTaskAttemptToList(List list, - int mapSlotsPerTask, TaskStatus.State state) { - TaskStatus ts = TaskStatus.createTaskStatus(true, - new TaskAttemptID("jt", 1, TaskType.MAP, 0, 0), 0.0f, mapSlotsPerTask, - state, "", "", trackers[0], - TaskStatus.Phase.MAP, null); - list.add(ts); - } - - public void testReservedSlots() throws Exception { - Configuration conf = mr.createJobConf(); - conf.setInt(JobContext.NUM_MAPS, 1); - - Job job = Job.getInstance(conf); - job.setNumReduceTasks(1); - job.setSpeculativeExecution(false); - job.setJobSetupCleanupNeeded(false); - - //Set task tracker objects for reservation. - TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]); - TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); - TaskTrackerStatus status1 = new TaskTrackerStatus( - trackers[0],JobInProgress.convertTrackerNameToHostName( - trackers[0]),0,new ArrayList(), 0, 2, 2); - TaskTrackerStatus status2 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - tt1.setStatus(status1); - tt2.setStatus(status2); - - fakeJob = new FakeJobInProgress(new JobConf(job.getConfiguration()), - jobTracker); - fakeJob.setClusterSize(3); - fakeJob.initTasks(); - - FakeObjectUtilities.sendHeartBeat(jobTracker, status1, false, - true, trackers[0], responseId); - FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false, - true, trackers[1], responseId); - responseId++; - ClusterMetrics metrics = cluster.getClusterStatus(); - assertEquals("reserved map slots do not match", - 2, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 2, metrics.getReservedReduceSlots()); - - // redo to test re-reservations. - FakeObjectUtilities.sendHeartBeat(jobTracker, status1, false, - true, trackers[0], responseId); - FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false, - true, trackers[1], responseId); - responseId++; - metrics = cluster.getClusterStatus(); - assertEquals("reserved map slots do not match", - 4, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 4, metrics.getReservedReduceSlots()); - - TaskAttemptID mTid = fakeJob.findMapTask(trackers[1]); - TaskAttemptID rTid = fakeJob.findReduceTask(trackers[1]); - - fakeJob.finishTask(mTid); - fakeJob.finishTask(rTid); - - assertEquals("Job didnt complete successfully complete", - fakeJob.getStatus().getRunState(), JobStatus.SUCCEEDED); - metrics = cluster.getClusterStatus(); - assertEquals("reserved map slots do not match", - 0, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 0, metrics.getReservedReduceSlots()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java deleted file mode 100644 index bbe1c6f7313..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestCompositeTaskTrackerInstrumentation { - private static final Log LOG = LogFactory.getLog( - TestCompositeTaskTrackerInstrumentation.class); - - @Test - public void testCompositeInstrumentation() throws IOException { - // Create two instrumentation instances - TaskTracker tt = new TaskTracker(); - DummyTaskTrackerInstrumentation inst1 = - new DummyTaskTrackerInstrumentation(tt); - DummyTaskTrackerInstrumentation inst2 = - new DummyTaskTrackerInstrumentation(tt); - - // Add them to a composite object - ArrayList insts = - new ArrayList(); - insts.add(inst1); - insts.add(inst2); - CompositeTaskTrackerInstrumentation comp = - new CompositeTaskTrackerInstrumentation(tt, insts); - - // Create some dummy objects to pass to instrumentation methods - TaskAttemptID tid = new TaskAttemptID(); - File file = new File("file"); - Task task = new MapTask(); - TaskStatus status = new MapTaskStatus(); - - // Test that completeTask propagates to listeners - assertFalse(inst1.completeTaskCalled); - assertFalse(inst2.completeTaskCalled); - comp.completeTask(tid); - assertTrue(inst1.completeTaskCalled); - assertTrue(inst2.completeTaskCalled); - - // Test that timedoutTask propagates to listeners - assertFalse(inst1.timedoutTaskCalled); - assertFalse(inst2.timedoutTaskCalled); - comp.timedoutTask(tid); - assertTrue(inst1.timedoutTaskCalled); - assertTrue(inst2.timedoutTaskCalled); - - // Test that taskFailedPing propagates to listeners - assertFalse(inst1.taskFailedPingCalled); - assertFalse(inst2.taskFailedPingCalled); - comp.taskFailedPing(tid); - assertTrue(inst1.taskFailedPingCalled); - assertTrue(inst2.taskFailedPingCalled); - - // Test that reportTaskLaunch propagates to listeners - assertFalse(inst1.reportTaskLaunchCalled); - assertFalse(inst2.reportTaskLaunchCalled); - comp.reportTaskLaunch(tid, file, file); - assertTrue(inst1.reportTaskLaunchCalled); - assertTrue(inst2.reportTaskLaunchCalled); - - // Test that reportTaskEnd propagates to listeners - assertFalse(inst1.reportTaskEndCalled); - assertFalse(inst2.reportTaskEndCalled); - comp.reportTaskEnd(tid); - assertTrue(inst1.reportTaskEndCalled); - assertTrue(inst2.reportTaskEndCalled); - - // Test that statusUpdate propagates to listeners - assertFalse(inst1.statusUpdateCalled); - assertFalse(inst2.statusUpdateCalled); - comp.statusUpdate(task, status); - assertTrue(inst1.statusUpdateCalled); - assertTrue(inst2.statusUpdateCalled); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompressedEmptyMapOutputs.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompressedEmptyMapOutputs.java deleted file mode 100644 index acf7242bd99..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCompressedEmptyMapOutputs.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.RandomWriter; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.util.ToolRunner; - -import junit.framework.TestCase; - -/** - * A JUnit test to test the Map-Reduce framework's sort in presence of - * null intermediate map-outputs, when compression is enabled for intermediate - * map-outputs. - */ -public class TestCompressedEmptyMapOutputs extends TestCase { - // Input/Output paths for sort - private static final Path SORT_INPUT_PATH = new Path("/sort/input"); - private static final Path SORT_OUTPUT_PATH = new Path("/sort/output"); - - // Knobs to control randomwriter; and hence sort - private static final int NUM_HADOOP_SLAVES = 3; - private static final int RW_BYTES_PER_MAP = 50000; - private static final int RW_MAPS_PER_HOST = 5; - - private static void runRandomWriter(JobConf job, Path sortInput) - throws Exception { - // Scale down the default settings for RandomWriter for the test-case - // Generates NUM_HADOOP_SLAVES * RW_MAPS_PER_HOST * RW_BYTES_PER_MAP -> 1MB - job.setInt(RandomWriter.BYTES_PER_MAP, RW_BYTES_PER_MAP); - job.setInt(RandomWriter.MAPS_PER_HOST, RW_MAPS_PER_HOST); - String[] rwArgs = {sortInput.toString()}; - - // Run RandomWriter - assertEquals(ToolRunner.run(job, new RandomWriter(), rwArgs), 0); - } - - - static class SinkMapper - extends MapReduceBase implements Mapper { - - public void map(K key, V val, - OutputCollector output, Reporter reporter) - throws IOException { - // Don't output anything! - if (false) output.collect(key, val); - } - } - - private static void runSort(JobConf jobConf, Path sortInput, Path sortOutput) - throws Exception { - // Set up the job - jobConf.setJobName("null-sorter"); - - jobConf.setMapperClass(SinkMapper.class); - jobConf.setReducerClass(IdentityReducer.class); - - jobConf.setNumReduceTasks(2); - - jobConf.setInputFormat(SequenceFileInputFormat.class); - jobConf.setOutputFormat(SequenceFileOutputFormat.class); - - jobConf.setOutputKeyClass(BytesWritable.class); - jobConf.setOutputValueClass(BytesWritable.class); - - FileInputFormat.setInputPaths(jobConf, sortInput); - FileOutputFormat.setOutputPath(jobConf, sortOutput); - - // Compress the intermediate map-outputs! - jobConf.setCompressMapOutput(true); - - // Run the job - JobClient.runJob(jobConf); - } - - public void testMapReduceSortWithCompressedEmptyMapOutputs() - throws Exception { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - Configuration conf = new Configuration(); - - // Start the mini-MR and mini-DFS clusters - dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null); - fileSys = dfs.getFileSystem(); - mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); - - // Run randomwriter to generate input for 'sort' - runRandomWriter(mr.createJobConf(), SORT_INPUT_PATH); - - // Run sort - runSort(mr.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); - } - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestControlledMapReduceJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestControlledMapReduceJob.java deleted file mode 100644 index f71a9a20829..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestControlledMapReduceJob.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.mapred.ControlledMapReduceJob.ControlledMapReduceJobRunner; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; - -/** - * Test to verify the controlled behavior of a ControlledMapReduceJob. - * - */ -public class TestControlledMapReduceJob extends ClusterMapReduceTestCase { - static final Log LOG = LogFactory.getLog(TestControlledMapReduceJob.class); - - /** - * Starts a job with 5 maps and 5 reduces. Then controls the finishing of - * tasks. Signals finishing tasks in batches and then verifies their - * completion. - * - * @throws Exception - */ - public void testControlledMapReduceJob() - throws Exception { - - Properties props = new Properties(); - props.setProperty(TTConfig.TT_MAP_SLOTS, "2"); - props.setProperty(TTConfig.TT_REDUCE_SLOTS, "2"); - startCluster(true, props); - LOG.info("Started the cluster"); - - ControlledMapReduceJobRunner jobRunner = - ControlledMapReduceJobRunner - .getControlledMapReduceJobRunner(createJobConf(), 7, 6); - jobRunner.start(); - ControlledMapReduceJob controlledJob = jobRunner.getJob(); - JobInProgress jip = - getMRCluster().getJobTrackerRunner().getJobTracker().getJob( - jobRunner.getJobID()); - - ControlledMapReduceJob.waitTillNTasksStartRunning(jip, true, 4); - LOG.info("Finishing 3 maps"); - controlledJob.finishNTasks(true, 3); - ControlledMapReduceJob.waitTillNTotalTasksFinish(jip, true, 3); - - ControlledMapReduceJob.waitTillNTasksStartRunning(jip, true, 4); - LOG.info("Finishing 4 more maps"); - controlledJob.finishNTasks(true, 4); - ControlledMapReduceJob.waitTillNTotalTasksFinish(jip, true, 7); - - ControlledMapReduceJob.waitTillNTasksStartRunning(jip, false, 4); - LOG.info("Finishing 2 reduces"); - controlledJob.finishNTasks(false, 2); - ControlledMapReduceJob.waitTillNTotalTasksFinish(jip, false, 2); - - ControlledMapReduceJob.waitTillNTasksStartRunning(jip, false, 4); - LOG.info("Finishing 4 more reduces"); - controlledJob.finishNTasks(false, 4); - ControlledMapReduceJob.waitTillNTotalTasksFinish(jip, false, 6); - - jobRunner.join(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java deleted file mode 100644 index 6c17da5b987..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.PrintWriter; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.security.Groups; -import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; -import org.apache.hadoop.security.UserGroupInformation; -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestDebugScript { - - // base directory which is used by the debug script - private static final String BASE_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).getAbsolutePath(); - - // script directory which is passed as dummy input + where debugscript - // is written. - private static final String SCRIPT_DIR = new File(BASE_DIR, "debugscript") - .getAbsolutePath(); - - // script which is used as debug script. - private static final String SCRIPT_FILE = new File(SCRIPT_DIR, - "debugscript.sh").getAbsolutePath(); - - // making an assumption we have bash in path. Same as TaskLog. - // The debug script just accesses the stderr file of the task - // and does a 'cat' on it - private static final String SCRIPT_CONTENT = "cat $2"; - - @Before - public void setup() throws Exception { - setupDebugScriptDirs(); - } - - @After - public void tearDown() throws Exception { - cleanupDebugScriptDirs(); - } - - /** - * Cleanup method which is used to delete the files folder which are generated - * by the testcase. - * - */ - static void cleanupDebugScriptDirs() { - File scriptFile = new File(SCRIPT_FILE); - scriptFile.delete(); - File scriptDir = new File(SCRIPT_DIR); - scriptDir.delete(); - } - - /** - * Setup method which is used to create necessary files and folder for the - * testcase. - * - * @throws Exception - */ - static void setupDebugScriptDirs() throws Exception { - File scriptDir = new File(SCRIPT_DIR); - if (!scriptDir.exists()) { - scriptDir.mkdirs(); - } - scriptDir.setExecutable(true, false); - scriptDir.setReadable(true, false); - scriptDir.setWritable(true, false); - File scriptFile = new File(SCRIPT_FILE); - PrintWriter writer = new PrintWriter(scriptFile); - writer.println(SCRIPT_CONTENT); - writer.flush(); - writer.close(); - scriptFile.setExecutable(true, false); - scriptFile.setReadable(true, false); - } - - /** - * Main test case which checks proper execution of the testcase. - * - * @throws Exception - */ - @Test - public void testDebugScript() throws Exception { - JobConf conf = new JobConf(); - conf.setLong(TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL, 0L); - MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, conf); - Path inputPath = new Path(SCRIPT_DIR); - Path outputPath = new Path(SCRIPT_DIR, "task_output"); - - // Run a failing mapper so debug script is launched. - JobID jobId = runFailingMapJob(mrCluster.createJobConf(), inputPath, - outputPath); - // construct the task id of first map task of failmap - TaskAttemptID taskId = new TaskAttemptID( - new TaskID(jobId,TaskType.MAP, 0), 0); - // verify if debug script was launched correctly and ran correctly. - verifyDebugScriptOutput(taskId); - } - - /** - * Method which verifies if debug script ran and ran correctly. - * - * @param taskId - * @param expectedUser - * expected user id from debug script - * @throws Exception - */ - static void verifyDebugScriptOutput(TaskAttemptID taskId) throws Exception { - verifyDebugScriptOutput(taskId, null, null, null); - } - /** - * Method which verifies if debug script ran and ran correctly. - * - * @param taskId - * @param expectedUser - * expected user id from debug script - * @param expectedPerms the expected permissions on the debugout file - * @throws Exception - */ - static void verifyDebugScriptOutput(TaskAttemptID taskId, String expectedUser, - String expectedGroup, String expectedPerms) throws Exception { - File output = TaskLog.getRealTaskLogFileLocation(taskId, false, - TaskLog.LogName.DEBUGOUT); - // Check the presence of the output file if the script is to be run. - assertTrue("Output file does not exists. DebugScript has not been run", - output.exists()); - // slurp the output from file, which is one line - BufferedReader reader = new BufferedReader(new FileReader(output)); - String out = reader.readLine(); - // close the file. - reader.close(); - // Check if there is any output - assertNotNull("DebugScript didn't generate output.", out); - assertTrue(out.contains("failing map")); - if (expectedPerms != null && expectedUser != null) { - //check whether the debugout file ownership/permissions are as expected - TestTaskTrackerLocalization.checkFilePermissions(output.getAbsolutePath(), - expectedPerms, expectedUser, expectedGroup); - } - } - - /** - * Method to run a failing mapper on a given Cluster. - * - * @param conf - * the JobConf for the job - * @param inputPath - * input path for the job. - * @param outputDir - * output directory for job. - * @throws IOException - */ - static JobID runFailingMapJob(JobConf conf, Path inputPath, Path outputDir) - throws IOException { - conf.setMapDebugScript(SCRIPT_FILE); - conf.setMaxMapAttempts(0); - conf.set("mapred.committer.job.setup.cleanup.needed", "false"); - RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir); - return rJob.getID(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java deleted file mode 100644 index 81d22afa435..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.InetAddress; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; - -/** - * A JUnit test to test Map-Reduce empty jobs. - */ -public class TestEmptyJob extends TestCase { - private static final Log LOG = - LogFactory.getLog(TestEmptyJob.class.getName()); - - private static String TEST_ROOT_DIR = - new File(System.getProperty("test.build.data", "/tmp")).toURI() - .toString().replace(' ', '+'); - - MiniMRCluster mr = null; - - /** Committer with commit waiting on a signal - */ - static class CommitterWithDelayCommit extends FileOutputCommitter { - @Override - public void commitJob(JobContext context) throws IOException { - Configuration conf = context.getConfiguration(); - Path share = new Path(conf.get("share")); - FileSystem fs = FileSystem.get(conf); - - - while (true) { - if (fs.exists(share)) { - break; - } - UtilsForTests.waitFor(100); - } - super.commitJob(context); - } - } - - /** - * Simple method running a MapReduce job with no input data. Used to test that - * such a job is successful. - * - * @param fileSys - * @param numMaps - * @param numReduces - * @return true if the MR job is successful, otherwise false - * @throws IOException - */ - private boolean launchEmptyJob(URI fileSys, int numMaps, int numReduces) - throws IOException { - // create an empty input dir - final Path inDir = new Path(TEST_ROOT_DIR, "testing/empty/input"); - final Path outDir = new Path(TEST_ROOT_DIR, "testing/empty/output"); - final Path inDir2 = new Path(TEST_ROOT_DIR, "testing/dummy/input"); - final Path outDir2 = new Path(TEST_ROOT_DIR, "testing/dummy/output"); - final Path share = new Path(TEST_ROOT_DIR, "share"); - - JobConf conf = mr.createJobConf(); - FileSystem fs = FileSystem.get(fileSys, conf); - fs.delete(new Path(TEST_ROOT_DIR), true); - fs.delete(outDir, true); - if (!fs.mkdirs(inDir)) { - LOG.warn("Can't create " + inDir); - return false; - } - - // use WordCount example - FileSystem.setDefaultUri(conf, fileSys); - conf.setJobName("empty"); - // use an InputFormat which returns no split - conf.setInputFormat(EmptyInputFormat.class); - conf.setOutputCommitter(CommitterWithDelayCommit.class); - conf.setOutputKeyClass(Text.class); - conf.setOutputValueClass(IntWritable.class); - conf.setMapperClass(IdentityMapper.class); - conf.setReducerClass(IdentityReducer.class); - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReduces); - conf.set("share", share.toString()); - - // run job and wait for completion - JobClient jc = new JobClient(conf); - RunningJob runningJob = jc.submitJob(conf); - JobInProgress job = mr.getJobTrackerRunner().getJobTracker().getJob(runningJob.getID()); - - InetAddress ip = InetAddress.getLocalHost(); - if (ip != null) { - assertTrue(job.getJobSubmitHostAddress().equalsIgnoreCase( - ip.getHostAddress())); - assertTrue(job.getJobSubmitHostName().equalsIgnoreCase(ip.getHostName())); - } - - while (true) { - if (job.isCleanupLaunched()) { - LOG.info("Waiting for cleanup to be launched for job " - + runningJob.getID()); - break; - } - UtilsForTests.waitFor(100); - } - - // submit another job so that the map load increases and scheduling happens - LOG.info("Launching dummy job "); - RunningJob dJob = null; - try { - JobConf dConf = new JobConf(conf); - dConf.setOutputCommitter(FileOutputCommitter.class); - dJob = UtilsForTests.runJob(dConf, inDir2, outDir2, 2, 0); - } catch (Exception e) { - LOG.info("Exception ", e); - throw new IOException(e); - } - - while (true) { - LOG.info("Waiting for job " + dJob.getID() + " to complete"); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - } - if (dJob.isComplete()) { - break; - } - } - - // check if the second job is successful - assertTrue(dJob.isSuccessful()); - - // signal the cleanup - fs.create(share).close(); - - while (true) { - LOG.info("Waiting for job " + runningJob.getID() + " to complete"); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - } - if (runningJob.isComplete()) { - break; - } - } - - assertTrue(runningJob.isComplete()); - assertTrue(runningJob.isSuccessful()); - JobID jobID = runningJob.getID(); - - TaskReport[] jobSetupTasks = jc.getSetupTaskReports(jobID); - assertTrue("Number of job-setup tips is not 2!", jobSetupTasks.length == 2); - assertTrue("Setup progress is " + runningJob.setupProgress() - + " and not 1.0", runningJob.setupProgress() == 1.0); - assertTrue("Setup task is not finished!", mr.getJobTrackerRunner() - .getJobTracker().getJob(jobID).isSetupFinished()); - - assertTrue("Number of maps is not zero!", jc.getMapTaskReports(runningJob - .getID()).length == 0); - assertTrue( - "Map progress is " + runningJob.mapProgress() + " and not 1.0!", - runningJob.mapProgress() == 1.0); - - assertTrue("Reduce progress is " + runningJob.reduceProgress() - + " and not 1.0!", runningJob.reduceProgress() == 1.0); - assertTrue("Number of reduces is not " + numReduces, jc - .getReduceTaskReports(runningJob.getID()).length == numReduces); - - TaskReport[] jobCleanupTasks = jc.getCleanupTaskReports(jobID); - assertTrue("Number of job-cleanup tips is not 2!", - jobCleanupTasks.length == 2); - assertTrue("Cleanup progress is " + runningJob.cleanupProgress() - + " and not 1.0", runningJob.cleanupProgress() == 1.0); - - assertTrue("Job output directory doesn't exit!", fs.exists(outDir)); - FileStatus[] list = fs.listStatus(outDir, - new Utils.OutputFileUtils.OutputFilesFilter()); - assertTrue("Number of part-files is " + list.length + " and not " - + numReduces, list.length == numReduces); - - // cleanup - fs.delete(outDir, true); - - // return job result - LOG.info("job is complete: " + runningJob.isSuccessful()); - return (runningJob.isSuccessful()); - } - - /** - * Test that a job with no input data (and thus with no input split and no map - * task to execute) is successful. - * - * @throws IOException - */ - public void testEmptyJob() - throws IOException { - FileSystem fileSys = null; - try { - final int taskTrackers = 2; - JobConf conf = new JobConf(); - fileSys = FileSystem.get(conf); - - conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); - conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "127.0.0.1:0"); - conf.set(TTConfig.TT_HTTP_ADDRESS, "127.0.0.1:0"); - - mr = - new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, - null, null, conf); - - assertTrue(launchEmptyJob(fileSys.getUri(), 3, 1)); - assertTrue(launchEmptyJob(fileSys.getUri(), 0, 0)); - } finally { - if (fileSys != null) { - fileSys.close(); - } - if (mr != null) { - mr.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java deleted file mode 100644 index cdbd96f5e59..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.security.UserGroupInformation; - -public class TestJobDirCleanup extends TestCase { - //The testcase brings up a cluster with many trackers, and - //runs a job with a single map and many reduces. The check is - //to see whether the job directories are cleaned up at the - //end of the job (indirectly testing whether all tasktrackers - //got a KillJobAction). - private JobID runSleepJob(JobConf conf) throws Exception { - SleepJob sleep = new SleepJob(); - sleep.setConf(conf); - Job job = sleep.createJob(1, 10, 1000, 1, 10000, 1); - job.waitForCompletion(true); - return job.getJobID(); - } - - public void testJobDirCleanup() throws Exception { - String namenode = null; - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int taskTrackers = 10; - Configuration conf = new Configuration(); - JobConf mrConf = new JobConf(); - mrConf.set(TTConfig.TT_REDUCE_SLOTS, "1"); - dfs = new MiniDFSCluster(conf, 1, true, null); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - mr = new MiniMRCluster(10, namenode, 3, - null, null, mrConf); - // make cleanup inline sothat validation of existence of these directories - // can be done - mr.setInlineCleanupThreads(); - - // run the sleep job - JobConf jobConf = mr.createJobConf(); - JobID jobid = runSleepJob(jobConf); - - // verify the job directories are cleaned up. - verifyJobDirCleanup(mr, taskTrackers, jobid); - } finally { - if (fileSys != null) { fileSys.close(); } - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); } - } - } - - static void verifyJobDirCleanup(MiniMRCluster mr, int numTT, JobID jobid) - throws IOException { - // wait till killJobAction is sent to all trackers. - // this loops waits atmost for 10 seconds - boolean sent = true; - for (int i = 0; i < 100; i++) { - sent = true; - for (int j = 0; j < numTT; j++) { - if (mr.getTaskTrackerRunner(j).getTaskTracker().getRunningJob( - org.apache.hadoop.mapred.JobID.downgrade(jobid)) != null) { - sent = false; - break; - } - } - if (!sent) { - UtilsForTests.waitFor(100); - } else { - break; - } - } - - assertTrue("KillJobAction not sent for all trackers", sent); - String user = UserGroupInformation.getCurrentUser().getShortUserName(); - String jobDirStr = TaskTracker.getLocalJobDir(user, jobid.toString()); - for(int i=0; i < numTT; ++i) { - for (String localDir : mr.getTaskTrackerLocalDirs(i)) { - File jobDir = new File(localDir, jobDirStr); - assertFalse(jobDir + " is not cleaned up.", jobDir.exists()); - } - } - } -} - - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java deleted file mode 100644 index 6efe0b6f5e7..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java +++ /dev/null @@ -1,1047 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.JobACL; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.TaskID; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.jobhistory.JobHistory; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; -import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; -import org.apache.hadoop.net.Node; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AccessControlList; - -/** - * - * testJobHistoryFile - * Run a job that will be succeeded and validate its history file format and - * content. - * - * testJobHistoryJobStatus - * Run jobs that will be (1) succeeded (2) failed (3) killed. - * Validate job status read from history file in each case. - * - */ -public class TestJobHistory extends TestCase { - private static final Log LOG = LogFactory.getLog(TestJobHistory.class); - - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).toURI().toString().replace(' ', '+'); - - private static final String LOG_DIR = System.getProperty("hadoop.log.dir"); - - private static final String LOCAL_LOG_DIR_URI = new File(LOG_DIR).toURI() - .toString().replace(' ', '+') + "/history"; - - private static final String DIGITS = "[0-9]+"; - - // hostname like /default-rack/host1.foo.com OR host1.foo.com - private static final Pattern hostNamePattern = Pattern.compile( - "(/(([\\w\\-\\.]+)/)+)?([\\w\\-\\.]+)"); - - private static final String IP_ADDR = - "\\d\\d?\\d?\\.\\d\\d?\\d?\\.\\d\\d?\\d?\\.\\d\\d?\\d?"; - - private static final Pattern trackerNamePattern = Pattern.compile( - "tracker_" + hostNamePattern + ":([\\w\\-\\.]+)/" + - IP_ADDR + ":" + DIGITS); - - - private static final Pattern splitsPattern = Pattern.compile( - hostNamePattern + "(," + hostNamePattern + ")*"); - - private static Map> taskIDsToAttemptIDs = - new HashMap>(); - - //Each Task End seen from history file is added here - private static List taskEnds = new ArrayList(); - - - // Validate Format of Job Level Keys, Values read from history file - private static void validateJobLevelKeyValuesFormat(JobInfo jobInfo, - String status) { - long submitTime = jobInfo.getSubmitTime(); - long launchTime = jobInfo.getLaunchTime(); - long finishTime = jobInfo.getFinishTime(); - - assertTrue("Invalid submit time", submitTime > 0); - assertTrue("SubmitTime > LaunchTime", submitTime <= launchTime); - assertTrue("LaunchTime > FinishTime", launchTime <= finishTime); - - String stat = jobInfo.getJobStatus(); - - assertTrue("Unexpected JOB_STATUS \"" + stat + "\" is seen in" + - " history file", (status.equals(stat))); - String priority = jobInfo.getPriority(); - - assertNotNull(priority); - assertTrue("Unknown priority for the job in history file", - (priority.equals("HIGH") || - priority.equals("LOW") || priority.equals("NORMAL") || - priority.equals("VERY_HIGH") || priority.equals("VERY_LOW"))); - } - - // Validate Format of Task Level Keys, Values read from history file - private static void validateTaskLevelKeyValuesFormat(JobInfo job, - boolean splitsCanBeEmpty) { - Map tasks = job.getAllTasks(); - - // validate info of each task - for (TaskInfo task : tasks.values()) { - - TaskID tid = task.getTaskId(); - long startTime = task.getStartTime(); - assertTrue("Invalid Start time", startTime > 0); - - long finishTime = task.getFinishTime(); - assertTrue("Task FINISH_TIME is < START_TIME in history file", - startTime < finishTime); - - // Make sure that the Task type exists and it is valid - TaskType type = task.getTaskType(); - assertTrue("Unknown Task type \"" + type + "\" is seen in " + - "history file for task " + tid, - (type.equals(TaskType.MAP) || - type.equals(TaskType.REDUCE) || - type.equals(TaskType.JOB_CLEANUP) || - type.equals(TaskType.JOB_SETUP))); - - if (type.equals(TaskType.MAP)) { - String splits = task.getSplitLocations(); - //order in the condition OR check is important here - if (!splitsCanBeEmpty || splits.length() != 0) { - Matcher m = splitsPattern.matcher(splits); - assertTrue("Unexpected format of SPLITS \"" + splits + "\" is seen" + - " in history file for task " + tid, m.matches()); - } - } - - // Validate task status - String status = task.getTaskStatus(); - assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" + - " history file for task " + tid, (status.equals("SUCCEEDED") || - status.equals("FAILED") || status.equals("KILLED"))); - } - } - - // Validate foramt of Task Attempt Level Keys, Values read from history file - private static void validateTaskAttemptLevelKeyValuesFormat(JobInfo job) { - Map tasks = job.getAllTasks(); - - // For each task - for (TaskInfo task : tasks.values()) { - // validate info of each attempt - for (TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) { - - TaskAttemptID id = attempt.getAttemptId(); - assertNotNull(id); - - long startTime = attempt.getStartTime(); - assertTrue("Invalid Start time", startTime > 0); - - long finishTime = attempt.getFinishTime(); - assertTrue("Task FINISH_TIME is < START_TIME in history file", - startTime < finishTime); - - // Make sure that the Task type exists and it is valid - TaskType type = attempt.getTaskType(); - assertTrue("Unknown Task type \"" + type + "\" is seen in " + - "history file for task attempt " + id, - (type.equals(TaskType.MAP) || type.equals(TaskType.REDUCE) || - type.equals(TaskType.JOB_CLEANUP) || - type.equals(TaskType.JOB_SETUP))); - - // Validate task status - String status = attempt.getTaskStatus(); - assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" + - " history file for task attempt " + id, - (status.equals(TaskStatus.State.SUCCEEDED.toString()) || - status.equals(TaskStatus.State.FAILED.toString()) || - status.equals(TaskStatus.State.KILLED.toString()))); - - // Successful Reduce Task Attempts should have valid SHUFFLE_FINISHED - // time and SORT_FINISHED time - if (type.equals(TaskType.REDUCE) && - status.equals(TaskStatus.State.SUCCEEDED.toString())) { - long shuffleFinishTime = attempt.getShuffleFinishTime(); - assertTrue(startTime < shuffleFinishTime); - - long sortFinishTime = attempt.getSortFinishTime(); - assertTrue(shuffleFinishTime < sortFinishTime); - } - else if (type.equals(TaskType.MAP) && - status.equals(TaskStatus.State.SUCCEEDED.toString())) { - // Successful MAP Task Attempts should have valid MAP_FINISHED time - long mapFinishTime = attempt.getMapFinishTime(); - assertTrue(startTime < mapFinishTime); - } - - // check if hostname is valid - String hostname = attempt.getHostname(); - Matcher m = hostNamePattern.matcher(hostname); - assertTrue("Unexpected Host name of task attempt " + id, m.matches()); - - // check if trackername is valid - String trackerName = attempt.getTrackerName(); - m = trackerNamePattern.matcher(trackerName); - assertTrue("Unexpected tracker name of task attempt " + id, - m.matches()); - - if (!status.equals("KILLED")) { - // check if http port is valid - int httpPort = attempt.getHttpPort(); - assertTrue(httpPort > 0); - } - - // check if counters are parsable - Counters counters = attempt.getCounters(); - assertNotNull(counters); - } - } - } - - /** - * Returns the conf file name in the same - * @param path path of the jobhistory file - * @param running whether the job is running or completed - */ - private static Path getPathForConf(Path path, Path dir) { - String parts[] = path.getName().split("_"); - //TODO this is a hack :( - // jobtracker-hostname_jobtracker-identifier_ - String id = parts[0] + "_" + parts[1] + "_" + parts[2]; - return new Path(dir, id + "_conf.xml"); - } - - /** - * Validates the format of contents of history file - * (1) history file exists and in correct location - * (2) Verify if the history file is parsable - * (3) Validate the contents of history file - * (a) Format of all TIMEs are checked against a regex - * (b) validate legality/format of job level key, values - * (c) validate legality/format of task level key, values - * (d) validate legality/format of attempt level key, values - * (e) check if all the TaskAttempts, Tasks started are finished. - * Check finish of each TaskAttemptID against its start to make sure - * that all TaskAttempts, Tasks started are indeed finished and the - * history log lines are in the proper order. - * We want to catch ordering of history lines like - * Task START - * Attempt START - * Task FINISH - * Attempt FINISH - * (speculative execution is turned off for this). - * @param id job id - * @param conf job conf - */ - public static void validateJobHistoryFileFormat(JobHistory jobHistory, - JobID id, JobConf conf, - String status, boolean splitsCanBeEmpty) throws IOException { - - // Get the history file name - Path dir = jobHistory.getCompletedJobHistoryLocation(); - String logFileName = getDoneFile(jobHistory, conf, id, dir); - - // Framework history log file location - Path logFile = new Path(dir, logFileName); - FileSystem fileSys = logFile.getFileSystem(conf); - - // Check if the history file exists - assertTrue("History file does not exist", fileSys.exists(logFile)); - - JobHistoryParser parser = new JobHistoryParser(fileSys, - logFile.toUri().getPath()); - JobHistoryParser.JobInfo jobInfo = parser.parse(); - - // validate format of job level key, values - validateJobLevelKeyValuesFormat(jobInfo, status); - - // validate format of task level key, values - validateTaskLevelKeyValuesFormat(jobInfo, splitsCanBeEmpty); - - // validate format of attempt level key, values - validateTaskAttemptLevelKeyValuesFormat(jobInfo); - - // check if all the TaskAttempts, Tasks started are finished for - // successful jobs - if (status.equals("SUCCEEDED")) { - // Make sure that the lists in taskIDsToAttemptIDs are empty. - for(Iterator it = - taskIDsToAttemptIDs.keySet().iterator();it.hasNext();) { - String taskid = it.next(); - assertTrue("There are some Tasks which are not finished in history " + - "file.", taskEnds.contains(taskid)); - List attemptIDs = taskIDsToAttemptIDs.get(taskid); - if(attemptIDs != null) { - assertTrue("Unexpected. TaskID " + taskid + " has task attempt(s)" + - " that are not finished.", (attemptIDs.size() == 1)); - } - } - } - } - - // Validate Job Level Keys, Values read from history file by - // comparing them with the actual values from JT. - private static void validateJobLevelKeyValues(MiniMRCluster mr, - RunningJob job, JobInfo jobInfo, JobConf conf) throws IOException { - - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = jt.getJob(job.getID()); - - assertTrue("SUBMIT_TIME of job obtained from history file did not " + - "match the expected value", jip.getStartTime() == - jobInfo.getSubmitTime()); - - assertTrue("LAUNCH_TIME of job obtained from history file did not " + - "match the expected value", jip.getLaunchTime() == - jobInfo.getLaunchTime()); - - assertTrue("FINISH_TIME of job obtained from history file did not " + - "match the expected value", jip.getFinishTime() == - jobInfo.getFinishTime()); - - assertTrue("Job Status of job obtained from history file did not " + - "match the expected value", - jobInfo.getJobStatus().equals("SUCCEEDED")); - - assertTrue("Job Priority of job obtained from history file did not " + - "match the expected value", jip.getPriority().toString().equals( - jobInfo.getPriority())); - - assertTrue("Job Name of job obtained from history file did not " + - "match the expected value", - conf.getJobName().equals( - jobInfo.getJobname())); - String user = UserGroupInformation.getCurrentUser().getUserName(); - assertTrue("User Name of job obtained from history file did not " + - "match the expected value", - user.equals( - jobInfo.getUsername())); - - // Validate job counters - Counters c = new Counters(jip.getCounters()); - Counters jiCounters = jobInfo.getTotalCounters(); - assertTrue("Counters of job obtained from history file did not " + - "match the expected value", - c.equals(jiCounters)); - - // Validate number of total maps, total reduces, finished maps, - // finished reduces, failed maps, failed recudes - assertTrue("Unexpected number of total maps in history file", - jobInfo.getTotalMaps() == jip.desiredMaps()); - - assertTrue("Unexpected number of total reduces in history file", - jobInfo.getTotalReduces() == jip.desiredReduces()); - - assertTrue("Unexpected number of finished maps in history file", - jobInfo.getFinishedMaps() == jip.finishedMaps()); - - assertTrue("Unexpected number of finished reduces in history file", - jobInfo.getFinishedReduces() == jip.finishedReduces()); - - assertTrue("Unexpected number of failed maps in history file", - jobInfo.getFailedMaps() == jip.failedMapTasks); - - assertTrue("Unexpected number of failed reduces in history file", - jobInfo.getFailedReduces() == jip.failedReduceTasks); - } - - // Validate Task Level Keys, Values read from history file by - // comparing them with the actual values from JT. - private static void validateTaskLevelKeyValues(MiniMRCluster mr, - RunningJob job, JobInfo jobInfo) throws IOException { - - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = jt.getJob(job.getID()); - - // Get the 1st map, 1st reduce, cleanup & setup taskIDs and - // validate their history info - TaskID mapTaskId = new TaskID(job.getID(), TaskType.MAP, 0); - TaskID reduceTaskId = new TaskID(job.getID(), TaskType.REDUCE, 0); - - TaskInProgress cleanups[] = jip.cleanup; - TaskID cleanupTaskId; - if (cleanups[0].isComplete()) { - cleanupTaskId = cleanups[0].getTIPId(); - } - else { - cleanupTaskId = cleanups[1].getTIPId(); - } - - TaskInProgress setups[] = jip.setup; - TaskID setupTaskId; - if (setups[0].isComplete()) { - setupTaskId = setups[0].getTIPId(); - } - else { - setupTaskId = setups[1].getTIPId(); - } - - Map tasks = jobInfo.getAllTasks(); - - // validate info of the 4 tasks(cleanup, setup, 1st map, 1st reduce) - - for (TaskInfo task : tasks.values()) { - TaskID tid = task.getTaskId(); - - if (tid.equals(mapTaskId) || - tid.equals(reduceTaskId) || - tid.equals(cleanupTaskId) || - tid.equals(setupTaskId)) { - - TaskInProgress tip = jip.getTaskInProgress - (org.apache.hadoop.mapred.TaskID.downgrade(tid)); - assertTrue("START_TIME of Task " + tid + " obtained from history " + - "file did not match the expected value", - tip.getExecStartTime() == - task.getStartTime()); - - assertTrue("FINISH_TIME of Task " + tid + " obtained from history " + - "file did not match the expected value", - tip.getExecFinishTime() == - task.getFinishTime()); - - if (tid == mapTaskId) {//check splits only for map task - assertTrue("Splits of Task " + tid + " obtained from history file " + - " did not match the expected value", - tip.getSplitNodes().equals(task.getSplitLocations())); - } - - TaskAttemptID attemptId = tip.getSuccessfulTaskid(); - TaskStatus ts = tip.getTaskStatus( - org.apache.hadoop.mapred.TaskAttemptID.downgrade(attemptId)); - - // Validate task counters - Counters c = new Counters(ts.getCounters()); - assertTrue("Counters of Task " + tid + " obtained from history file " + - " did not match the expected value", - c.equals(task.getCounters())); - } - } - } - - // Validate Task Attempt Level Keys, Values read from history file by - // comparing them with the actual values from JT. - private static void validateTaskAttemptLevelKeyValues(MiniMRCluster mr, - RunningJob job, JobInfo jobInfo) throws IOException { - - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = jt.getJob(job.getID()); - - Map tasks = jobInfo.getAllTasks(); - - // For each task - for (TaskInfo task : tasks.values()) { - // validate info of each attempt - for (TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) { - - TaskAttemptID attemptId = attempt.getAttemptId(); - TaskID tid = attemptId.getTaskID(); - - TaskInProgress tip = jip.getTaskInProgress - (org.apache.hadoop.mapred.TaskID.downgrade(tid)); - - TaskStatus ts = tip.getTaskStatus( - org.apache.hadoop.mapred.TaskAttemptID.downgrade(attemptId)); - - // Validate task attempt start time - assertTrue("START_TIME of Task attempt " + attemptId + - " obtained from " + - "history file did not match the expected value", - ts.getStartTime() == attempt.getStartTime()); - - // Validate task attempt finish time - assertTrue("FINISH_TIME of Task attempt " + attemptId + - " obtained from " + - "history file " + ts.getFinishTime() + - " did not match the expected value, " + - attempt.getFinishTime(), - ts.getFinishTime() == attempt.getFinishTime()); - - - TaskTrackerStatus ttStatus = - jt.getTaskTrackerStatus(ts.getTaskTracker()); - - if (ttStatus != null) { - assertTrue("http port of task attempt " + attemptId + - " obtained from " + - "history file did not match the expected value", - ttStatus.getHttpPort() == - attempt.getHttpPort()); - - if (attempt.getTaskStatus().equals("SUCCEEDED")) { - Node node = jt.getNode(ttStatus.getHost()); - String ttHostname = node.getName(); - - // check if hostname is valid - assertTrue("Host name : " + attempt.getHostname() + " of task attempt " + attemptId + - " obtained from" + - " history file did not match the expected value " + ttHostname, - ttHostname.equals(attempt.getHostname())); - } - } - if (attempt.getTaskStatus().equals("SUCCEEDED")) { - // Validate SHUFFLE_FINISHED time and SORT_FINISHED time of - // Reduce Task Attempts - if (attempt.getTaskType().equals("REDUCE")) { - assertTrue("SHUFFLE_FINISHED time of task attempt " + attemptId + - " obtained from history file did not match the expected" + - " value", ts.getShuffleFinishTime() == - attempt.getShuffleFinishTime()); - assertTrue("SORT_FINISHED time of task attempt " + attemptId + - " obtained from history file did not match the expected" + - " value", ts.getSortFinishTime() == - attempt.getSortFinishTime()); - } - - //Validate task counters - Counters c = new Counters(ts.getCounters()); - assertTrue("Counters of Task Attempt " + attemptId + " obtained from " + - "history file did not match the expected value", - c.equals(attempt.getCounters())); - } - - // check if tracker name is valid - assertTrue("Tracker name of task attempt " + attemptId + - " obtained from " + - "history file did not match the expected value", - ts.getTaskTracker().equals(attempt.getTrackerName())); - } - } - } - - /** - * Checks if the history file content is as expected comparing with the - * actual values obtained from JT. - * Job Level, Task Level and Task Attempt Level Keys, Values are validated. - * @param job RunningJob object of the job whose history is to be validated - * @param conf job conf - */ - public static void validateJobHistoryFileContent(MiniMRCluster mr, - RunningJob job, JobConf conf) throws IOException { - - JobID id = job.getID(); - JobHistory jobHistory = - mr.getJobTrackerRunner().getJobTracker().getJobHistory(); - Path doneDir = jobHistory.getCompletedJobHistoryLocation(); - // Get the history file name - String logFileName = getDoneFile(jobHistory, conf, id, doneDir); - - // Framework history log file location - Path logFile = new Path(doneDir, logFileName); - FileSystem fileSys = logFile.getFileSystem(conf); - - // Check if the history file exists - assertTrue("History file does not exist", fileSys.exists(logFile)); - - JobHistoryParser parser = new JobHistoryParser(fileSys, - logFile.toUri().getPath()); - - JobHistoryParser.JobInfo jobInfo = parser.parse(); - // Now the history file contents are available in jobInfo. Let us compare - // them with the actual values from JT. - validateJobLevelKeyValues(mr, job, jobInfo, conf); - validateTaskLevelKeyValues(mr, job, jobInfo); - validateTaskAttemptLevelKeyValues(mr, job, jobInfo); - - // Also JobACLs should be correct - if (mr.getJobTrackerRunner().getJobTracker() - .areACLsEnabled()) { - AccessControlList acl = new AccessControlList( - conf.get(JobACL.VIEW_JOB.getAclName(), " ")); - assertTrue("VIEW_JOB ACL is not properly logged to history file.", - acl.toString().equals( - jobInfo.getJobACLs().get(JobACL.VIEW_JOB).toString())); - acl = new AccessControlList( - conf.get(JobACL.MODIFY_JOB.getAclName(), " ")); - assertTrue("MODIFY_JOB ACL is not properly logged to history file.", - acl.toString().equals( - jobInfo.getJobACLs().get(JobACL.MODIFY_JOB).toString())); - } - - // Validate the job queue name - assertTrue(jobInfo.getJobQueueName().equals(conf.getQueueName())); - } - - /** - * Tests the case where the log directory is on local disk, the done folder is on HDFS, - * and the default FS is local. - */ - public void testDoneFolderOnHDFS() throws IOException, InterruptedException { - runDoneFolderTest("history_done", LOCAL_LOG_DIR_URI); - } - - /** - * Tests the case where the log directory and done folder is on local disk - * and the default FS is local. - */ - public void testDoneFolderNotOnDefaultFileSystem() throws IOException, InterruptedException { - runDoneFolderTest(TEST_ROOT_DIR + "/history_done", LOCAL_LOG_DIR_URI); - } - - /** - * Tests the case where the log directory is on HDFS and done folder is on local disk - * and the default FS is local. - */ - public void testHistoryFolderOnHDFS() throws IOException, InterruptedException { - String logDir = "hdfs://localhost:%d/history"; - runDoneFolderTest(TEST_ROOT_DIR + "/done", logDir); - } - - private void runDoneFolderTest(String doneFolder, String historyFolder) throws IOException, InterruptedException { - MiniMRCluster mr = null; - MiniDFSCluster dfsCluster = null; - try { - JobConf conf = new JobConf(); - // keep for less time - conf.setLong("mapred.jobtracker.retirejob.check", 1000); - conf.setLong("mapred.jobtracker.retirejob.interval", 1000); - - //set the done folder location - conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder); - - dfsCluster = new MiniDFSCluster(conf, 2, true, null); - String logDir = String.format(historyFolder, dfsCluster.getNameNodePort()); - - //set the history folder location - conf.set(JTConfig.JT_JOBHISTORY_LOCATION, logDir); - - Path logDirPath = new Path(logDir); - FileSystem logDirFs = logDirPath.getFileSystem(conf); - //there may be some stale files, clean them - if (logDirFs.exists(logDirPath)) { - boolean deleted = logDirFs.delete(logDirPath, true); - LOG.info(logDirPath + " deleted " + deleted); - } - - logDirFs.mkdirs(logDirPath); - assertEquals("No of file in logDir not correct", 0, - logDirFs.listStatus(logDirPath).length); - logDirFs.create(new Path(logDirPath, "f1")); - logDirFs.create(new Path(logDirPath, "f2")); - assertEquals("No of file in logDir not correct", 2, - logDirFs.listStatus(logDirPath).length); - - mr = new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(), - 3, null, null, conf); - - assertEquals("Files in logDir did not move to DONE folder", - 0, logDirFs.listStatus(logDirPath).length); - - JobHistory jobHistory = - mr.getJobTrackerRunner().getJobTracker().getJobHistory(); - Path doneDir = jobHistory.getCompletedJobHistoryLocation(); - - assertEquals("Files in DONE dir not correct", - 2, doneDir.getFileSystem(conf).listStatus(doneDir).length); - - // run the TCs - conf = mr.createJobConf(); - - FileSystem fs = FileSystem.get(conf); - // clean up - fs.delete(new Path("succeed"), true); - - Path inDir = new Path("succeed/input"); - Path outDir = new Path("succeed/output"); - - //Disable speculative execution - conf.setSpeculativeExecution(false); - - // Make sure that the job is not removed from memory until we do finish - // the validation of history file content - conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10); - - // Run a job that will be succeeded and validate its history file - RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir); - - assertEquals("History DONE folder not correct", - new Path(doneFolder).getName(), doneDir.getName()); - JobID id = job.getID(); - String logFileName = getDoneFile(jobHistory, conf, id, doneDir); - - // Framework history log file location - Path logFile = new Path(doneDir, logFileName); - FileSystem fileSys = logFile.getFileSystem(conf); - - Cluster cluster = new Cluster(conf); - assertEquals("Client returned wrong history url", logFile.toString(), - cluster.getJobHistoryUrl(id)); - - // Check if the history file exists - assertTrue("History file does not exist", fileSys.exists(logFile)); - - // check if the corresponding conf file exists - Path confFile = getPathForConf(logFile, doneDir); - assertTrue("Config for completed jobs doesnt exist", - fileSys.exists(confFile)); - - // check if the file exists in a done folder - assertTrue("Completed job config doesnt exist in the done folder", - doneDir.getName().equals(confFile.getParent().getName())); - - // check if the file exists in a done folder - assertTrue("Completed jobs doesnt exist in the done folder", - doneDir.getName().equals(logFile.getParent().getName())); - - - // check if the job file is removed from the history location - Path runningJobsHistoryFolder = logFile.getParent().getParent(); - Path runningJobHistoryFilename = - new Path(runningJobsHistoryFolder, logFile.getName()); - Path runningJobConfFilename = - new Path(runningJobsHistoryFolder, confFile.getName()); - assertFalse("History file not deleted from the running folder", - fileSys.exists(runningJobHistoryFilename)); - assertFalse("Config for completed jobs not deleted from running folder", - fileSys.exists(runningJobConfFilename)); - - validateJobHistoryFileFormat(jobHistory, - job.getID(), conf, "SUCCEEDED", false); - validateJobHistoryFileContent(mr, job, conf); - - // get the job conf filename - } finally { - if (mr != null) { - cleanupLocalFiles(mr); - mr.shutdown(); - } - if (dfsCluster != null) { - dfsCluster.shutdown(); - } - } - } - - /** Run a job that will be succeeded and validate its history file format - * and its content. - */ - public void testJobHistoryFile() throws Exception { - MiniMRCluster mr = null; - try { - JobConf conf = new JobConf(); - - // keep for less time - conf.setLong("mapred.jobtracker.retirejob.check", 1000); - conf.setLong("mapred.jobtracker.retirejob.interval", 1000); - - //set the done folder location - String doneFolder = TEST_ROOT_DIR + "history_done"; - conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder); - - // Enable ACLs so that they are logged to history - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - - mr = new MiniMRCluster(2, "file:///", 3, null, null, conf); - - // run the TCs - conf = mr.createJobConf(); - - FileSystem fs = FileSystem.get(conf); - // clean up - fs.delete(new Path(TEST_ROOT_DIR + "/succeed"), true); - - Path inDir = new Path(TEST_ROOT_DIR + "/succeed/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/succeed/output"); - - //Disable speculative execution - conf.setSpeculativeExecution(false); - - // set the job acls - conf.set(JobACL.VIEW_JOB.getAclName(), "user1,user2 group1,group2"); - conf.set(JobACL.MODIFY_JOB.getAclName(), "user3,user4 group3,group4"); - - // Make sure that the job is not removed from memory until we do finish - // the validation of history file content - conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10); - - // Run a job that will be succeeded and validate its history file - RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir); - JobHistory jobHistory = - mr.getJobTrackerRunner().getJobTracker().getJobHistory(); - Path doneDir = jobHistory.getCompletedJobHistoryLocation(); - assertEquals("History DONE folder not correct", - doneFolder, doneDir.toString()); - JobID id = job.getID(); - String logFileName = getDoneFile(jobHistory, conf, id, doneDir); - - // Framework history log file location - Path logFile = new Path(doneDir, logFileName); - FileSystem fileSys = logFile.getFileSystem(conf); - - // Check if the history file exists - assertTrue("History file does not exist", fileSys.exists(logFile)); - - // check if the corresponding conf file exists - Path confFile = getPathForConf(logFile, doneDir); - assertTrue("Config for completed jobs doesnt exist", - fileSys.exists(confFile)); - - // check if the conf file exists in a done folder - assertTrue("Completed job config doesnt exist in the done folder", - doneDir.getName().equals(confFile.getParent().getName())); - - // check if the file exists in a done folder - assertTrue("Completed jobs doesnt exist in the done folder", - doneDir.getName().equals(logFile.getParent().getName())); - - // check if the job file is removed from the history location - Path runningJobsHistoryFolder = logFile.getParent().getParent(); - Path runningJobHistoryFilename = - new Path(runningJobsHistoryFolder, logFile.getName()); - Path runningJobConfFilename = - new Path(runningJobsHistoryFolder, confFile.getName()); - assertFalse("History file not deleted from the running folder", - fileSys.exists(runningJobHistoryFilename)); - assertFalse("Config for completed jobs not deleted from running folder", - fileSys.exists(runningJobConfFilename)); - - validateJobHistoryFileFormat(jobHistory, job.getID(), conf, - "SUCCEEDED", false); - validateJobHistoryFileContent(mr, job, conf); - - // get the job conf filename - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - String name = jt.getLocalJobFilePath(job.getID()); - File file = new File(name); - - // check if the file get deleted - while (file.exists()) { - LOG.info("Waiting for " + file + " to be deleted"); - UtilsForTests.waitFor(100); - } - } finally { - if (mr != null) { - cleanupLocalFiles(mr); - mr.shutdown(); - } - } - } - - //Returns the file in the done folder - //Waits for sometime to get the file moved to done - private static String getDoneFile(JobHistory jobHistory, - JobConf conf, JobID id, - Path doneDir) throws IOException { - String name = null; - String user = UserGroupInformation.getCurrentUser().getUserName(); - for (int i = 0; name == null && i < 20; i++) { - Path path = JobHistory.getJobHistoryFile( - jobHistory.getCompletedJobHistoryLocation(), id, user); - if (path.getFileSystem(conf).exists(path)) { - name = path.toString(); - } - UtilsForTests.waitFor(1000); - } - assertNotNull("Job history file not created", name); - return name; - } - - private void cleanupLocalFiles(MiniMRCluster mr) - throws IOException { - Configuration conf = mr.createJobConf(); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - Path sysDir = new Path(jt.getSystemDir()); - FileSystem fs = sysDir.getFileSystem(conf); - fs.delete(sysDir, true); - Path jobHistoryDir = - mr.getJobTrackerRunner().getJobTracker().getJobHistory(). - getJobHistoryLocation(); - fs = jobHistoryDir.getFileSystem(conf); - fs.delete(jobHistoryDir, true); - } - - /** - * Checks if the history file has expected job status - * @param id job id - * @param conf job conf - */ - private static void validateJobHistoryJobStatus(JobHistory jobHistory, - JobID id, JobConf conf, String status) throws IOException { - - // Get the history file name - Path doneDir = jobHistory.getCompletedJobHistoryLocation(); - String logFileName = getDoneFile(jobHistory, conf, id, doneDir); - - // Framework history log file location - Path logFile = new Path(doneDir, logFileName); - FileSystem fileSys = logFile.getFileSystem(conf); - - // Check if the history file exists - assertTrue("History file does not exist", fileSys.exists(logFile)); - - // check history file permission - assertTrue("History file permissions does not match", - fileSys.getFileStatus(logFile).getPermission().equals( - new FsPermission(JobHistory.HISTORY_FILE_PERMISSION))); - - JobHistoryParser parser = new JobHistoryParser(fileSys, - logFile.toUri().getPath()); - JobHistoryParser.JobInfo jobInfo = parser.parse(); - - - assertTrue("Job Status read from job history file is not the expected" + - " status", status.equals(jobInfo.getJobStatus())); - } - - // run jobs that will be (1) succeeded (2) failed (3) killed - // and validate job status read from history file in each case - public void testJobHistoryJobStatus() throws IOException { - MiniMRCluster mr = null; - try { - mr = new MiniMRCluster(2, "file:///", 3); - - // run the TCs - JobConf conf = mr.createJobConf(); - - FileSystem fs = FileSystem.get(conf); - // clean up - fs.delete(new Path(TEST_ROOT_DIR + "/succeedfailkilljob"), true); - - Path inDir = new Path(TEST_ROOT_DIR + "/succeedfailkilljob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/succeedfailkilljob/output"); - - // Run a job that will be succeeded and validate its job status - // existing in history file - RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir); - - JobHistory jobHistory = - mr.getJobTrackerRunner().getJobTracker().getJobHistory(); - validateJobHistoryJobStatus(jobHistory, job.getID(), conf, - JobStatus.getJobRunState(JobStatus.SUCCEEDED)); - - // Run a job that will be failed and validate its job status - // existing in history file - job = UtilsForTests.runJobFail(conf, inDir, outDir); - validateJobHistoryJobStatus(jobHistory, job.getID(), conf, - JobStatus.getJobRunState(JobStatus.FAILED)); - - // Run a job that will be killed and validate its job status - // existing in history file - job = UtilsForTests.runJobKill(conf, inDir, outDir); - validateJobHistoryJobStatus(jobHistory, job.getID(), conf, - JobStatus.getJobRunState(JobStatus.KILLED)); - - } finally { - if (mr != null) { - cleanupLocalFiles(mr); - mr.shutdown(); - } - } - } - - public void testHistoryInitWithCorruptFiles() throws IOException { - MiniMRCluster mr = null; - try { - JobConf conf = new JobConf(); - Path historyDir = new Path(System.getProperty("test.build.data", "."), - "history"); - conf.set(JTConfig.JT_JOBHISTORY_LOCATION, - historyDir.toString()); - conf.setUser("user"); - - FileSystem localFs = FileSystem.getLocal(conf); - - //there may be some stale files, clean them - if (localFs.exists(historyDir)) { - boolean deleted = localFs.delete(historyDir, true); - LOG.info(historyDir + " deleted " + deleted); - } - - // Start the cluster, create a history file - mr = new MiniMRCluster(0, "file:///", 3, null, null, conf); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobHistory jh = jt.getJobHistory(); - final JobID jobId = JobID.forName("job_200809171136_0001"); - jh.setupEventWriter(jobId, conf); - Map jobACLs = - new HashMap(); - JobSubmittedEvent jse = - new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs, - "default"); - jh.logEvent(jse, jobId); - jh.closeWriter(jobId); - - // Corrupt the history file. User RawLocalFileSystem so that we - // do keep the original CRC file intact. - String historyFileName = jobId.toString() + "_" + "user"; - Path historyFilePath = new Path (historyDir.toString(), historyFileName); - - RawLocalFileSystem fs = (RawLocalFileSystem) - FileSystem.getLocal(conf).getRaw(); - - FSDataOutputStream out = fs.create(historyFilePath, true); - byte[] corruptData = new byte[32]; - new Random().nextBytes(corruptData); - out.write (corruptData, 0, 32); - out.close(); - - // Stop and start the tracker. The tracker should come up nicely - mr.stopJobTracker(); - mr.startJobTracker(); - jt = mr.getJobTrackerRunner().getJobTracker(); - assertNotNull("JobTracker did not come up", jt ); - jh = jt.getJobHistory(); - assertNotNull("JobHistory did not get initialized correctly", jh); - - // Only the done folder should remain in the history directory - assertEquals("Files in logDir did not move to DONE folder", - 1, historyDir.getFileSystem(conf).listStatus(historyDir).length); - } finally { - if (mr != null) { - cleanupLocalFiles(mr); - mr.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java deleted file mode 100644 index 35abd48d124..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.JobACL; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent; -import org.apache.hadoop.mapreduce.jobhistory.JobHistory; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; -import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent; -import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent; -import org.apache.hadoop.security.authorize.AccessControlList; - -/** - * Unit test to test if the JobHistory writer/parser is able to handle - * values with special characters - * This test also tests if the job history module is able to gracefully - * ignore events after the event writer is closed - * - */ -public class TestJobHistoryParsing extends TestCase { - - public void testHistoryParsing() throws IOException { - // open a test history file - Path historyDir = new Path(System.getProperty("test.build.data", "."), - "history"); - JobConf conf = new JobConf(); - conf.set("hadoop.job.history.location", historyDir.toString()); - FileSystem fs = FileSystem.getLocal(new JobConf()); - - // Some weird strings - String username = "user"; - String weirdJob = "Value has \n new line \n and " + - "dot followed by new line .\n in it +" + - "ends with escape\\"; - String weirdPath = "Value has characters: " + - "`1234567890-=qwertyuiop[]\\asdfghjkl;'zxcvbnm,./" + - "~!@#$%^&*()_+QWERTYUIOP{}|ASDFGHJKL:\"'ZXCVBNM<>?" + - "\t\b\n\f\"\n in it"; - - String weirdJobQueueName = "my\njob\nQueue\\"; - conf.setUser(username); - - MiniMRCluster mr = null; - mr = new MiniMRCluster(2, "file:///", 3, null, null, conf); - - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobHistory jh = jt.getJobHistory(); - - jh.init(jt, conf, "localhost", 1234); - JobID jobId = JobID.forName("job_200809171136_0001"); - jh.setupEventWriter(jobId, conf); - Map jobACLs = - new HashMap(); - AccessControlList viewJobACL = - new AccessControlList("user1,user2 group1,group2"); - AccessControlList modifyJobACL = - new AccessControlList("user3,user4 group3, group4"); - jobACLs.put(JobACL.VIEW_JOB, viewJobACL); - jobACLs.put(JobACL.MODIFY_JOB, modifyJobACL); - JobSubmittedEvent jse = - new JobSubmittedEvent(jobId, weirdJob, username, 12345, weirdPath, - jobACLs, weirdJobQueueName); - jh.logEvent(jse, jobId); - - JobFinishedEvent jfe = - new JobFinishedEvent(jobId, 12346, 1, 1, 0, 0, new Counters(), - new Counters(), new Counters()); - jh.logEvent(jfe, jobId); - jh.closeWriter(jobId); - - // Try to write one more event now, should not fail - TaskID tid = TaskID.forName("task_200809171136_0001_m_000002"); - TaskFinishedEvent tfe = - new TaskFinishedEvent(tid, null, 0, TaskType.MAP, "", null); - boolean caughtException = false; - - try { - jh.logEvent(tfe, jobId); - } catch (Exception e) { - caughtException = true; - } - - assertFalse("Writing an event after closing event writer is not handled", - caughtException); - - String historyFileName = jobId.toString() + "_" + username; - Path historyFilePath = new Path (historyDir.toString(), - historyFileName); - - System.out.println("History File is " + historyFilePath.toString()); - - JobHistoryParser parser = - new JobHistoryParser(fs, historyFilePath); - - JobHistoryParser.JobInfo jobInfo = parser.parse(); - - assertTrue (jobInfo.getUsername().equals(username)); - assertTrue(jobInfo.getJobname().equals(weirdJob)); - assertTrue(jobInfo.getJobQueueName().equals(weirdJobQueueName)); - assertTrue(jobInfo.getJobConfPath().equals(weirdPath)); - Map parsedACLs = jobInfo.getJobACLs(); - assertEquals(2, parsedACLs.size()); - assertTrue(parsedACLs.get(JobACL.VIEW_JOB).toString().equals( - viewJobACL.toString())); - assertTrue(parsedACLs.get(JobACL.MODIFY_JOB).toString().equals( - modifyJobACL.toString())); - - if (mr != null) { - mr.shutdown(); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java deleted file mode 100644 index 6b16518e42a..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * TestJobInProgress is a unit test to test consistency of JobInProgress class - * data structures under different conditions (speculation/locality) and at - * different stages (tasks are running/pending/killed) - */ - -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.junit.Assert.*; -import org.junit.Test; -import org.junit.BeforeClass; -import static org.mockito.Mockito.*; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapred.TaskStatus.Phase; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.JobCounter; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; -import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.net.Node; -import org.apache.hadoop.net.NodeBase; -import org.apache.hadoop.net.StaticMapping; - -@SuppressWarnings("deprecation") -public class TestJobInProgress { - static final Log LOG = LogFactory.getLog(TestJobInProgress.class); - - static FakeJobTracker jobTracker; - - static String trackers[] = new String[] { - "tracker_tracker1.r1.com:1000", - "tracker_tracker2.r1.com:1000", - "tracker_tracker3.r2.com:1000", - "tracker_tracker4.r3.com:1000" - }; - - static String[] hosts = new String[] { - "tracker1.r1.com", - "tracker2.r1.com", - "tracker3.r2.com", - "tracker4.r3.com" - }; - - static String[] racks = new String[] { "/r1", "/r1", "/r2", "/r3" }; - - static int numUniqueHosts = hosts.length; - static int clusterSize = trackers.length; - - @BeforeClass - public static void setup() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setClass("topology.node.switch.mapping.impl", - StaticMapping.class, DNSToSwitchMapping.class); - jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); - // Set up the Topology Information - for (int i = 0; i < hosts.length; i++) { - StaticMapping.addNodeToRack(hosts[i], racks[i]); - } - for (String s: trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, s); - } - } - - static class MyFakeJobInProgress extends FakeJobInProgress { - - MyFakeJobInProgress(JobConf jc, JobTracker jt) throws IOException { - super(jc, jt); - } - - @Override - TaskSplitMetaInfo[] createSplits(org.apache.hadoop.mapreduce.JobID jobId) { - // Set all splits to reside on one host. This will ensure that - // one tracker gets data local, one gets rack local and two others - // get non-local maps - TaskSplitMetaInfo[] splits = new TaskSplitMetaInfo[numMapTasks]; - String[] splitHosts0 = new String[] { hosts[0] }; - for (int i = 0; i < numMapTasks; i++) { - splits[i] = new TaskSplitMetaInfo(splitHosts0, 0, 0); - } - return splits; - } - - private void makeRunning(TaskAttemptID taskId, TaskInProgress tip, - String taskTracker) { - TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, - 0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker, - tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters()); - updateTaskStatus(tip, status); - } - - private TaskInProgress getTipForTaskID(TaskAttemptID tid, boolean isMap) { - TaskInProgress result = null; - TaskID id = tid.getTaskID(); - TaskInProgress[] arrayToLook = isMap ? maps : reduces; - - for (int i = 0; i < arrayToLook.length; i++) { - TaskInProgress tip = arrayToLook[i]; - if (tip.getTIPId() == id) { - result = tip; - break; - } - } - return result; - } - - /** - * Find a new Map or a reduce task and mark it as running on the specified - * tracker - */ - public TaskAttemptID findAndRunNewTask(boolean isMap, - String tt, String host, - int clusterSize, - int numUniqueHosts) - throws IOException { - TaskTrackerStatus tts = new TaskTrackerStatus(tt, host); - Task task = isMap ? - obtainNewMapTask(tts, clusterSize, numUniqueHosts) : - obtainNewReduceTask(tts, clusterSize, numUniqueHosts); - TaskAttemptID tid = task.getTaskID(); - makeRunning(task.getTaskID(), getTipForTaskID(tid, isMap), tt); - return tid; - } - } - - //@Test - public void testPendingMapTaskCount() throws Exception { - - int numMaps = 4; - int numReds = 4; - - JobConf conf = new JobConf(); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - conf.setSpeculativeExecution(false); - conf.setBoolean( - JobContext.SETUP_CLEANUP_NEEDED, false); - MyFakeJobInProgress job1 = new MyFakeJobInProgress(conf, jobTracker); - job1.initTasks(); - - TaskAttemptID[] tid = new TaskAttemptID[numMaps]; - - for (int i = 0; i < numMaps; i++) { - tid[i] = job1.findAndRunNewTask(true, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - } - - // Fail all maps - for (int i = 0; i < numMaps; i++) { - job1.failTask(tid[i]); - } - - MyFakeJobInProgress job2 = new MyFakeJobInProgress(conf, jobTracker); - job2.initTasks(); - - for (int i = 0; i < numMaps; i++) { - tid[i] = job2.findAndRunNewTask(true, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - job2.finishTask(tid[i]); - } - - for (int i = 0; i < numReds/2; i++) { - tid[i] = job2.findAndRunNewTask(false, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - } - - for (int i = 0; i < numReds/4; i++) { - job2.finishTask(tid[i]); - } - - for (int i = numReds/4; i < numReds/2; i++) { - job2.failTask(tid[i]); - } - - // Job1. All Maps have failed, no reduces have been scheduled - checkTaskCounts(job1, 0, numMaps, 0, numReds); - - // Job2. All Maps have completed. One reducer has completed, one has - // failed and two others have not been scheduled - checkTaskCounts(job2, 0, 0, 0, 3 * numReds / 4); - } - - /** - * Test if running tasks are correctly maintained for various types of jobs - */ - static void testRunningTaskCount(boolean speculation) throws Exception { - LOG.info("Testing running jobs with speculation : " + speculation); - - JobConf conf = new JobConf(); - conf.setNumMapTasks(2); - conf.setNumReduceTasks(2); - conf.setSpeculativeExecution(speculation); - MyFakeJobInProgress jip = new MyFakeJobInProgress(conf, jobTracker); - jip.initTasks(); - - TaskAttemptID[] tid = new TaskAttemptID[4]; - - for (int i = 0; i < 2; i++) { - tid[i] = jip.findAndRunNewTask(true, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - } - - // check if the running structures are populated - Set uniqueTasks = new HashSet(); - for (Map.Entry> s : - jip.getRunningMapCache().entrySet()) { - uniqueTasks.addAll(s.getValue()); - } - - // add non local map tasks - uniqueTasks.addAll(jip.getNonLocalRunningMaps()); - - assertEquals("Running map count doesnt match for jobs with speculation " - + speculation, - jip.runningMaps(), uniqueTasks.size()); - - for (int i = 0; i < 2; i++ ) { - tid[i] = jip.findAndRunNewTask(false, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - } - - assertEquals("Running reducer count doesnt match for" + - " jobs with speculation " - + speculation, - jip.runningReduces(), jip.getRunningReduces().size()); - - } - - //@Test - public void testRunningTaskCount() throws Exception { - // test with spec = false - testRunningTaskCount(false); - - // test with spec = true - testRunningTaskCount(true); - - } - - static void checkTaskCounts(JobInProgress jip, int runningMaps, - int pendingMaps, int runningReduces, int pendingReduces) { - Counters counter = jip.getJobCounters(); - long totalTaskCount = counter.getCounter(JobCounter.TOTAL_LAUNCHED_MAPS) - + counter.getCounter(JobCounter.TOTAL_LAUNCHED_REDUCES); - - LOG.info("totalTaskCount is " + totalTaskCount); - LOG.info(" Running Maps:" + jip.runningMaps() + - " Pending Maps:" + jip.pendingMaps() + - " Running Reds:" + jip.runningReduces() + - " Pending Reds:" + jip.pendingReduces()); - - assertEquals(jip.getNumTaskCompletionEvents(),totalTaskCount); - assertEquals(runningMaps, jip.runningMaps()); - assertEquals(pendingMaps, jip.pendingMaps()); - assertEquals(runningReduces, jip.runningReduces()); - assertEquals(pendingReduces, jip.pendingReduces()); - } - - //@Test - public void testJobSummary() throws Exception { - int numMaps = 2; - int numReds = 2; - JobConf conf = new JobConf(); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - // Spying a fake is easier than mocking here - MyFakeJobInProgress jspy = spy(new MyFakeJobInProgress(conf, jobTracker)); - jspy.initTasks(); - TaskAttemptID tid; - - // Launch some map tasks - for (int i = 0; i < numMaps; i++) { - jspy.maps[i].setExecStartTime(i + 1); - tid = jspy.findAndRunNewTask(true, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - jspy.finishTask(tid); - } - - // Launch some reduce tasks - for (int i = 0; i < numReds; i++) { - jspy.reduces[i].setExecStartTime(i + numMaps + 1); - tid = jspy.findAndRunNewTask(false, trackers[i], hosts[i], - clusterSize, numUniqueHosts); - jspy.finishTask(tid); - } - - // Should be invoked numMaps + numReds times by different TIP objects - verify(jspy, times(4)).setFirstTaskLaunchTime(any(TaskInProgress.class)); - - ClusterStatus cspy = spy(new ClusterStatus(4, 0, 0, 0, 0, 4, 4, - JobTrackerStatus.RUNNING, 0)); - - JobInProgress.JobSummary.logJobSummary(jspy, cspy); - - verify(jspy).getStatus(); - verify(jspy).getProfile(); - verify(jspy, atLeastOnce()).getJobCounters(); - verify(jspy, atLeastOnce()).getJobID(); - verify(jspy).getStartTime(); - verify(jspy).getFirstTaskLaunchTimes(); - verify(jspy).getFinishTime(); - verify(jspy).getTasks(TaskType.MAP); - verify(jspy).getTasks(TaskType.REDUCE); - verify(jspy).getNumSlotsPerMap(); - verify(jspy).getNumSlotsPerReduce(); - verify(cspy).getMaxMapTasks(); - verify(cspy).getMaxReduceTasks(); - - assertEquals("firstMapTaskLaunchTime", 1, - jspy.getFirstTaskLaunchTimes().get(TaskType.MAP).longValue()); - assertEquals("firstReduceTaskLaunchTime", 3, - jspy.getFirstTaskLaunchTimes().get(TaskType.REDUCE).longValue()); - } - - @Test - public void testLocality() throws Exception { - NetworkTopology nt = new NetworkTopology(); - - Node r1n1 = new NodeBase("/default/rack1/node1"); - nt.add(r1n1); - Node r1n2 = new NodeBase("/default/rack1/node2"); - nt.add(r1n2); - - Node r2n3 = new NodeBase("/default/rack2/node3"); - nt.add(r2n3); - - LOG.debug("r1n1 parent: " + r1n1.getParent() + "\n" + - "r1n2 parent: " + r1n2.getParent() + "\n" + - "r2n3 parent: " + r2n3.getParent()); - - // Same host - assertEquals(0, JobInProgress.getMatchingLevelForNodes(r1n1, r1n1, 3)); - // Same rack - assertEquals(1, JobInProgress.getMatchingLevelForNodes(r1n1, r1n2, 3)); - // Different rack - assertEquals(2, JobInProgress.getMatchingLevelForNodes(r1n1, r2n3, 3)); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgressListener.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgressListener.java deleted file mode 100644 index 3c409cdbb07..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgressListener.java +++ /dev/null @@ -1,464 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.util.ArrayList; -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TestNoJobSetupCleanup; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Test whether the JobInProgressListeners are informed as expected. - */ -public class TestJobInProgressListener extends TestCase { - private static final Log LOG = - LogFactory.getLog(TestJobInProgressListener.class); - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).toURI().toString().replace(' ', '+'); - private final Path testDir = - new Path(TEST_ROOT_DIR, "test-jip-listener-update"); - private static MiniMRCluster mr; - private static JobTracker jobtracker; - private static JobConf conf; - private static MyScheduler myScheduler; - - public static Test suite() { - TestSetup setup = - new TestSetup(new TestSuite(TestJobInProgressListener.class)) { - @Override - protected void setUp() throws Exception { - conf = new JobConf(); - conf.setClass(JTConfig.JT_TASK_SCHEDULER, MyScheduler.class, - TaskScheduler.class); - mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); - jobtracker = mr.getJobTrackerRunner().getJobTracker(); - myScheduler = (MyScheduler)jobtracker.getScheduler(); - conf = mr.createJobConf(); - } - - @Override - protected void tearDown() throws Exception { - conf = null; - try { - mr.shutdown(); - } catch (Exception e) { - LOG.info("Error in shutting down the MR cluster", e); - } - jobtracker = null; - myScheduler.terminate(); - } - }; - return setup; - } - - /** - * This test case tests if external updates to JIP do not result into - * undesirable effects - * Test is as follows - * - submit 2 jobs of normal priority. job1 is a waiting job which waits and - * blocks the cluster - * - change one parameter of job2 such that the job bumps up in the queue - * - check if the queue looks ok - * - */ - public void testJobQueueChanges() throws IOException { - LOG.info("Testing job queue changes"); - - // stop the job initializer - myScheduler.stopInitializer(); - - JobQueueJobInProgressListener myListener = - new JobQueueJobInProgressListener(); - - // add the listener - jobtracker.addJobInProgressListener(myListener); - - Path inDir = new Path(testDir, "input"); - Path outputDir1 = new Path(testDir, "output1"); - Path outputDir2 = new Path(testDir, "output2"); - - RunningJob rJob1 = - UtilsForTests.runJob(conf, inDir, outputDir1, 1, 0); - LOG.info("Running job " + rJob1.getID().toString()); - - RunningJob rJob2 = - UtilsForTests.runJob(conf, inDir, outputDir2, 1, 0); - LOG.info("Running job " + rJob2.getID().toString()); - - // I. Check job-priority change - LOG.info("Testing job priority changes"); - - // bump up job2's priority - LOG.info("Increasing job2's priority to HIGH"); - rJob2.setJobPriority("HIGH"); - - // check if the queue is sane - assertTrue("Priority change garbles the queue", - myListener.getJobQueue().size() == 2); - - JobInProgress[] queue = - myListener.getJobQueue().toArray(new JobInProgress[0]); - - // check if the bump has happened - assertTrue("Priority change failed to bump up job2 in the queue", - queue[0].getJobID().equals(rJob2.getID())); - - assertTrue("Priority change failed to bump down job1 in the queue", - queue[1].getJobID().equals(rJob1.getID())); - - assertEquals("Priority change has garbled the queue", - 2, queue.length); - - // II. Check start-time change - LOG.info("Testing job start-time changes"); - - // reset the priority which will make the order as - // - job1 - // - job2 - // this will help in bumping job2 on start-time change - LOG.info("Increasing job2's priority to NORMAL"); - rJob2.setJobPriority("NORMAL"); - - // create the change event - JobInProgress jip2 = jobtracker.getJob(rJob2.getID()); - JobInProgress jip1 = jobtracker.getJob(rJob1.getID()); - - JobStatus prevStatus = (JobStatus)jip2.getStatus().clone(); - - // change job2's start-time and the status - jip2.startTime = jip1.startTime - 1; - jip2.status.setStartTime(jip2.startTime); - - - JobStatus newStatus = (JobStatus)jip2.getStatus().clone(); - - // inform the listener - LOG.info("Updating the listener about job2's start-time change"); - JobStatusChangeEvent event = - new JobStatusChangeEvent(jip2, EventType.START_TIME_CHANGED, - prevStatus, newStatus); - myListener.jobUpdated(event); - - // check if the queue is sane - assertTrue("Start time change garbles the queue", - myListener.getJobQueue().size() == 2); - - queue = myListener.getJobQueue().toArray(new JobInProgress[0]); - - // check if the bump has happened - assertTrue("Start time change failed to bump up job2 in the queue", - queue[0].getJobID().equals(rJob2.getID())); - - assertTrue("Start time change failed to bump down job1 in the queue", - queue[1].getJobID().equals(rJob1.getID())); - - assertEquals("Start time change has garbled the queue", - 2, queue.length); - } - - /** - * Check the queue status upon - * - failed job - * - killed job - * - successful job - */ - public void testJobCompletion() throws Exception { - MyListener mainListener = new MyListener(); - jobtracker.addJobInProgressListener(mainListener); - - // stop the job initializer - myScheduler.stopInitializer(); - - // check queued jobs - testQueuedJobKill(conf, mainListener); - - myScheduler.startInitializer(); - - // check the queue state for job states - testFailedJob(conf, mainListener); - - testKilledJob(conf, mainListener); - - testSuccessfulJob(conf, mainListener); - } - - // A listener that inits the tasks one at a time and also listens to the - // events - public static class MyListener extends JobInProgressListener { - private List wjobs = new ArrayList(); - private List rjobs = new ArrayList(); - // list of job added to the wait queue - private List wjobsAdded = new ArrayList(); - // list of job added to the running queue - private List rjobsAdded = new ArrayList(); - - public boolean contains (JobID id) { - return contains(id, true) || contains(id, false); - } - - public boolean contains (JobID id, boolean waiting) { - if (!wjobsAdded.contains(id)) { - throw new RuntimeException("Job " + id + " not seen in waiting queue"); - } - if (!waiting) { - if (!rjobsAdded.contains(id)) { - throw new RuntimeException("Job " + id + " not seen in run queue"); - } - } - List queue = waiting ? wjobs : rjobs; - for (JobInProgress job : queue) { - if (job.getJobID().equals(id)) { - return true; - } - } - return false; - } - - public void jobAdded(JobInProgress job) { - LOG.info("Job " + job.getJobID().toString() + " added"); - wjobs.add(job); - wjobsAdded.add(job.getJobID()); - } - - public void jobRemoved(JobInProgress job) { - LOG.info("Job " + job.getJobID().toString() + " removed"); - wjobs.remove(job); - rjobs.remove(job); - } - - public void jobUpdated(JobChangeEvent event) { - LOG.info("Job " + event.getJobInProgress().getJobID().toString() + " updated"); - // remove the job is the event is for a completed job - if (event instanceof JobStatusChangeEvent) { - JobStatusChangeEvent statusEvent = (JobStatusChangeEvent)event; - if (statusEvent.getEventType() == EventType.RUN_STATE_CHANGED) { - // check if the state changes from - // RUNNING->COMPLETE(SUCCESS/KILLED/FAILED) - JobInProgress jip = event.getJobInProgress(); - String jobId = jip.getJobID().toString(); - if (jip.isComplete()) { - LOG.info("Job " + jobId + " deleted from the running queue"); - if (statusEvent.getOldStatus().getRunState() == JobStatus.PREP) { - wjobs.remove(jip); - } else { - rjobs.remove(jip); - } - } else { - // PREP->RUNNING - LOG.info("Job " + jobId + " deleted from the waiting queue"); - wjobs.remove(jip); - rjobs.add(jip); - rjobsAdded.add(jip.getJobID()); - } - } - } - } - } - - private void testFailedJob(JobConf job, MyListener myListener) - throws IOException { - LOG.info("Testing job-fail"); - - Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerfailjob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerfailjob/output"); - - job.setNumMapTasks(1); - job.setNumReduceTasks(0); - job.setMaxMapAttempts(1); - - // submit a job that fails - RunningJob rJob = UtilsForTests.runJobFail(job, inDir, outDir); - JobID id = rJob.getID(); - - // check if the job failure was notified - assertFalse("Missing event notification on failing a running job", - myListener.contains(id)); - - // check if failed - assertEquals("Job failed!", JobStatus.FAILED, rJob.getJobState()); - } - - private void testKilledJob(JobConf job, MyListener myListener) - throws IOException { - LOG.info("Testing job-kill"); - - Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerkilljob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerkilljob/output"); - - job.setNumMapTasks(1); - job.setNumReduceTasks(0); - - // submit and kill the job - RunningJob rJob = UtilsForTests.runJobKill(job, inDir, outDir); - JobID id = rJob.getID(); - - // check if the job failure was notified - assertFalse("Missing event notification on killing a running job", - myListener.contains(id)); - - // check if killed - assertEquals("Job failed!", JobStatus.KILLED, rJob.getJobState()); - } - - private void testSuccessfulJob(JobConf job, MyListener myListener) - throws Exception { - LOG.info("Testing job-success"); - - Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output"); - - job.setNumMapTasks(1); - job.setNumReduceTasks(0); - - // submit the job - RunningJob rJob = UtilsForTests.runJobSucceed(job, inDir, outDir); - - // wait for the job to be successful - rJob.waitForCompletion(); - - // check if the job success was notified - assertFalse("Missing event notification for a successful job", - myListener.contains(rJob.getID())); - - // check if successful - assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState()); - - // test if 0-task jobs with setup-cleanup works fine - LOG.info("Testing job with no task job with setup and cleanup"); - - job.setNumMapTasks(0); - job.setNumReduceTasks(0); - - outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks"); - - // submit the job - rJob = UtilsForTests.runJobSucceed(job, inDir, outDir); - - // wait for the job to be successful - rJob.waitForCompletion(); - - // check if the job success was notified - assertFalse("Missing event notification for a successful job with no tasks", - myListener.contains(rJob.getID(), true)); - - // check if successful - assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState()); - - // test if jobs with no tasks (0 maps, 0 red) update the listener properly - LOG.info("Testing job with no-set-cleanup no task"); - - outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks-no-set"); - - Job j = MapReduceTestUtil.createJob(mr.createJobConf(), inDir, outDir, 0, 0); - j.setJobSetupCleanupNeeded(false); - j.setOutputFormatClass(TestNoJobSetupCleanup.MyOutputFormat.class); - j.submit(); - j.waitForCompletion(true); - - JobID id = JobID.downgrade(j.getJobID()); - - // check if the job is in the waiting queue - assertFalse("Missing event notification on no-set-cleanup no task job", - myListener.contains(id, true)); - - // check if the job is successful - assertEquals("Job status doesnt reflect success", - JobStatus.SUCCEEDED, rJob.getJobState()); - } - - /** - * This scheduler never schedules any task as it doesnt init any task. So all - * the jobs are queued forever. - */ - public static class MyScheduler extends JobQueueTaskScheduler { - - @Override - public synchronized void start() throws IOException { - super.start(); - } - - void stopInitializer() throws IOException { - // Remove the eager task initializer - taskTrackerManager.removeJobInProgressListener( - eagerTaskInitializationListener); - // terminate it - eagerTaskInitializationListener.terminate(); - } - - void startInitializer() throws IOException { - eagerTaskInitializationListener = - new EagerTaskInitializationListener(getConf()); - eagerTaskInitializationListener.setTaskTrackerManager(taskTrackerManager); - // start it - eagerTaskInitializationListener.start(); - // add the eager task initializer - taskTrackerManager.addJobInProgressListener( - eagerTaskInitializationListener); - } - } - - private void testQueuedJobKill(JobConf conf, MyListener myListener) - throws IOException { - LOG.info("Testing queued-job-kill"); - - Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerqueuedjob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/jiplistener1ueuedjob/output"); - - conf.setMapperClass(IdentityMapper.class); - conf.setReducerClass(IdentityReducer.class); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(0); - RunningJob rJob = UtilsForTests.runJob(conf, inDir, outDir); - JobID id = rJob.getID(); - LOG.info("Job : " + id.toString() + " submitted"); - - // check if the job is in the waiting queue - assertTrue("Missing event notification on submiting a job", - myListener.contains(id, true)); - - // kill the job - LOG.info("Killing job : " + id.toString()); - rJob.killJob(); - - // check if the job is killed - assertEquals("Job status doesnt reflect the kill-job action", - JobStatus.KILLED, rJob.getJobState()); - - // check if the job is correctly moved - // from the waiting list - assertFalse("Missing event notification on killing a waiting job", - myListener.contains(id, true)); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobKillAndFail.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobKillAndFail.java deleted file mode 100644 index 410207ee15e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobKillAndFail.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.File; -import java.io.InputStreamReader; -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.SleepJob; - -/** - * A JUnit test to test Kill Job & Fail Job functionality with local file - * system. - */ -public class TestJobKillAndFail extends TestCase { - - static final Log LOG = LogFactory.getLog(TestJobKillAndFail.class); - - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).toURI().toString().replace(' ', '+'); - - /** - * TaskController instance that just sets a flag when a stack dump - * is performed in a child thread. - */ - static class MockStackDumpTaskController extends DefaultTaskController { - - static volatile int numStackDumps = 0; - - static final Log LOG = LogFactory.getLog(TestJobKillAndFail.class); - - public MockStackDumpTaskController() { - LOG.info("Instantiated MockStackDumpTC"); - } - - @Override - void dumpTaskStack(TaskControllerContext context) { - LOG.info("Got stack-dump request in TaskController"); - MockStackDumpTaskController.numStackDumps++; - super.dumpTaskStack(context); - } - - } - - /** If a task was killed, then dumpTaskStack() should have been - * called. Test whether or not the counter was incremented - * and succeed/fail based on this. */ - private void checkForStackDump(boolean expectDump, int lastNumDumps) { - int curNumDumps = MockStackDumpTaskController.numStackDumps; - - LOG.info("curNumDumps=" + curNumDumps + "; lastNumDumps=" + lastNumDumps - + "; expect=" + expectDump); - - if (expectDump) { - assertTrue("No stack dump recorded!", lastNumDumps < curNumDumps); - } else { - assertTrue("Stack dump happened anyway!", lastNumDumps == curNumDumps); - } - } - - public void testJobFailAndKill() throws Exception { - MiniMRCluster mr = null; - try { - JobConf jtConf = new JobConf(); - jtConf.set("mapred.jobtracker.instrumentation", - JTInstrumentation.class.getName()); - jtConf.set("mapreduce.tasktracker.taskcontroller", - MockStackDumpTaskController.class.getName()); - mr = new MiniMRCluster(2, "file:///", 3, null, null, jtConf); - JTInstrumentation instr = (JTInstrumentation) - mr.getJobTrackerRunner().getJobTracker().getInstrumentation(); - - // run the TCs - JobConf conf = mr.createJobConf(); - conf.setInt(Job.COMPLETION_POLL_INTERVAL_KEY, 50); - - Path inDir = new Path(TEST_ROOT_DIR + "/failkilljob/input"); - Path outDir = new Path(TEST_ROOT_DIR + "/failkilljob/output"); - RunningJob runningJob = UtilsForTests.runJobFail(conf, inDir, outDir); - // Checking that the Job got failed - assertEquals(runningJob.getJobState(), JobStatus.FAILED); - assertTrue(instr.verifyJob()); - assertEquals(1, instr.failed); - instr.reset(); - - int prevNumDumps = MockStackDumpTaskController.numStackDumps; - runningJob = UtilsForTests.runJobKill(conf, inDir, outDir); - // Checking that the Job got killed - assertTrue(runningJob.isComplete()); - assertEquals(runningJob.getJobState(), JobStatus.KILLED); - assertTrue(instr.verifyJob()); - assertEquals(1, instr.killed); - // check that job kill does not put a stacktrace in task logs. - checkForStackDump(false, prevNumDumps); - - // Test that a task that times out does have a stack trace - conf = mr.createJobConf(); - conf.setInt(JobContext.TASK_TIMEOUT, 10000); - conf.setInt(Job.COMPLETION_POLL_INTERVAL_KEY, 50); - SleepJob sleepJob = new SleepJob(); - sleepJob.setConf(conf); - Job job = sleepJob.createJob(1, 0, 30000, 1,0, 0); - job.setMaxMapAttempts(1); - prevNumDumps = MockStackDumpTaskController.numStackDumps; - job.waitForCompletion(true); - checkForStackDump(true, prevNumDumps); - } finally { - if (mr != null) { - mr.shutdown(); - } - } - } - - static class JTInstrumentation extends JobTrackerInstrumentation { - volatile int failed; - volatile int killed; - volatile int addPrep; - volatile int decPrep; - volatile int addRunning; - volatile int decRunning; - - void reset() { - failed = 0; - killed = 0; - addPrep = 0; - decPrep = 0; - addRunning = 0; - decRunning = 0; - } - - boolean verifyJob() { - return addPrep==1 && decPrep==1 && addRunning==1 && decRunning==1; - } - - public JTInstrumentation(JobTracker jt, JobConf conf) { - super(jt, conf); - } - - public synchronized void addPrepJob(JobConf conf, JobID id) - { - addPrep++; - } - - public synchronized void decPrepJob(JobConf conf, JobID id) - { - decPrep++; - } - - public synchronized void addRunningJob(JobConf conf, JobID id) - { - addRunning++; - } - - public synchronized void decRunningJob(JobConf conf, JobID id) - { - decRunning++; - } - - public synchronized void failedJob(JobConf conf, JobID id) - { - failed++; - } - - public synchronized void killedJob(JobConf conf, JobID id) - { - killed++; - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueClient.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueClient.java deleted file mode 100644 index d25a165f435..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueClient.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.apache.hadoop.mapred.QueueManagerTestUtils.QUEUES_CONFIG_FILE_PATH; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createDocument; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createSimpleDocumentWithAcls; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.miniMRCluster; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.deleteQueuesConfigFile; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.writeToFile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; - -import junit.framework.Assert; - -import org.apache.hadoop.mapreduce.QueueInfo; -import org.junit.After; -import org.junit.Test; -import org.w3c.dom.Document; - -public class TestJobQueueClient { - - @After - public void tearDown() throws Exception { - deleteQueuesConfigFile(); - } - - @Test - public void testQueueOrdering() throws Exception { - // create some sample queues in a hierarchy.. - JobQueueInfo[] roots = new JobQueueInfo[2]; - roots[0] = new JobQueueInfo("q1", "q1 scheduling info"); - roots[1] = new JobQueueInfo("q2", "q2 scheduling info"); - - List children = new ArrayList(); - children.add(new JobQueueInfo("q1:1", null)); - children.add(new JobQueueInfo("q1:2", null)); - roots[0].setChildren(children); - - // test dfs ordering - JobQueueClient client = new JobQueueClient(new JobConf()); - List allQueues = client.expandQueueList(roots); - assertEquals(4, allQueues.size()); - assertEquals("q1", allQueues.get(0).getQueueName()); - assertEquals("q1:1", allQueues.get(1).getQueueName()); - assertEquals("q1:2", allQueues.get(2).getQueueName()); - assertEquals("q2", allQueues.get(3).getQueueName()); - } - - @Test - public void testQueueInfoPrinting() throws Exception { - // create a test queue with children. - // create some sample queues in a hierarchy.. - JobQueueInfo root = new JobQueueInfo("q1", "q1 scheduling info"); - - List children = new ArrayList(); - children.add(new JobQueueInfo("q1:1", null)); - children.add(new JobQueueInfo("q1:2", null)); - root.setChildren(children); - - JobQueueClient client = new JobQueueClient(new JobConf()); - StringWriter writer = new StringWriter(); - client.printJobQueueInfo(root, writer); - - Assert.assertTrue(writer.toString().contains("Queue Name : q1")); - Assert.assertTrue(writer.toString().contains("Queue State : running")); - Assert.assertTrue(writer.toString().contains("Scheduling Info : q1 scheduling info")); - Assert.assertTrue(writer.toString().contains("Queue Name : q1:1")); - Assert.assertTrue(writer.toString().contains("Queue Name : q1:2")); - } - - @Test - public void testGetQueue() throws Exception { - - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocumentWithAcls(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - JobConf jobConf = new JobConf(); - String namenode = "file:///"; - miniMRCluster = new MiniMRCluster(0, namenode, 3, null, null, jobConf); - - JobClient jc = new JobClient(miniMRCluster.createJobConf()); - // test for existing queue - QueueInfo queueInfo = jc.getQueueInfo("q1"); - assertEquals("q1",queueInfo.getQueueName()); - // try getting a non-existing queue - queueInfo = jc.getQueueInfo("queue"); - assertNull(queueInfo); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java deleted file mode 100644 index ea2980c4244..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; - -import junit.framework.TestCase; - -public class TestJobQueueInformation extends TestCase { - - private MiniMRCluster mrCluster; - private MiniDFSCluster dfsCluster; - private JobConf jc; - private static final String JOB_SCHEDULING_INFO = "TESTSCHEDULINGINFO"; - private static final Path TEST_DIR = - new Path(System.getProperty("test.build.data","/tmp"), - "job-queue-info-testing"); - private static final Path IN_DIR = new Path(TEST_DIR, "input"); - private static final Path SHARE_DIR = new Path(TEST_DIR, "share"); - private static final Path OUTPUT_DIR = new Path(TEST_DIR, "output"); - - static String getSignalFile() { - return (new Path(SHARE_DIR, "signal")).toString(); - } - - // configure a waiting job with 2 maps - private JobConf configureWaitingJob(JobConf conf) throws IOException { - - UtilsForTests.configureWaitingJobConf(conf, IN_DIR, OUTPUT_DIR, 2, 0, - "test-job-queue-info", getSignalFile(), getSignalFile()); - return conf; - } - - public static class TestTaskScheduler extends LimitTasksPerJobTaskScheduler { - - @Override - public synchronized List assignTasks(TaskTracker taskTracker) - throws IOException { - Collection jips = jobQueueJobInProgressListener - .getJobQueue(); - if (jips != null && !jips.isEmpty()) { - for (JobInProgress jip : jips) { - jip.setSchedulingInfo(JOB_SCHEDULING_INFO); - } - } - return super.assignTasks(taskTracker); - } - } - - @Override - protected void setUp() throws Exception { - super.setUp(); - final int taskTrackers = 4; - Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, 4, true, null); - - jc = new JobConf(); - jc.setClass(JTConfig.JT_TASK_SCHEDULER, TestTaskScheduler.class, - TaskScheduler.class); - jc.setLong(JTConfig.JT_RUNNINGTASKS_PER_JOB, 10L); - mrCluster = new MiniMRCluster(0, 0, taskTrackers, dfsCluster - .getFileSystem().getUri().toString(), 1, null, null, null, jc); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - mrCluster.shutdown(); - dfsCluster.shutdown(); - } - - public void testJobQueues() throws Exception { - JobClient jc = new JobClient(mrCluster.createJobConf()); - String expectedQueueInfo = "Maximum Tasks Per Job :: 10"; - JobQueueInfo[] queueInfos = jc.getQueues(); - assertNotNull(queueInfos); - assertEquals(1, queueInfos.length); - assertEquals("default", queueInfos[0].getQueueName()); - assertEquals(QueueState.RUNNING.getStateName(), - queueInfos[0].getQueueState()); - JobConf conf = mrCluster.createJobConf(); - FileSystem fileSys = dfsCluster.getFileSystem(); - - // configure a waiting job - conf = configureWaitingJob(conf); - conf.setJobName("test-job-queue-info-test"); - - // clear the signal file if any - fileSys.delete(SHARE_DIR, true); - - RunningJob rJob = jc.submitJob(conf); - - while (rJob.getJobState() != JobStatus.RUNNING) { - UtilsForTests.waitFor(10); - } - - int numberOfJobs = 0; - - for (JobQueueInfo queueInfo : queueInfos) { - JobStatus[] jobStatusList = jc.getJobsFromQueue(queueInfo - .getQueueName()); - assertNotNull(queueInfo.getQueueName()); - assertNotNull(queueInfo.getSchedulingInfo()); - assertEquals(expectedQueueInfo, queueInfo.getSchedulingInfo()); - numberOfJobs += jobStatusList.length; - for (JobStatus status : jobStatusList) { - assertEquals(JOB_SCHEDULING_INFO, status.getSchedulingInfo()); - } - } - assertEquals(1, numberOfJobs); - - UtilsForTests.signalTasks(dfsCluster, fileSys, getSignalFile(), - getSignalFile(), 4); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java deleted file mode 100644 index dd297b3f5c0..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java +++ /dev/null @@ -1,343 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; -import org.apache.hadoop.mapreduce.split.JobSplit; - -public class TestJobQueueTaskScheduler extends TestCase { - - private static int jobCounter; - private static int taskCounter; - - static void resetCounters() { - jobCounter = 0; - taskCounter = 0; - } - - static class FakeJobInProgress extends JobInProgress { - - private FakeTaskTrackerManager taskTrackerManager; - - public FakeJobInProgress(JobConf jobConf, - FakeTaskTrackerManager taskTrackerManager, JobTracker jt) - throws IOException { - super(new JobID("test", ++jobCounter), jobConf, jt); - this.taskTrackerManager = taskTrackerManager; - this.startTime = System.currentTimeMillis(); - this.status = new JobStatus(getJobID(), 0f, 0f, JobStatus.PREP, - jobConf.getUser(), - jobConf.getJobName(), "", ""); - this.status.setJobPriority(JobPriority.NORMAL); - this.status.setStartTime(startTime); - } - - @Override - public synchronized void initTasks() throws IOException { - // do nothing - } - - @Override - public Task obtainNewLocalMapTask(TaskTrackerStatus tts, int clusterSize, - int ignored) - throws IOException { - return obtainNewMapTask(tts, clusterSize, ignored); - } - - @Override - public Task obtainNewNonLocalMapTask(TaskTrackerStatus tts, int clusterSize, - int ignored) - throws IOException { - return obtainNewMapTask(tts, clusterSize, ignored); - } - - @Override - public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize, - int ignored) throws IOException { - TaskAttemptID attemptId = getTaskAttemptID(TaskType.MAP); - Task task = new MapTask("", attemptId, 0, new JobSplit.TaskSplitIndex(), 1) { - @Override - public String toString() { - return String.format("%s on %s", getTaskID(), tts.getTrackerName()); - } - }; - taskTrackerManager.update(tts.getTrackerName(), task); - runningMapTasks++; - return task; - } - - @Override - public Task obtainNewReduceTask(final TaskTrackerStatus tts, - int clusterSize, int ignored) throws IOException { - TaskAttemptID attemptId = getTaskAttemptID(TaskType.REDUCE); - Task task = new ReduceTask("", attemptId, 0, 10, 1) { - @Override - public String toString() { - return String.format("%s on %s", getTaskID(), tts.getTrackerName()); - } - }; - taskTrackerManager.update(tts.getTrackerName(), task); - runningReduceTasks++; - return task; - } - - private TaskAttemptID getTaskAttemptID(TaskType type) { - JobID jobId = getJobID(); - return new TaskAttemptID(jobId.getJtIdentifier(), - jobId.getId(), type, ++taskCounter, 0); - } - } - - static class FakeTaskTrackerManager implements TaskTrackerManager { - - int maps = 0; - int reduces = 0; - int maxMapTasksPerTracker = 2; - int maxReduceTasksPerTracker = 2; - List listeners = - new ArrayList(); - QueueManager queueManager; - - private Map trackers = - new HashMap(); - - public FakeTaskTrackerManager() { - JobConf conf = new JobConf(); - queueManager = new QueueManager(conf); - - TaskTracker tt1 = new TaskTracker("tt1"); - tt1.setStatus(new TaskTrackerStatus("tt1", "tt1.host", 1, - new ArrayList(), 0, - maxMapTasksPerTracker, - maxReduceTasksPerTracker)); - trackers.put("tt1", tt1); - - TaskTracker tt2 = new TaskTracker("tt2"); - tt2.setStatus(new TaskTrackerStatus("tt2", "tt2.host", 2, - new ArrayList(), 0, - maxMapTasksPerTracker, - maxReduceTasksPerTracker)); - trackers.put("tt2", tt2); - } - - @Override - public ClusterStatus getClusterStatus() { - int numTrackers = trackers.size(); - return new ClusterStatus(numTrackers, 0, - 10 * 60 * 1000, - maps, reduces, - numTrackers * maxMapTasksPerTracker, - numTrackers * maxReduceTasksPerTracker, - JobTrackerStatus.RUNNING); - } - - @Override - public int getNumberOfUniqueHosts() { - return 0; - } - - @Override - public Collection taskTrackers() { - List statuses = new ArrayList(); - for (TaskTracker tt : trackers.values()) { - statuses.add(tt.getStatus()); - } - return statuses; - } - - - @Override - public void addJobInProgressListener(JobInProgressListener listener) { - listeners.add(listener); - } - - @Override - public void removeJobInProgressListener(JobInProgressListener listener) { - listeners.remove(listener); - } - - @Override - public QueueManager getQueueManager() { - return queueManager; - } - - @Override - public int getNextHeartbeatInterval() { - return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT; - } - - @Override - public void killJob(JobID jobid) { - return; - } - - @Override - public JobInProgress getJob(JobID jobid) { - return null; - } - - @Override - public boolean killTask(TaskAttemptID attemptId, boolean shouldFail) { - return true; - } - - public void initJob(JobInProgress job) { - // do nothing - } - - public void failJob(JobInProgress job) { - // do nothing - } - - // Test methods - - public void submitJob(JobInProgress job) throws IOException { - for (JobInProgressListener listener : listeners) { - listener.jobAdded(job); - } - } - - public TaskTracker getTaskTracker(String trackerID) { - return trackers.get(trackerID); - } - - public void update(String taskTrackerName, final Task t) { - if (t.isMapTask()) { - maps++; - } else { - reduces++; - } - TaskStatus status = new TaskStatus() { - @Override - public boolean getIsMap() { - return t.isMapTask(); - } - - @Override - public void addFetchFailedMap(TaskAttemptID mapTaskId) { - - } - }; - status.setRunState(TaskStatus.State.RUNNING); - trackers.get(taskTrackerName).getStatus().getTaskReports().add(status); - } - - } - - protected JobConf jobConf; - protected TaskScheduler scheduler; - private FakeTaskTrackerManager taskTrackerManager; - - @Override - protected void setUp() throws Exception { - resetCounters(); - jobConf = new JobConf(); - jobConf.setNumMapTasks(10); - jobConf.setNumReduceTasks(10); - taskTrackerManager = new FakeTaskTrackerManager(); - scheduler = createTaskScheduler(); - scheduler.setConf(jobConf); - scheduler.setTaskTrackerManager(taskTrackerManager); - scheduler.start(); - } - - @Override - protected void tearDown() throws Exception { - if (scheduler != null) { - scheduler.terminate(); - } - } - - protected TaskScheduler createTaskScheduler() { - return new JobQueueTaskScheduler(); - } - - static void submitJobs(FakeTaskTrackerManager taskTrackerManager, JobConf jobConf, - int numJobs, int state) - throws IOException { - for (int i = 0; i < numJobs; i++) { - JobInProgress job = new FakeJobInProgress(jobConf, taskTrackerManager, - UtilsForTests.getJobTracker()); - job.getStatus().setRunState(state); - taskTrackerManager.submitJob(job); - } - } - - public void testTaskNotAssignedWhenNoJobsArePresent() throws IOException { - assertEquals(0, scheduler.assignTasks(tracker(taskTrackerManager, "tt1")).size()); - } - - public void testNonRunningJobsAreIgnored() throws IOException { - submitJobs(taskTrackerManager, jobConf, 1, JobStatus.PREP); - submitJobs(taskTrackerManager, jobConf, 1, JobStatus.SUCCEEDED); - submitJobs(taskTrackerManager, jobConf, 1, JobStatus.FAILED); - submitJobs(taskTrackerManager, jobConf, 1, JobStatus.KILLED); - assertEquals(0, scheduler.assignTasks(tracker(taskTrackerManager, "tt1")).size()); - } - - public void testDefaultTaskAssignment() throws IOException { - submitJobs(taskTrackerManager, jobConf, 2, JobStatus.RUNNING); - // All slots are filled with job 1 - checkAssignment(scheduler, tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_m_000001_0 on tt1", - "attempt_test_0001_m_000002_0 on tt1", - "attempt_test_0001_r_000003_0 on tt1"}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_r_000004_0 on tt1"}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt1"), new String[] {}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0001_m_000005_0 on tt2", - "attempt_test_0001_m_000006_0 on tt2", - "attempt_test_0001_r_000007_0 on tt2"}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0001_r_000008_0 on tt2"}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt2"), new String[] {}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt1"), new String[] {}); - checkAssignment(scheduler, tracker(taskTrackerManager, "tt2"), new String[] {}); - } - - static TaskTracker tracker(FakeTaskTrackerManager taskTrackerManager, - String taskTrackerName) { - return taskTrackerManager.getTaskTracker(taskTrackerName); - } - - static void checkAssignment(TaskScheduler scheduler, TaskTracker taskTracker, - String[] expectedTaskStrings) throws IOException { - List tasks = scheduler.assignTasks(taskTracker); - assertNotNull(tasks); - assertEquals(expectedTaskStrings.length, tasks.size()); - for (int i=0; i < expectedTaskStrings.length; ++i) { - assertEquals(expectedTaskStrings[i], tasks.get(i).toString()); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java deleted file mode 100644 index ac68a1577b0..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.MiniMRCluster.TaskTrackerRunner; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.TaskID; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.mapreduce.split.JobSplit; - -/** - * Test if the job retire works fine. - */ -public class TestJobRetire extends TestCase { - static final Log LOG = LogFactory.getLog(TestJobRetire.class); - static final Path testDir = - new Path(System.getProperty("test.build.data","/tmp"), - "job-expiry-testing"); - - private MiniMRCluster startCluster(JobConf conf, int numTrackers) - throws IOException { - conf.setBoolean(JTConfig.JT_RETIREJOBS, true); - conf.setLong(JTConfig.JT_RETIREJOB_CACHE_SIZE, 1); - return new MiniMRCluster(0, 0, numTrackers, "file:///", 1, null, null, null, - conf, 0); - } - - public void testJobRetire() throws Exception { - MiniMRCluster mr = null; - try { - JobConf conf = new JobConf(); - mr = startCluster(conf, 1); - - JobConf jobConf = mr.createJobConf(); - JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - Path inDir = new Path(testDir, "input1"); - Path outDir = new Path(testDir, "output1"); - - JobID id1 = validateJobRetire(jobConf, inDir, outDir, jobtracker); - - outDir = new Path(testDir, "output2"); - JobID id2 = validateJobRetire(jobConf, inDir, outDir, jobtracker); - - assertNull("Job not removed from cache", jobtracker.getJobStatus(id1)); - - assertEquals("Total job in cache not correct", - 1, jobtracker.getAllJobs().length); - } finally { - if (mr != null) { mr.shutdown();} - FileUtil.fullyDelete(new File(testDir.toString())); - } - } - - private JobID validateJobRetire(JobConf jobConf, Path inDir, Path outDir, - JobTracker jobtracker) throws IOException { - - RunningJob rj = UtilsForTests.runJob(jobConf, inDir, outDir, 0, 0); - rj.waitForCompletion(); - assertTrue(rj.isSuccessful()); - JobID id = rj.getID(); - - //wait for job to get retired - waitTillRetire(id, jobtracker); - - assertTrue("History url not set", rj.getHistoryUrl() != null && - rj.getHistoryUrl().length() > 0); - assertNotNull("Job is not in cache", jobtracker.getJobStatus(id)); - - // get the job conf filename - String name = jobtracker.getLocalJobFilePath(id); - File file = new File(name); - - assertFalse("JobConf file not deleted", file.exists()); - - // test redirections - final String JOBDETAILS = "jobdetails"; - final String JOBCONF = "jobconf"; - final String JOBTASKS = "jobtasks"; - final String TASKSTATS = "taskstats"; - final String TASKDETAILS = "taskdetails"; - - // test redirections of job related pages - String jobUrlStr = rj.getTrackingURL(); - URL jobUrl = new URL(jobUrlStr); - URL jobConfUrl = new URL(jobUrlStr.replace(JOBDETAILS, JOBCONF)); - URL jobTasksUrl = new URL(jobUrlStr.replace(JOBDETAILS, JOBTASKS) - + "&type=map&pagenum=1"); - verifyRedirection(jobConfUrl); - verifyRedirection(jobTasksUrl); - verifyRedirection(jobUrl); - - // test redirections of task and task attempt pages - String jobTrackerUrlStr = - jobUrlStr.substring(0, jobUrlStr.indexOf(JOBDETAILS)); - Path logFile = new Path(jobtracker.getJobHistory().getHistoryFilePath(id)); - JobHistoryParser.JobInfo jobInfo = - JSPUtil.getJobInfo(logFile, logFile.getFileSystem(jobConf), jobtracker); - for (TaskID tid : jobInfo.getAllTasks().keySet()) { - URL taskDetailsUrl = new URL(jobTrackerUrlStr + TASKDETAILS + - ".jsp?tipid=" + tid); - // test redirections of all tasks - verifyRedirection(taskDetailsUrl); - } - for (JobHistoryParser.TaskInfo task : jobInfo.getAllTasks().values()) { - for(org.apache.hadoop.mapreduce.TaskAttemptID attemptid : - task.getAllTaskAttempts().keySet()) { - URL taskstats = new URL(jobTrackerUrlStr + TASKSTATS + - ".jsp?attemptid=" + attemptid); - // test redirections of all task attempts - verifyRedirection(taskstats); - } - } - return id; - } - - private void verifyRedirection(URL url) throws IOException { - LOG.info("Verifying redirection of " + url); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setInstanceFollowRedirects(false); - conn.connect(); - assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, conn.getResponseCode()); - conn.disconnect(); - URL redirectedUrl = new URL(conn.getHeaderField("Location")); - conn = (HttpURLConnection) redirectedUrl.openConnection(); - conn.connect(); - assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - conn.disconnect(); - } - - // wait till the job retires - private void waitTillRetire(JobID id, JobTracker jobtracker) { - JobInProgress job = jobtracker.getJob(id); - //wait for job to get retired - for (int i = 0; i < 10 && job != null; i++) { - UtilsForTests.waitFor(1000); - job = jobtracker.getJob(id); - } - assertNull("Job did not retire", job); - } - - /** - * Custom TaskTracker which waits forever after a successful contact to - * the JobTracker. - */ - class WaitingTaskTracker extends TaskTracker { - - private volatile boolean alive = true; - - WaitingTaskTracker(JobConf conf) throws IOException, InterruptedException { - super(conf); - } - - @Override - HeartbeatResponse transmitHeartBeat(long now) throws IOException { - HeartbeatResponse response = super.transmitHeartBeat(now); - LOG.info("WaitingTaskTracker waiting"); - // wait forever - while (alive) { - UtilsForTests.waitFor(1000); - } - throw new IOException ("WaitingTaskTracker shutdown. Bailing out"); - } - - @Override - public synchronized void shutdown() throws IOException { - alive = false; - super.shutdown(); - } - } - - /** - * Test job retire with tasks that report their *first* status only after the - * job retires. - * Steps : - * - Start a mini-mr cluster with 1 task-tracker having only map slots. - * Note that this task-tracker will take care of setup/cleanup and map - * tasks. - * - Submit a job with 1 map task and 1 reduce task - * - Wait for the job to finish the map task - * - Start a 2nd tracker that waits for a long time after contacting the JT. - * - Wait for the 2nd tracker to get stuck - * - Kill the job - * - Wait for the job to retire - * - Check if the tip mappings are cleaned up. - */ - public void testJobRetireWithUnreportedTasks() throws Exception { - MiniMRCluster mr = null; - try { - JobConf conf = new JobConf(); - conf.setInt(TTConfig.TT_MAP_SLOTS, 1); - conf.setInt(TTConfig.TT_REDUCE_SLOTS, 0); - mr = startCluster(conf, 1); - JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - // submit a job - Path inDir = new Path(testDir, "in-1"); - Path outDir = new Path(testDir, "out-1"); - JobConf jConf = mr.createJobConf(); - FileInputFormat.setInputPaths(jConf, new Path[] {inDir}); - FileOutputFormat.setOutputPath(jConf, outDir); - SleepJob sleepJob = new SleepJob(); - sleepJob.setConf(jConf); - Job job = sleepJob.createJob(1, 1, 0, 1, 0, 1); - - job.submit(); - JobID id = JobID.downgrade(job.getStatus().getJobID()); - JobInProgress jip = jobtracker.getJob(id); - - // wait 100 secs for the map to complete - for (int i = 0; i < 100 && (jip.finishedMaps() < 1); i++) { - UtilsForTests.waitFor(1000); - } - assertEquals(jip.finishedMaps(), 1); - - // start a tracker that will wait - LOG.info("Adding a waiting tracker"); - TaskTrackerRunner testTrackerRunner = - mr.new TaskTrackerRunner(1, 1, null, mr.createJobConf()) { - @Override - TaskTracker createTaskTracker(JobConf conf) - throws IOException, InterruptedException { - return new WaitingTaskTracker(conf); - } - }; - mr.addTaskTracker(testTrackerRunner); - LOG.info("Waiting tracker added"); - - WaitingTaskTracker testTT = - (WaitingTaskTracker)testTrackerRunner.getTaskTracker(); - - // wait 100 secs for the newly started task-tracker to join - for (int i = 0; i < 1000 && (jobtracker.taskTrackers().size() < 2); i++) { - UtilsForTests.waitFor(100); - } - assertEquals(jobtracker.taskTrackers().size(), 2); - LOG.info("Cluster is now ready"); - - // stop the test-tt as its no longer required - mr.stopTaskTracker(mr.getTaskTrackerID(testTT.getName())); - - // check if a reduce task got scheduled or not - assertEquals("Waiting tracker joined but no reduce task got scheduled", - 1, jip.runningReduces()); - - // kill the job - job.killJob(); - - // check if the reduce task attempt status is missing - TaskInProgress tip = jip.getTasks(TaskType.REDUCE)[0]; - assertNull(tip.getTaskStatus(tip.getAllTaskAttemptIDs()[0])); - - // wait for the job to retire - waitTillRetire(id, jobtracker); - - // check the taskidToTIPMap - for (TaskAttemptID tid : jobtracker.taskidToTIPMap.keySet()) { - LOG.info("TaskidToTIP mapping left over : " + tid); - } - assertEquals("'taskid' to TIP mapping still exists", - 0, jobtracker.taskidToTIPMap.size()); - } finally { - if (mr != null) { mr.shutdown(); } - FileUtil.fullyDelete(new File(testDir.toString())); - } - } - - /** - * (Mock)Test JobTracker.removeJobTasks() which is called only when the job - * retires. - */ - public void testJobRemoval() throws Exception { - MiniMRCluster mr = null; - try { - JobConf conf = new JobConf(); - mr = startCluster(conf, 0); - JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - // test map task removal - testRemoveJobTasks(jobtracker, conf, TaskType.MAP); - // test reduce task removal - testRemoveJobTasks(jobtracker, conf, TaskType.REDUCE); - // test job setup removal - testRemoveJobTasks(jobtracker, conf, TaskType.JOB_SETUP); - // test job cleanup removal - testRemoveJobTasks(jobtracker, conf, TaskType.JOB_CLEANUP); - } finally { - if (mr != null) { mr.shutdown();} - // cleanup - FileUtil.fullyDelete(new File(testDir.toString())); - } - } - - // create a new job and add it to the jobtracker - private JobInProgress createAndAddJob(JobTracker jobtracker, JobConf conf) { - // submit a job in a fake manner - // get the new job-id - JobID id = - new JobID(jobtracker.getTrackerIdentifier(), jobtracker.jobs.size() + 1); - // create a JobInProgress for this fake job - JobInProgress jip = new JobInProgress(id, conf, jobtracker); - - // insert this fake completed job in the jobtracker - jobtracker.jobs.put(id, jip); - - return jip; - } - - // create a new TaskInProgress and make it running by adding it to jobtracker - private TaskInProgress createAndAddTIP(JobTracker jobtracker, - JobInProgress jip, TaskType type) { - JobConf conf = jip.getJobConf(); - JobID id = jip.getJobID(); - // now create a fake tip for this fake job - TaskInProgress tip = null; - if (type == TaskType.MAP) { - tip = new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, - jobtracker, conf, jip, 0, 1); - jip.maps = new TaskInProgress[] {tip}; - } else if (type == TaskType.REDUCE) { - tip = new TaskInProgress(id, "dummy", jip.desiredMaps(), 0, - jobtracker, conf, jip, 1); - jip.reduces = new TaskInProgress[] {tip}; - } else if (type == TaskType.JOB_SETUP) { - tip = - new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, - jobtracker, conf, jip, 0, 1); - jip.setup = new TaskInProgress[] {tip}; - } else if (type == TaskType.JOB_CLEANUP) { - tip = - new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, - jobtracker, conf, jip, 0, 1); - jip.cleanup = new TaskInProgress[] {tip}; - } - return tip; - } - - // create a new Task for the given tip and make it running - private TaskAttemptID createAndAddAttempt(TaskInProgress tip, int attemptId) { - // create a fake attempt for this fake task - TaskAttemptID taskid = new TaskAttemptID(tip.getTIPId(), attemptId); - - // insert this fake task into the jobtracker by making it running - tip.addRunningTask(taskid, "test-tt"); - - return taskid; - } - - // Mock a job run such that the jobtracker is in a state similar to that - // resulting from an actual job run. - // Steps : - // - generate a new job-id - // - create and add a JobInProgress object using the fake job-id - // - create and add a fake tip of the passed type 't' under the fake job - // Note that t can be a MAP or a REDUCE or a JOB_SETUP or a JOB_CLEANUP. - // - create and add a fake attempt under the fake tip - // - remove the job from the jobtracker - // - check if the fake attempt is removed from the jobtracker - private void testRemoveJobTasks(JobTracker jobtracker, JobConf conf, - TaskType type) { - // create and submit a job - JobInProgress jip = createAndAddJob(jobtracker, conf); - // create and add a tip - TaskInProgress tip = createAndAddTIP(jobtracker, jip, type); - // create and add an attempt - TaskAttemptID taskid = createAndAddAttempt(tip, 0); - - // this fake attempt should not have any status - assertNull(tip.getTaskStatus(taskid)); - - // remove the job tasks for this fake job from the jobtracker - jobtracker.removeJobTasks(jip); - - // check the taskidToTIPMap - for (TaskAttemptID tid : jobtracker.taskidToTIPMap.keySet()) { - LOG.info("TaskidToTIP : " + tid); - } - - // check if the fake attempt is removed from the jobtracker - assertEquals("'taskid' to TIP mapping still exists", - 0, jobtracker.taskidToTIPMap.size()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobStatusPersistency.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobStatusPersistency.java deleted file mode 100644 index 9a51ae91cb4..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobStatusPersistency.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.io.IOException; -import java.util.Properties; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -public class TestJobStatusPersistency extends ClusterMapReduceTestCase { - static final Path TEST_DIR = - new Path(System.getProperty("test.build.data","/tmp"), - "job-status-persistence"); - - @Override - protected void setUp() throws Exception { - // Don't start anything by default - } - - private JobID runJob() throws Exception { - OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); - Writer wr = new OutputStreamWriter(os); - wr.write("hello1\n"); - wr.write("hello2\n"); - wr.write("hello3\n"); - wr.write("hello4\n"); - wr.close(); - - JobConf conf = createJobConf(); - conf.setJobName("mr"); - - conf.setInputFormat(TextInputFormat.class); - - conf.setMapOutputKeyClass(LongWritable.class); - conf.setMapOutputValueClass(Text.class); - - conf.setOutputFormat(TextOutputFormat.class); - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - - conf.setMapperClass(org.apache.hadoop.mapred.lib.IdentityMapper.class); - conf.setReducerClass(org.apache.hadoop.mapred.lib.IdentityReducer.class); - - FileInputFormat.setInputPaths(conf, getInputDir()); - - FileOutputFormat.setOutputPath(conf, getOutputDir()); - - return JobClient.runJob(conf).getID(); - } - - public void testNonPersistency() throws Exception { - startCluster(true, null); - JobID jobId = runJob(); - JobClient jc = new JobClient(createJobConf()); - RunningJob rj = jc.getJob(jobId); - assertNotNull(rj); - stopCluster(); - startCluster(false, null); - jc = new JobClient(createJobConf()); - rj = jc.getJob(jobId); - assertNull(rj); - } - - public void testPersistency() throws Exception { - Properties config = new Properties(); - config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "true"); - config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); - startCluster(true, config); - JobID jobId = runJob(); - JobClient jc = new JobClient(createJobConf()); - RunningJob rj0 = jc.getJob(jobId); - assertNotNull(rj0); - boolean sucessfull0 = rj0.isSuccessful(); - String jobName0 = rj0.getJobName(); - Counters counters0 = rj0.getCounters(); - TaskCompletionEvent[] events0 = rj0.getTaskCompletionEvents(0); - - stopCluster(); - startCluster(false, config); - - jc = new JobClient(createJobConf()); - RunningJob rj1 = jc.getJob(jobId); - assertNotNull(rj1); - assertEquals(sucessfull0, rj1.isSuccessful()); - assertEquals(jobName0, rj0.getJobName()); - assertEquals(counters0.size(), rj1.getCounters().size()); - - TaskCompletionEvent[] events1 = rj1.getTaskCompletionEvents(0); - assertEquals(events0.length, events1.length); - for (int i = 0; i < events0.length; i++) { - assertEquals(events0[i].getTaskAttemptId(), events1[i].getTaskAttemptId()); - assertEquals(events0[i].getTaskStatus(), events1[i].getTaskStatus()); - } - } - - /** - * Test if the completed job status is persisted to localfs. - */ - public void testLocalPersistency() throws Exception { - FileSystem fs = FileSystem.getLocal(new JobConf()); - - fs.delete(TEST_DIR, true); - - Properties config = new Properties(); - config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "true"); - config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); - config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_DIR, - fs.makeQualified(TEST_DIR).toString()); - startCluster(true, config); - JobID jobId = runJob(); - JobClient jc = new JobClient(createJobConf()); - RunningJob rj = jc.getJob(jobId); - assertNotNull(rj); - - // check if the local fs has the data - Path jobInfo = new Path(TEST_DIR, rj.getID() + ".info"); - assertTrue("Missing job info from the local fs", fs.exists(jobInfo)); - fs.delete(TEST_DIR, true); - } - - /** - * Verify that completed-job store is inactive if the jobinfo path is not - * writable. - * - * @throws Exception - */ - public void testJobStoreDisablingWithInvalidPath() throws Exception { - MiniMRCluster mr = null; - Path parent = new Path(TEST_DIR, "parent"); - try { - FileSystem fs = FileSystem.getLocal(new JobConf()); - - if (fs.exists(TEST_DIR) && !fs.delete(TEST_DIR, true)) { - fail("Cannot delete TEST_DIR!"); - } - - if (fs.mkdirs(new Path(TEST_DIR, parent))) { - if (!(new File(parent.toUri().getPath()).setWritable(false, false))) { - fail("Cannot chmod parent!"); - } - } else { - fail("Cannot create parent dir!"); - } - JobConf config = new JobConf(); - config.set(JTConfig.JT_PERSIST_JOBSTATUS, "true"); - config.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); - config.set(JTConfig.JT_PERSIST_JOBSTATUS_DIR, new Path(parent, - "child").toUri().getPath()); - boolean started = true; - JobConf conf = MiniMRCluster.configureJobConf(config, "file:///", 0, 0, null); - try { - JobTracker jt = JobTracker.startTracker(conf); - } catch (IOException ex) { - started = false; - } - assertFalse(started); - } finally { - new File(parent.toUri().getPath()).setWritable(true, false); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java deleted file mode 100644 index 0262060ea2b..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java +++ /dev/null @@ -1,394 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTrackerMetricsInst; -import org.apache.hadoop.mapred.TestTaskTrackerBlacklisting.FakeJobTracker; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; - -@SuppressWarnings("deprecation") -public class TestJobTrackerInstrumentation extends TestCase { - - static String trackers[] = new String[] { - "tracker_tracker1:1000", - "tracker_tracker2:1000", - "tracker_tracker3:1000" }; - - static String hosts[] = new String[] { "tracker1", "tracker2", "tracker3" }; - // heartbeat responseId. increment this after sending a heartbeat - private static short responseId = 1; - - private static FakeJobTracker jobTracker; - private static FakeJobInProgress fakeJob; - - private static int mapSlotsPerTracker = 4; - private static int reduceSlotsPerTracker = 2; - - private static int numMapSlotsToReserve = 2; - private static int numReduceSlotsToReserve = 2; - - private static FakeJobTrackerMetricsInst mi; - - - - public static Test suite() { - TestSetup setup = - new TestSetup(new TestSuite(TestJobTrackerInstrumentation.class)) { - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1); - conf.setClass(JTConfig.JT_TASK_SCHEDULER, - FakeTaskScheduler.class, TaskScheduler.class); - - conf.set(JTConfig.JT_INSTRUMENTATION, - FakeJobTrackerMetricsInst.class.getName()); - jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); - mi = (FakeJobTrackerMetricsInst) jobTracker.getInstrumentation(); - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - - } - protected void tearDown() throws Exception { - } - }; - return setup; - } - - private TaskTrackerStatus getTTStatus(String trackerName, - List taskStatuses) { - return new TaskTrackerStatus(trackerName, - JobInProgress.convertTrackerNameToHostName(trackerName), 0, - taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker); - } - - public void testMetrics() throws Exception { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[3]; - - // create TaskTrackerStatus and send heartbeats - TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length]; - status[0] = getTTStatus(trackers[0], new ArrayList()); - status[1] = getTTStatus(trackers[1], new ArrayList()); - status[2] = getTTStatus(trackers[2], new ArrayList()); - for (int i = 0; i< trackers.length; i++) { - FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false, - false, trackers[i], responseId); - } - responseId++; - - assertEquals("Mismatch in number of trackers", - trackers.length, mi.numTrackers); - - int numMaps = 2; - int numReds = 1; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - conf.setMaxTaskFailuresPerTracker(1); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - assertTrue(mi.numJobsPreparing == 1); - - job.setClusterSize(trackers.length); - job.initTasks(); - jobTracker.addJob(job.getJobID(), job); - - taskAttemptID[0] = job.findMapTask(trackers[0]); - taskAttemptID[1] = job.findMapTask(trackers[1]); - taskAttemptID[2] = job.findReduceTask(trackers[2]); - - job.finishTask(taskAttemptID[0]); - job.finishTask(taskAttemptID[1]); - job.finishTask(taskAttemptID[2]); - jobTracker.finalizeJob(job); - - assertTrue("Mismatch in map tasks launched", - mi.numMapTasksLaunched == numMaps); - assertTrue("Mismatch in map tasks completed", - mi.numMapTasksCompleted == numMaps); - assertTrue("Mismatch in map tasks failed", - mi.numMapTasksFailed == 0); - - assertTrue("Mismatch in reduce tasks launched", - mi.numReduceTasksLaunched == numReds); - assertTrue("Mismatch in reduce tasks completed", - mi.numReduceTasksCompleted == numReds); - assertTrue("Mismatch in reduce tasks failed", - mi.numReduceTasksFailed == 0); - - assertTrue("Mismatch in num Jobs submitted", - mi.numJobsSubmitted == 1); - - assertTrue("Mismatch in num map slots", - mi.numMapSlots == (mapSlotsPerTracker * trackers.length)); - assertTrue("Mismatch in num reduce slots", - mi.numReduceSlots == (reduceSlotsPerTracker * trackers.length)); - - assertTrue("No heartbeats were recorded, but at least one was sent.", - mi.numHeartbeats > 0); - } - - public void testBlackListing() throws IOException { - int numMaps, numReds; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setMaxTaskFailuresPerTracker(1); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - TaskAttemptID[] taskAttemptID = new TaskAttemptID[3]; - - numMaps = 1; - numReds = 1; - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - - FakeJobInProgress job1 = new FakeJobInProgress(conf, jobTracker); - job1.setClusterSize(trackers.length); - job1.initTasks(); - jobTracker.addJob(job1.getJobID(), job1); - taskAttemptID[0] = job1.findMapTask(trackers[0]); - job1.failTask(taskAttemptID[0]); - taskAttemptID[1] = job1.findMapTask(trackers[1]); - job1.finishTask(taskAttemptID[1]); - taskAttemptID[2] = job1.findReduceTask(trackers[0]); - job1.failTask(taskAttemptID[2]); - taskAttemptID[2] = job1.findReduceTask(trackers[2]); - job1.finishTask(taskAttemptID[2]); - jobTracker.finalizeJob(job1); - - assertEquals("Mismatch in number of failed map tasks", - 1, mi.numMapTasksFailed); - assertEquals("Mismatch in number of failed reduce tasks", - 1, mi.numReduceTasksFailed); - - assertEquals("Mismatch in number of blacklisted trackers", - 1, mi.numTrackersBlackListed); - - assertEquals("Mismatch in blacklisted map slots", - mi.numBlackListedMapSlots, - (mapSlotsPerTracker * mi.numTrackersBlackListed)); - - assertEquals("Mismatch in blacklisted reduce slots", - mi.numBlackListedReduceSlots, - (reduceSlotsPerTracker * mi.numTrackersBlackListed)); - } - - public void testOccupiedSlotCounts() throws Exception { - - TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length]; - - List list = new ArrayList(); - - // create a map task status, which uses 2 slots. - int mapSlotsPerTask = 2; - TaskStatus ts = TaskStatus.createTaskStatus(true, - new TaskAttemptID("jt", 1, TaskType.MAP, 0, 0), 0.0f, mapSlotsPerTask, - TaskStatus.State.RUNNING, "", "", trackers[0], - TaskStatus.Phase.MAP, null); - list.add(ts); - int mapSlotsPerTask1 = 1; - ts = TaskStatus.createTaskStatus(true, - new TaskAttemptID("jt", 1, TaskType.MAP, 0, 0), 0.0f, mapSlotsPerTask1, - TaskStatus.State.RUNNING, "", "", trackers[0], - TaskStatus.Phase.MAP, null); - list.add(ts); - - // create a reduce task status, which uses 3 slot. - int reduceSlotsPerTask = 3; - ts = TaskStatus.createTaskStatus(false, - new TaskAttemptID("jt", 1, TaskType.REDUCE, 0, 0), 0.0f, - reduceSlotsPerTask, - TaskStatus.State.RUNNING, "", "", trackers[0], - TaskStatus.Phase.REDUCE, null); - list.add(ts); - int reduceSlotsPerTask1 = 1; - ts = TaskStatus.createTaskStatus(false, - new TaskAttemptID("jt", 1, TaskType.REDUCE, 0, 0), 0.0f, - reduceSlotsPerTask1, - TaskStatus.State.RUNNING, "", "", trackers[0], - TaskStatus.Phase.REDUCE, null); - list.add(ts); - - // create TaskTrackerStatus and send heartbeats - status = new TaskTrackerStatus[trackers.length]; - status[0] = getTTStatus(trackers[0], list); - status[1] = getTTStatus(trackers[1], new ArrayList()); - status[2] = getTTStatus(trackers[2], new ArrayList()); - for (int i = 0; i< trackers.length; i++) { - FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false, - false, trackers[i], responseId); - } - responseId++; - - assertEquals("Mismatch in map slots occupied", - mapSlotsPerTask+mapSlotsPerTask1, mi.numOccupiedMapSlots); - assertEquals("Mismatch in reduce slots occupied", - reduceSlotsPerTask+reduceSlotsPerTask1, mi.numOccupiedReduceSlots); - assertEquals("Mismatch in num running maps", - 2, mi.numRunningMaps); - assertEquals("Mismatch in num running reduces", - 2, mi.numRunningReduces); - - //now send heartbeat with no running tasks - status = new TaskTrackerStatus[1]; - status[0] = getTTStatus(trackers[0], new ArrayList()); - FakeObjectUtilities.sendHeartBeat(jobTracker, status[0], false, - false, trackers[0], responseId); - - assertEquals("Mismatch in map slots occupied", - 0, mi.numOccupiedMapSlots); - assertEquals("Mismatch in reduce slots occupied", - 0, mi.numOccupiedReduceSlots); - assertEquals("Mismatch in num running maps", - 0, mi.numRunningMaps); - assertEquals("Mismatch in num running reduces", - 0, mi.numRunningReduces); - } - - public void testReservedSlots() throws IOException { - JobConf conf = new JobConf(); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(1); - conf.setSpeculativeExecution(false); - - //Set task tracker objects for reservation. - TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); - TaskTrackerStatus status2 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - tt2.setStatus(status2); - - fakeJob = new FakeJobInProgress(conf, jobTracker); - fakeJob.setClusterSize(3); - fakeJob.initTasks(); - - FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false, - true, trackers[1], responseId); - responseId++; - - assertEquals("Mismtach in reserved map slots", - numMapSlotsToReserve, mi.numReservedMapSlots); - assertEquals("Mismtach in reserved red slots", - numReduceSlotsToReserve, mi.numReservedReduceSlots); - } - - public void testDecomissionedTrackers() throws IOException { - // create TaskTrackerStatus and send heartbeats - TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length]; - status[0] = getTTStatus(trackers[0], new ArrayList()); - status[1] = getTTStatus(trackers[1], new ArrayList()); - status[2] = getTTStatus(trackers[2], new ArrayList()); - for (int i = 0; i< trackers.length; i++) { - FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false, - false, trackers[i], responseId); - } - - assertEquals("Mismatch in number of trackers", - trackers.length, mi.numTrackers); - Set dHosts = new HashSet(); - dHosts.add(hosts[1]); - assertEquals("Mismatch in number of decommissioned trackers", - 0, mi.numTrackersDecommissioned); - jobTracker.decommissionNodes(dHosts); - assertEquals("Mismatch in number of decommissioned trackers", - 1, mi.numTrackersDecommissioned); - assertEquals("Mismatch in number of trackers", - trackers.length - 1, mi.numTrackers); - } - - public void testKillTasks() throws IOException { - int numMaps, numReds; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setMaxTaskFailuresPerTracker(1); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - TaskAttemptID[] taskAttemptID = new TaskAttemptID[2]; - - numMaps = 1; - numReds = 1; - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - - assertEquals("Mismatch in number of killed map tasks", - 0, mi.numMapTasksKilled); - assertEquals("Mismatch in number of killed reduce tasks", - 0, mi.numReduceTasksKilled); - - FakeJobInProgress job1 = new FakeJobInProgress(conf, jobTracker); - job1.setClusterSize(trackers.length); - job1.initTasks(); - jobTracker.addJob(job1.getJobID(), job1); - taskAttemptID[0] = job1.findMapTask(trackers[0]); - job1.killTask(taskAttemptID[0]); - taskAttemptID[1] = job1.findReduceTask(trackers[0]); - job1.killTask(taskAttemptID[1]); - jobTracker.finalizeJob(job1); - - assertEquals("Mismatch in number of killed map tasks", - 1, mi.numMapTasksKilled); - assertEquals("Mismatch in number of killed reduce tasks", - 1, mi.numReduceTasksKilled); - } - - static class FakeTaskScheduler extends JobQueueTaskScheduler { - public FakeTaskScheduler() { - super(); - } - public List assignTasks(TaskTracker tt) { - tt.reserveSlots(TaskType.MAP, fakeJob, numMapSlotsToReserve); - tt.reserveSlots(TaskType.REDUCE, fakeJob, numReduceSlotsToReserve); - return new ArrayList(); - } - } - - static class FakeJobInProgress extends - org.apache.hadoop.mapred.TestTaskTrackerBlacklisting.FakeJobInProgress { - - FakeJobInProgress(JobConf jobConf, JobTracker tracker) throws IOException { - super(jobConf, tracker); - } - - @Override - public synchronized void initTasks() throws IOException { - super.initTasks(); - jobtracker.getInstrumentation().addWaitingMaps(getJobID(), - numMapTasks); - jobtracker.getInstrumentation().addWaitingReduces(getJobID(), - numReduceTasks); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java deleted file mode 100644 index 8f78074b504..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; - -import junit.framework.TestCase; - -/** - * Test {@link JobTracker} w.r.t config parameters. - */ -public class TestJobTrackerStart extends TestCase { - - public void testJobTrackerStartConfig() throws Exception { - JobConf conf = new JobConf(); - conf = MiniMRCluster.configureJobConf(conf, "file:///", 0, 0, null); - - // test JobTracker's default clock - Clock c = JobTracker.getClock(); - assertNotNull(c); - assertEquals(c, JobTracker.DEFAULT_CLOCK); - - // test with default values - JobTracker jt = JobTracker.startTracker(conf); - c = JobTracker.getClock(); - // test clock - assertNotNull(c); - assertEquals(c, JobTracker.DEFAULT_CLOCK); - // test identifier - assertEquals(12, jt.getTrackerIdentifier().length()); // correct upto mins - jt.stopTracker(); - - // test with special clock - FakeClock myClock = new FakeClock(); - jt = JobTracker.startTracker(conf, myClock); - c = JobTracker.getClock(); - assertNotNull(c); - assertEquals(c, myClock); - jt.stopTracker(); - - // test with special identifier - String identifier = "test-identifier"; - jt = JobTracker.startTracker(conf, JobTracker.DEFAULT_CLOCK, identifier); - assertEquals(identifier, jt.getTrackerIdentifier()); - jt.stopTracker(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java deleted file mode 100644 index e1074c03376..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerXmlJsp.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class TestJobTrackerXmlJsp extends ClusterMapReduceTestCase { - - private static final Log LOG = LogFactory.getLog(TestJobTrackerXmlJsp.class); - - /** - * Read the jobtracker.jspx status page and validate that the XML is well formed. - */ - public void testXmlWellFormed() throws IOException, ParserConfigurationException, SAXException { - MiniMRCluster cluster = getMRCluster(); - int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); - - String xmlJspUrl = "http://localhost:" + infoPort + "/jobtracker.jspx"; - LOG.info("Retrieving XML from URL: " + xmlJspUrl); - - DocumentBuilder parser = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document doc = parser.parse(xmlJspUrl); - - // If we get here, then the document was successfully parsed by SAX and is well-formed. - LOG.info("Document received and parsed."); - - // Make sure it has a element as top-level. - NodeList clusterNodes = doc.getElementsByTagName("cluster"); - assertEquals("There should be exactly 1 element", 1, clusterNodes.getLength()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java deleted file mode 100644 index 29faa5dc0e9..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.Vector; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.Executors; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.mapred.JvmManager.JvmManagerForType; -import org.apache.hadoop.mapred.JvmManager.JvmManagerForType.JvmRunner; -import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.junit.After; -import static org.junit.Assert.*; -import org.junit.Before; -import org.junit.Test; - -public class TestJvmManager { - static final Log LOG = LogFactory.getLog(TestJvmManager.class); - - private static File TEST_DIR = new File(System.getProperty("test.build.data", - "/tmp"), TestJvmManager.class.getSimpleName()); - private static int MAP_SLOTS = 10; - private static int REDUCE_SLOTS = 10; - private TaskTracker tt; - private JvmManager jvmManager; - private JobConf ttConf; - - @Before - public void setUp() { - TEST_DIR.mkdirs(); - } - - @After - public void tearDown() { - FileUtil.fullyDelete(TEST_DIR); - } - - public TestJvmManager() throws Exception { - tt = new TaskTracker(); - ttConf = new JobConf(); - ttConf.setLong(TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL, 2000); - tt.setConf(ttConf); - tt.setMaxMapSlots(MAP_SLOTS); - tt.setMaxReduceSlots(REDUCE_SLOTS); - tt.setTaskController(new DefaultTaskController()); - jvmManager = new JvmManager(tt); - tt.setJvmManagerInstance(jvmManager); - } - - // write a shell script to execute the command. - private File writeScript(String fileName, String cmd, File pidFile) throws IOException { - File script = new File(TEST_DIR, fileName); - FileOutputStream out = new FileOutputStream(script); - // write pid into a file - out.write(("echo $$ >" + pidFile.toString() + ";").getBytes()); - // ignore SIGTERM - out.write(("trap '' 15\n").getBytes()); - // write the actual command it self. - out.write(cmd.getBytes()); - out.close(); - script.setExecutable(true); - return script; - } - - /** - * Tests the jvm kill from JvmRunner and JvmManager simultaneously. - * - * Starts a process, which sleeps for 60 seconds, in a thread. - * Calls JvmRunner.kill() in a thread. - * Also calls JvmManager.taskKilled(). - * Makes sure that the jvm is killed and JvmManager could launch another task - * properly. - * @throws Exception - */ - @Test - public void testJvmKill() throws Exception { - JvmManagerForType mapJvmManager = jvmManager - .getJvmManagerForType(TaskType.MAP); - // launch a jvm - JobConf taskConf = new JobConf(ttConf); - TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 0); - MapTask task = new MapTask(null, attemptID, 0, null, 1); - task.setConf(taskConf); - TaskInProgress tip = tt.new TaskInProgress(task, taskConf); - File pidFile = new File(TEST_DIR, "pid"); - final TaskRunner taskRunner = new MapTaskRunner(tip, tt, taskConf); - // launch a jvm which sleeps for 60 seconds - final Vector vargs = new Vector(2); - vargs.add(writeScript("SLEEP", "sleep 60\n", pidFile).getAbsolutePath()); - final File workDir = new File(TEST_DIR, "work"); - workDir.mkdir(); - final File stdout = new File(TEST_DIR, "stdout"); - final File stderr = new File(TEST_DIR, "stderr"); - - // launch the process and wait in a thread, till it finishes - Thread launcher = new Thread() { - public void run() { - try { - taskRunner.launchJvmAndWait(null, vargs, stdout, stderr, 100, - workDir, null); - } catch (InterruptedException e) { - e.printStackTrace(); - return; - } - } - }; - launcher.start(); - // wait till the jvm is launched - // this loop waits for at most 1 second - for (int i = 0; i < 10; i++) { - if (pidFile.exists()) { - break; - } - UtilsForTests.waitFor(100); - } - // assert that the process is launched - assertTrue("pidFile is not present", pidFile.exists()); - - // imitate Child code. - // set pid in jvmManager - BufferedReader in = new BufferedReader(new FileReader(pidFile)); - String pid = in.readLine(); - in.close(); - JVMId jvmid = mapJvmManager.runningTaskToJvm.get(taskRunner); - jvmManager.setPidToJvm(jvmid, pid); - - // kill JvmRunner - final JvmRunner jvmRunner = mapJvmManager.jvmIdToRunner.get(jvmid); - Thread killer = new Thread() { - public void run() { - jvmRunner.kill(); - } - }; - killer.start(); - - //wait for a while so that killer thread is started. - Thread.sleep(100); - - // kill the jvm externally - taskRunner.kill(); - - assertTrue(jvmRunner.killed); - - // launch another jvm and see it finishes properly - attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 1); - task = new MapTask(null, attemptID, 0, null, 1); - task.setConf(taskConf); - tip = tt.new TaskInProgress(task, taskConf); - TaskRunner taskRunner2 = new MapTaskRunner(tip, tt, taskConf); - // build dummy vargs to call ls - Vector vargs2 = new Vector(1); - vargs2.add(writeScript("LS", "ls", pidFile).getAbsolutePath()); - File workDir2 = new File(TEST_DIR, "work2"); - workDir.mkdir(); - File stdout2 = new File(TEST_DIR, "stdout2"); - File stderr2 = new File(TEST_DIR, "stderr2"); - taskRunner2.launchJvmAndWait(null, vargs2, stdout2, stderr2, 100, workDir2, - null); - // join all the threads - killer.join(); - jvmRunner.join(); - launcher.join(); - } - - - /** - * Create a bunch of tasks and use a special hash map to detect - * racy access to the various internal data structures of JvmManager. - * (Regression test for MAPREDUCE-2224) - */ - @Test - public void testForRaces() throws Exception { - JvmManagerForType mapJvmManager = jvmManager - .getJvmManagerForType(TaskType.MAP); - - // Sub out the HashMaps for maps that will detect racy access. - mapJvmManager.jvmToRunningTask = new RaceHashMap(); - mapJvmManager.runningTaskToJvm = new RaceHashMap(); - mapJvmManager.jvmIdToRunner = new RaceHashMap(); - - // Launch a bunch of JVMs, but only allow MAP_SLOTS to run at once. - final ExecutorService exec = Executors.newFixedThreadPool(MAP_SLOTS); - final AtomicReference failed = - new AtomicReference(); - - for (int i = 0; i < MAP_SLOTS*5; i++) { - JobConf taskConf = new JobConf(ttConf); - TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, i, 0); - Task task = new MapTask(null, attemptID, i, null, 1); - task.setConf(taskConf); - TaskInProgress tip = tt.new TaskInProgress(task, taskConf); - File pidFile = new File(TEST_DIR, "pid_" + i); - final TaskRunner taskRunner = new MapTaskRunner(tip, tt, taskConf); - // launch a jvm which sleeps for 60 seconds - final Vector vargs = new Vector(2); - vargs.add(writeScript("script_" + i, "echo hi\n", pidFile).getAbsolutePath()); - final File workDir = new File(TEST_DIR, "work_" + i); - workDir.mkdir(); - final File stdout = new File(TEST_DIR, "stdout_" + i); - final File stderr = new File(TEST_DIR, "stderr_" + i); - - // launch the process and wait in a thread, till it finishes - Runnable launcher = new Runnable() { - public void run() { - try { - taskRunner.launchJvmAndWait(null, vargs, stdout, stderr, 100, - workDir, null); - } catch (Throwable t) { - failed.compareAndSet(null, t); - exec.shutdownNow(); - return; - } - } - }; - exec.submit(launcher); - } - - exec.shutdown(); - exec.awaitTermination(3, TimeUnit.MINUTES); - if (failed.get() != null) { - throw new RuntimeException(failed.get()); - } - } - - /** - * HashMap which detects racy usage by sleeping during operations - * and checking that no other threads access the map while asleep. - */ - static class RaceHashMap extends HashMap { - Object syncData = new Object(); - RuntimeException userStack = null; - boolean raced = false; - - private void checkInUse() { - synchronized (syncData) { - RuntimeException thisStack = new RuntimeException(Thread.currentThread().toString()); - - if (userStack != null && raced == false) { - RuntimeException other = userStack; - raced = true; - LOG.fatal("Race between two threads."); - LOG.fatal("First", thisStack); - LOG.fatal("Second", other); - throw new RuntimeException("Raced"); - } else { - userStack = thisStack; - } - } - } - - private void sleepABit() { - try { - Thread.sleep(60); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - } - - private void done() { - synchronized (syncData) { - userStack = null; - } - } - - @Override - public V get(Object key) { - checkInUse(); - try { - sleepABit(); - return super.get(key); - } finally { - done(); - } - } - - @Override - public boolean containsKey(Object key) { - checkInUse(); - try { - sleepABit(); - return super.containsKey(key); - } finally { - done(); - } - } - - @Override - public V put(K key, V val) { - checkInUse(); - try { - sleepABit(); - return super.put(key, val); - } finally { - done(); - } - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java deleted file mode 100644 index 8e279df11da..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.DataOutputStream; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskType; -import static org.junit.Assert.*; -import org.junit.Test; - -public class TestJvmReuse { - private static Path rootDir = new Path(System.getProperty("test.build.data", - "/tmp"), TestJvmReuse.class.getName()); - private int numMappers = 5; - private static int taskWithCleanup = 2; // third task - - /** - * A mapper class in which all attempts log taskid. Zeroth attempt of task - * with id=taskWithCleanup, fails with System.exit to force a cleanup attempt - * for the task in a new jvm. - */ - public static class MapperClass extends MapReduceBase implements - Mapper { - String taskid; - static int instances = 0; - Reporter reporter = null; - - public void configure(JobConf job) { - taskid = job.get("mapred.task.id"); - } - - public void map(LongWritable key, Text value, - OutputCollector output, Reporter reporter) - throws IOException { - System.err.println(taskid); - this.reporter = reporter; - - if (TaskAttemptID.forName(taskid).getTaskID().getId() == taskWithCleanup) { - if (taskid.endsWith("_0")) { - System.exit(-1); - } - } - } - - public void close() throws IOException { - reporter.incrCounter("jvm", "use", ++instances); - } - } - - public RunningJob launchJob(JobConf conf, Path inDir, Path outDir) - throws IOException { - // set up the input file system and write input text. - FileSystem inFs = inDir.getFileSystem(conf); - FileSystem outFs = outDir.getFileSystem(conf); - outFs.delete(outDir, true); - if (!inFs.mkdirs(inDir)) { - throw new IOException("Mkdirs failed to create " + inDir.toString()); - } - for (int i = 0; i < numMappers; i++) { - // write input into input file - DataOutputStream file = inFs.create(new Path(inDir, "part-" + i)); - file.writeBytes("input"); - file.close(); - } - - // configure the mapred Job - conf.setMapperClass(MapperClass.class); - conf.setNumReduceTasks(0); - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - // enable jvm reuse - conf.setNumTasksToExecutePerJvm(-1); - // return the RunningJob handle. - return new JobClient(conf).submitJob(conf); - } - - private void validateAttempt(TaskInProgress tip, TaskAttemptID attemptId, - TaskStatus ts, boolean isCleanup) throws IOException { - assertEquals(isCleanup, tip.isCleanupAttempt(attemptId)); - // validate tasklogs for task attempt - String log = MapReduceTestUtil.readTaskLog(TaskLog.LogName.STDERR, - attemptId, false); - assertTrue(log.equals(attemptId.toString())); - assertTrue(ts != null); - if (!isCleanup) { - assertEquals(TaskStatus.State.SUCCEEDED, ts.getRunState()); - } else { - assertEquals(TaskStatus.State.FAILED, ts.getRunState()); - // validate tasklogs for cleanup attempt - log = MapReduceTestUtil.readTaskLog(TaskLog.LogName.STDERR, attemptId, - true); - assertTrue(log.equals(TestTaskFail.cleanupLog)); - } - } - - // validates logs of all attempts of the job. - private void validateJob(RunningJob job, MiniMRCluster mr) throws IOException { - assertEquals(JobStatus.SUCCEEDED, job.getJobState()); - long uses = job.getCounters().findCounter("jvm", "use").getValue(); - assertTrue("maps = " + numMappers + ", jvms = " + uses, numMappers < uses); - - JobID jobId = job.getID(); - - for (int i = 0; i < numMappers; i++) { - TaskAttemptID attemptId = new TaskAttemptID(new TaskID(jobId, - TaskType.MAP, i), 0); - TaskInProgress tip = mr.getJobTrackerRunner().getJobTracker().getTip( - attemptId.getTaskID()); - TaskStatus ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus( - attemptId); - validateAttempt(tip, attemptId, ts, i == taskWithCleanup); - if (i == taskWithCleanup) { - // validate second attempt of the task - attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, i), 1); - ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus(attemptId); - validateAttempt(tip, attemptId, ts, false); - } - } - } - - /** - * Runs job with jvm reuse and verifies that the logs for all attempts can be - * read properly. - * - * @throws IOException - */ - @Test - public void testTaskLogs() throws IOException { - MiniMRCluster mr = null; - try { - Configuration conf = new Configuration(); - final int taskTrackers = 1; // taskTrackers should be 1 to test jvm reuse. - conf.setInt("mapred.tasktracker.map.tasks.maximum", 1); - mr = new MiniMRCluster(taskTrackers, "file:///", 1); - - final Path inDir = new Path(rootDir, "input"); - final Path outDir = new Path(rootDir, "output"); - JobConf jobConf = mr.createJobConf(); - jobConf.setOutputCommitter(TestTaskFail.CommitterWithLogs.class); - RunningJob rJob = launchJob(jobConf, inDir, outDir); - rJob.waitForCompletion(); - validateJob(rJob, mr); - } finally { - if (mr != null) { - mr.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillCompletedJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillCompletedJob.java deleted file mode 100644 index 8ce2283e7cd..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillCompletedJob.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapred.TestRackAwareTaskPlacement.MyFakeJobInProgress; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -/** - * A JUnit test to test that killing completed jobs does not move them - * to the failed sate - See JIRA HADOOP-2132 - */ -public class TestKillCompletedJob extends TestCase { - - MyFakeJobInProgress job; - static FakeJobTracker jobTracker; - - static FakeClock clock; - - static String trackers[] = new String[] {"tracker_tracker1:1000"}; - - @Override - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); - jobTracker = new FakeJobTracker(conf, (clock = new FakeClock()), trackers); - } - - - @SuppressWarnings("deprecation") - public void testKillCompletedJob() throws IOException, InterruptedException { - job = new MyFakeJobInProgress(new JobConf(), jobTracker); - jobTracker.addJob(job.getJobID(), (JobInProgress)job); - job.status.setRunState(JobStatus.SUCCEEDED); - - jobTracker.killJob(job.getJobID()); - - assertTrue("Run state changed when killing completed job" , - job.status.getRunState() == JobStatus.SUCCEEDED); - - } - -} - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillSubProcesses.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillSubProcesses.java deleted file mode 100644 index 00cf61cb54b..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestKillSubProcesses.java +++ /dev/null @@ -1,542 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.util.Random; -import java.util.Iterator; -import java.util.StringTokenizer; - -import junit.framework.TestCase; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapreduce.util.TestProcfsBasedProcessTree; - -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.ProcessTree; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.Shell.ExitCodeException; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * A JUnit test to test Kill Job that has tasks with children and checks if the - * children(subprocesses of java task) are also killed when a task is killed. - */ -public class TestKillSubProcesses extends TestCase { - - private static volatile Log LOG = LogFactory - .getLog(TestKillSubProcesses.class); - - private static String BASE_TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).getAbsolutePath(); - private static String TEST_ROOT_DIR = BASE_TEST_ROOT_DIR + Path.SEPARATOR - + "killSubProcesses"; - - private static Path scriptDir = new Path(TEST_ROOT_DIR, "script"); - private static String scriptDirName = scriptDir.toUri().getPath(); - private static Path signalFile = new Path(TEST_ROOT_DIR - + "/script/signalFile"); - - private static JobClient jobClient = null; - - static MiniMRCluster mr = null; - - private static String pid = null; - - // number of levels in the subtree of subprocesses of map task - private static int numLevelsOfSubProcesses = 4; - - /** - * Runs a job, kills the job and verifies if the map task and its - * subprocesses are also killed properly or not. - */ - private static void runKillingJobAndValidate(JobTracker jt, JobConf conf) throws IOException { - - conf.setJobName("testkilljobsubprocesses"); - conf.setMapperClass(KillingMapperWithChildren.class); - - RunningJob job = runJobAndSetProcessHandle(jt, conf); - - // kill the job now - job.killJob(); - - while (job.cleanupProgress() == 0.0f) { - try { - Thread.sleep(100); - } catch (InterruptedException ie) { - LOG.warn("sleep is interrupted:" + ie); - break; - } - } - - validateKillingSubprocesses(job, conf); - // Checking the Job status - assertEquals(job.getJobState(), JobStatus.KILLED); - } - - /** - * Runs a job that will fail and verifies if the subprocesses of failed map - * task are killed properly or not. - */ - private static void runFailingJobAndValidate(JobTracker jt, JobConf conf) throws IOException { - - conf.setJobName("testfailjobsubprocesses"); - conf.setMapperClass(FailingMapperWithChildren.class); - - // We don't want to run the failing map task 4 times. So we run it once and - // check if all the subprocesses are killed properly. - conf.setMaxMapAttempts(1); - - RunningJob job = runJobAndSetProcessHandle(jt, conf); - signalTask(signalFile.toString(), conf); - validateKillingSubprocesses(job, conf); - // Checking the Job status - assertEquals(job.getJobState(), JobStatus.FAILED); - } - - /** - * Runs a job that will succeed and verifies if the subprocesses of succeeded - * map task are killed properly or not. - */ - private static void runSuccessfulJobAndValidate(JobTracker jt, JobConf conf) - throws IOException { - - conf.setJobName("testsucceedjobsubprocesses"); - conf.setMapperClass(MapperWithChildren.class); - - RunningJob job = runJobAndSetProcessHandle(jt, conf); - signalTask(signalFile.toString(), conf); - validateKillingSubprocesses(job, conf); - // Checking the Job status - assertEquals(job.getJobState(), JobStatus.SUCCEEDED); - } - - /** - * Runs the given job and saves the pid of map task. - * Also checks if the subprocesses of map task are alive. - */ - private static RunningJob runJobAndSetProcessHandle(JobTracker jt, JobConf conf) - throws IOException { - RunningJob job = runJob(conf); - while (job.getJobState() != JobStatus.RUNNING) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - - pid = null; - jobClient = new JobClient(conf); - - // get the taskAttemptID of the map task and use it to get the pid - // of map task - TaskReport[] mapReports = jobClient.getMapTaskReports(job.getID()); - - JobInProgress jip = jt.getJob(job.getID()); - for(TaskReport tr : mapReports) { - TaskInProgress tip = jip.getTaskInProgress(tr.getTaskID()); - - // for this tip, get active tasks of all attempts - while(tip.getActiveTasks().size() == 0) { - //wait till the activeTasks Tree is built - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - LOG.warn("sleep is interrupted:" + ie); - break; - } - } - - for (Iterator it = - tip.getActiveTasks().keySet().iterator(); it.hasNext();) { - TaskAttemptID id = it.next(); - LOG.info("taskAttemptID of map task is " + id); - - while(pid == null) { - pid = mr.getTaskTrackerRunner(0).getTaskTracker().getPid(id); - if (pid == null) { - try { - Thread.sleep(500); - } catch(InterruptedException e) {} - } - } - LOG.info("pid of map task is " + pid); - //Checking if the map task is alive - assertTrue("Map is no more alive", isAlive(pid)); - LOG.info("The map task is alive before Job completion, as expected."); - } - } - - // Checking if the descendant processes of map task are alive - if(ProcessTree.isSetsidAvailable) { - String childPid = TestProcfsBasedProcessTree.getPidFromPidFile( - scriptDirName + "/childPidFile" + 0); - while(childPid == null) { - LOG.warn(scriptDirName + "/childPidFile" + 0 + " is null; Sleeping..."); - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - LOG.warn("sleep is interrupted:" + ie); - break; - } - childPid = TestProcfsBasedProcessTree.getPidFromPidFile( - scriptDirName + "/childPidFile" + 0); - } - - // As childPidFile0(leaf process in the subtree of processes with - // map task as root) is created, all other child pid files should - // have been created already(See the script for details). - // Now check if the descendants of map task are alive. - for(int i=0; i <= numLevelsOfSubProcesses; i++) { - childPid = TestProcfsBasedProcessTree.getPidFromPidFile( - scriptDirName + "/childPidFile" + i); - LOG.info("pid of the descendant process at level " + i + - "in the subtree of processes(with the map task as the root)" + - " is " + childPid); - assertTrue("Unexpected: The subprocess at level " + i + - " in the subtree is not alive before Job completion", - isAlive(childPid)); - } - } - return job; - } - - /** - * Verifies if the subprocesses of the map task are killed properly. - */ - private static void validateKillingSubprocesses(RunningJob job, JobConf conf) - throws IOException { - // wait till the the job finishes - while (!job.isComplete()) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - break; - } - } - - // Checking if the map task got killed or not - assertTrue(!ProcessTree.isAlive(pid)); - LOG.info("The map task is not alive after Job is completed, as expected."); - - // Checking if the descendant processes of map task are killed properly - if(ProcessTree.isSetsidAvailable) { - for(int i=0; i <= numLevelsOfSubProcesses; i++) { - String childPid = TestProcfsBasedProcessTree.getPidFromPidFile( - scriptDirName + "/childPidFile" + i); - LOG.info("pid of the descendant process at level " + i + - "in the subtree of processes(with the map task as the root)" + - " is " + childPid); - assertTrue("Unexpected: The subprocess at level " + i + - " in the subtree is alive after Job completion", - !isAlive(childPid)); - } - } - FileSystem fs = FileSystem.getLocal(mr.createJobConf()); - if(fs.exists(scriptDir)) { - fs.delete(scriptDir, true); - } - } - - private static RunningJob runJob(JobConf conf) throws IOException { - - final Path inDir; - final Path outDir; - FileSystem fs = FileSystem.getLocal(conf); - FileSystem tempFs = FileSystem.get(conf); - //Check if test is run with hdfs or local file system. - //if local filesystem then prepend TEST_ROOT_DIR, otherwise - //killjob folder would be created in workspace root. - if (!tempFs.getUri().toASCIIString().equals( - fs.getUri().toASCIIString())) { - inDir = new Path("killjob/input"); - outDir = new Path("killjob/output"); - } else { - inDir = new Path(TEST_ROOT_DIR, "input"); - outDir = new Path(TEST_ROOT_DIR, "output"); - } - - - if(fs.exists(scriptDir)) { - fs.delete(scriptDir, true); - } - - conf.setNumMapTasks(1); - conf.setNumReduceTasks(0); - - conf.set(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, - conf.get(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, - conf.get(JobConf.MAPRED_TASK_JAVA_OPTS)) + - " -Dtest.build.data=" + BASE_TEST_ROOT_DIR); - conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, - conf.get(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, - conf.get(JobConf.MAPRED_TASK_JAVA_OPTS)) + - " -Dtest.build.data=" + BASE_TEST_ROOT_DIR); - - return UtilsForTests.runJob(conf, inDir, outDir); - } - - public void testJobKillFailAndSucceed() throws IOException { - if (Shell.WINDOWS) { - System.out.println( - "setsid doesn't work on WINDOWS as expected. Not testing"); - return; - } - - JobConf conf=null; - try { - mr = new MiniMRCluster(1, "file:///", 1); - - // run the TCs - conf = mr.createJobConf(); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - runTests(conf, jt); - } finally { - if (mr != null) { - mr.shutdown(); - } - } - } - - void runTests(JobConf conf, JobTracker jt) throws IOException { - FileSystem fs = FileSystem.getLocal(mr.createJobConf()); - Path rootDir = new Path(TEST_ROOT_DIR); - if(!fs.exists(rootDir)) { - fs.mkdirs(rootDir); - } - fs.setPermission(rootDir, - new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); - runKillingJobAndValidate(jt, conf); - runFailingJobAndValidate(jt, conf); - runSuccessfulJobAndValidate(jt, conf); - } - - /** - * Creates signal file - */ - private static void signalTask(String signalFile, JobConf conf) { - try { - FileSystem fs = FileSystem.getLocal(conf); - fs.createNewFile(new Path(signalFile)); - } catch(IOException e) { - LOG.warn("Unable to create signal file. " + e); - } - } - - /** - * Runs a recursive shell script to create a chain of subprocesses - */ - private static void runChildren(JobConf conf) throws IOException { - if (ProcessTree.isSetsidAvailable) { - FileSystem fs = FileSystem.getLocal(conf); - - if (fs.exists(scriptDir)) { - fs.delete(scriptDir, true); - } - - // Create the directory and set open permissions so that the TT can - // access. - fs.mkdirs(scriptDir); - fs.setPermission(scriptDir, new FsPermission(FsAction.ALL, FsAction.ALL, - FsAction.ALL)); - - // create shell script - Random rm = new Random(); - Path scriptPath = new Path(scriptDirName, "_shellScript_" + rm.nextInt() - + ".sh"); - String shellScript = scriptPath.toString(); - - // Construct the script. Set umask to 0000 so that TT can access all the - // files. - String script = - "umask 000\n" + - "echo $$ > " + scriptDirName + "/childPidFile" + "$1\n" + - "echo hello\n" + - "trap 'echo got SIGTERM' 15 \n" + - "if [ $1 != 0 ]\nthen\n" + - " sh " + shellScript + " $(($1-1))\n" + - "else\n" + - " while true\n do\n" + - " sleep 2\n" + - " done\n" + - "fi"; - DataOutputStream file = fs.create(scriptPath); - file.writeBytes(script); - file.close(); - - // Set executable permissions on the script. - new File(scriptPath.toUri().getPath()).setExecutable(true); - - LOG.info("Calling script from map task : " + shellScript); - Runtime.getRuntime() - .exec(shellScript + " " + numLevelsOfSubProcesses); - - String childPid = TestProcfsBasedProcessTree.getPidFromPidFile(scriptDirName - + "/childPidFile" + 0); - while (childPid == null) { - LOG.warn(scriptDirName + "/childPidFile" + 0 + " is null; Sleeping..."); - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - LOG.warn("sleep is interrupted:" + ie); - break; - } - childPid = TestProcfsBasedProcessTree.getPidFromPidFile(scriptDirName - + "/childPidFile" + 0); - } - } - } - - /** - * Mapper that starts children - */ - static class MapperWithChildren extends MapReduceBase implements - Mapper { - FileSystem fs = null; - public void configure(JobConf conf) { - try { - fs = FileSystem.getLocal(conf); - runChildren(conf); - } catch (Exception e) { - LOG.warn("Exception in configure: " + - StringUtils.stringifyException(e)); - } - } - - // Mapper waits for the signal(signal is the existence of a file) - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - while (!fs.exists(signalFile)) {// wait for signal file creation - try { - reporter.progress(); - synchronized (this) { - this.wait(1000); - } - } catch (InterruptedException ie) { - System.out.println("Interrupted while the map was waiting for " - + " the signal."); - break; - } - } - } - } - - /** - * Mapper that waits till it gets killed. - */ - static class KillingMapperWithChildren extends MapperWithChildren { - public void configure(JobConf conf) { - super.configure(conf); - } - - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - - try { - while(true) {//just wait till kill happens - Thread.sleep(1000); - } - } catch (InterruptedException e) { - LOG.warn("Exception in KillMapperWithChild.map:" + e); - } - } - } - - /** - * Mapper that fails when receives a signal. Signal is existence of a file. - */ - static class FailingMapperWithChildren extends MapperWithChildren { - public void configure(JobConf conf) { - super.configure(conf); - } - - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - while (!fs.exists(signalFile)) {// wait for signal file creation - try { - reporter.progress(); - synchronized (this) { - this.wait(1000); - } - } catch (InterruptedException ie) { - System.out.println("Interrupted while the map was waiting for " - + " the signal."); - break; - } - } - throw new RuntimeException("failing map"); - } - } - - /** - * Check for presence of the process with the pid passed is alive or not - * currently. - * - * @param pid pid of the process - * @return if a process is alive or not. - */ - private static boolean isAlive(String pid) throws IOException { - String commandString ="ps -o pid,command -e"; - String args[] = new String[] {"bash", "-c" , commandString}; - ShellCommandExecutor shExec = new ShellCommandExecutor(args); - try { - shExec.execute(); - }catch(ExitCodeException e) { - return false; - } catch (IOException e) { - LOG.warn("IOExecption thrown while checking if process is alive" + - StringUtils.stringifyException(e)); - throw e; - } - - String output = shExec.getOutput(); - - //Parse the command output and check for pid, ignore the commands - //which has ps or grep in it. - StringTokenizer strTok = new StringTokenizer(output, "\n"); - boolean found = false; - while(strTok.hasMoreTokens()) { - StringTokenizer pidToken = new StringTokenizer(strTok.nextToken(), - " "); - String pidStr = pidToken.nextToken(); - String commandStr = pidToken.nextToken(); - if(pid.equals(pidStr) && !(commandStr.contains("ps") - || commandStr.contains("grep"))) { - found = true; - break; - } - } - return found; - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLimitTasksPerJobTaskScheduler.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLimitTasksPerJobTaskScheduler.java deleted file mode 100644 index 2d2e45d3377..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLimitTasksPerJobTaskScheduler.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapred.TestJobQueueTaskScheduler.FakeTaskTrackerManager; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -public class TestLimitTasksPerJobTaskScheduler extends TestCase { - protected JobConf jobConf; - protected TaskScheduler scheduler; - private FakeTaskTrackerManager taskTrackerManager; - - @Override - protected void setUp() throws Exception { - TestJobQueueTaskScheduler.resetCounters(); - jobConf = new JobConf(); - jobConf.setNumMapTasks(10); - jobConf.setNumReduceTasks(10); - taskTrackerManager = new FakeTaskTrackerManager(); - scheduler = createTaskScheduler(); - scheduler.setConf(jobConf); - scheduler.setTaskTrackerManager(taskTrackerManager); - scheduler.start(); - } - - @Override - protected void tearDown() throws Exception { - if (scheduler != null) { - scheduler.terminate(); - } - } - - protected TaskScheduler createTaskScheduler() { - return new LimitTasksPerJobTaskScheduler(); - } - - public void testMaxRunningTasksPerJob() throws IOException { - jobConf.setLong(JTConfig.JT_RUNNINGTASKS_PER_JOB, - 4L); - scheduler.setConf(jobConf); - TestJobQueueTaskScheduler.submitJobs(taskTrackerManager, jobConf, - 2, JobStatus.RUNNING); - - // First 4 slots are filled with job 1, second 4 with job 2 - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_m_000001_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_m_000002_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_r_000003_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_r_000004_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_m_000005_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_m_000006_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_r_000007_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_r_000008_0 on tt2"}); - } - - public void testMaxRunningTasksPerJobWithInterleavedTrackers() - throws IOException { - jobConf.setLong(JTConfig.JT_RUNNINGTASKS_PER_JOB, - 4L); - scheduler.setConf(jobConf); - TestJobQueueTaskScheduler.submitJobs(taskTrackerManager, jobConf, 2, JobStatus.RUNNING); - - // First 4 slots are filled with job 1, second 4 with job 2 - // even when tracker requests are interleaved - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_m_000001_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_m_000002_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0001_m_000003_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0001_r_000004_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_m_000005_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt1"), - new String[] {"attempt_test_0002_r_000006_0 on tt1"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_r_000007_0 on tt2"}); - TestJobQueueTaskScheduler.checkAssignment( - scheduler, TestJobQueueTaskScheduler.tracker(taskTrackerManager, "tt2"), - new String[] {"attempt_test_0002_r_000008_0 on tt2"}); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLostTracker.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLostTracker.java deleted file mode 100644 index e6f1bb9f5bd..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLostTracker.java +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -/** - * A test to verify JobTracker's resilience to lost task trackers. - * - */ -@SuppressWarnings("deprecation") -public class TestLostTracker extends TestCase { - - FakeJobInProgress job; - static FakeJobTracker jobTracker; - - static FakeClock clock; - - static String trackers[] = new String[] {"tracker_tracker1:1000", - "tracker_tracker2:1000"}; - - @Override - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); - conf.set(JTConfig.JT_MAX_TRACKER_BLACKLISTS, "1"); - jobTracker = new FakeJobTracker(conf, (clock = new FakeClock()), trackers); - jobTracker.startExpireTrackersThread(); - } - - @Override - protected void tearDown() throws Exception { - jobTracker.stopExpireTrackersThread(); - } - - public void testLostTracker() throws IOException { - // Tracker 0 contacts JT - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[0]); - - TaskAttemptID[] tid = new TaskAttemptID[2]; - JobConf conf = new JobConf(); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(1); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - - // Tracker 0 gets the map task - tid[0] = job.findMapTask(trackers[0]); - - job.finishTask(tid[0]); - - // Advance clock. Tracker 0 would have got lost - clock.advance(8 * 1000); - - jobTracker.checkExpiredTrackers(); - - // Tracker 1 establishes contact with JT - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]); - - // Tracker1 should get assigned the lost map task - tid[1] = job.findMapTask(trackers[1]); - - assertNotNull("Map Task from Lost Tracker did not get reassigned", tid[1]); - - assertEquals("Task ID of reassigned map task does not match", - tid[0].getTaskID().toString(), tid[1].getTaskID().toString()); - - job.finishTask(tid[1]); - - } - - /** - * Test whether the tracker gets blacklisted after its lost. - */ - public void testLostTrackerBeforeBlacklisting() throws Exception { - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[0]); - TaskAttemptID[] tid = new TaskAttemptID[3]; - JobConf conf = new JobConf(); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(1); - conf.set(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, "1"); - conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - job.setClusterSize(4); - - // Tracker 0 gets the map task - tid[0] = job.findMapTask(trackers[0]); - - job.finishTask(tid[0]); - - // validate the total tracker count - assertEquals("Active tracker count mismatch", - 1, jobTracker.getClusterStatus(false).getTaskTrackers()); - - // lose the tracker - clock.advance(1100); - jobTracker.checkExpiredTrackers(); - assertFalse("Tracker 0 not lost", - jobTracker.getClusterStatus(false).getActiveTrackerNames() - .contains(trackers[0])); - - // validate the total tracker count - assertEquals("Active tracker count mismatch", - 0, jobTracker.getClusterStatus(false).getTaskTrackers()); - - // Tracker 1 establishes contact with JT - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]); - - // Tracker1 should get assigned the lost map task - tid[1] = job.findMapTask(trackers[1]); - - assertNotNull("Map Task from Lost Tracker did not get reassigned", tid[1]); - - assertEquals("Task ID of reassigned map task does not match", - tid[0].getTaskID().toString(), tid[1].getTaskID().toString()); - - // finish the map task - job.finishTask(tid[1]); - - // finish the reduce task - tid[2] = job.findReduceTask(trackers[1]); - job.finishTask(tid[2]); - - // check if job is successful - assertEquals("Job not successful", - JobStatus.SUCCEEDED, job.getStatus().getRunState()); - - // check if the tracker is lost - // validate the total tracker count - assertEquals("Active tracker count mismatch", - 1, jobTracker.getClusterStatus(false).getTaskTrackers()); - // validate blacklisted count .. since we lost one blacklisted tracker - assertEquals("Blacklisted tracker count mismatch", - 0, jobTracker.getClusterStatus(false).getBlacklistedTrackers()); - } - - /** - * Test whether the tracker gets lost after its blacklisted. - */ - public void testLostTrackerAfterBlacklisting() throws Exception { - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[0]); - clock.advance(600); - TaskAttemptID[] tid = new TaskAttemptID[2]; - JobConf conf = new JobConf(); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(0); - conf.set(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, "1"); - conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - job.setClusterSize(4); - - // check if the tracker count is correct - assertEquals("Active tracker count mismatch", - 1, jobTracker.taskTrackers().size()); - - // Tracker 0 gets the map task - tid[0] = job.findMapTask(trackers[0]); - // Fail the task - job.failTask(tid[0]); - - // Tracker 1 establishes contact with JT - FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]); - // check if the tracker count is correct - assertEquals("Active tracker count mismatch", - 2, jobTracker.taskTrackers().size()); - - // Tracker 1 gets the map task - tid[1] = job.findMapTask(trackers[1]); - // Finish the task and also the job - job.finishTask(tid[1]); - - // check if job is successful - assertEquals("Job not successful", - JobStatus.SUCCEEDED, job.getStatus().getRunState()); - - // check if the trackers 1 got blacklisted - assertTrue("Tracker 0 not blacklisted", - jobTracker.getBlacklistedTrackers()[0].getTaskTrackerName() - .equals(trackers[0])); - // check if the tracker count is correct - assertEquals("Active tracker count mismatch", - 2, jobTracker.taskTrackers().size()); - // validate blacklisted count - assertEquals("Blacklisted tracker count mismatch", - 1, jobTracker.getClusterStatus(false).getBlacklistedTrackers()); - - // Advance clock. Tracker 0 should be lost - clock.advance(500); - jobTracker.checkExpiredTrackers(); - - // check if the task tracker is lost - assertFalse("Tracker 0 not lost", - jobTracker.getClusterStatus(false).getActiveTrackerNames() - .contains(trackers[0])); - - // check if the lost tracker has removed from the jobtracker - assertEquals("Active tracker count mismatch", - 1, jobTracker.taskTrackers().size()); - // validate blacklisted count - assertEquals("Blacklisted tracker count mismatch", - 0, jobTracker.getClusterStatus(false).getBlacklistedTrackers()); - - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMRServerPorts.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMRServerPorts.java deleted file mode 100644 index 2fab1460eac..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMRServerPorts.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import javax.security.auth.login.LoginException; - -import junit.framework.TestCase; -import org.apache.hadoop.hdfs.TestHDFSServerPorts; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -/** - * This test checks correctness of port usage by mapreduce components: - * JobTracker, and TaskTracker. - * - * The correct behavior is:
- * - when a specific port is provided the server must either start on that port - * or fail by throwing {@link java.net.BindException}.
- * - if the port = 0 (ephemeral) then the server should choose - * a free port and start on it. - */ -public class TestMRServerPorts extends TestCase { - static final String THIS_HOST = TestHDFSServerPorts.getFullHostName() + ":0"; - - TestHDFSServerPorts hdfs = new TestHDFSServerPorts(); - - // Runs the JT in a separate thread - private static class JTRunner extends Thread { - JobTracker jt; - void setJobTracker(JobTracker jt) { - this.jt = jt; - } - - public void run() { - if (jt != null) { - try { - jt.offerService(); - } catch (Exception ioe) {} - } - } - } - /** - * Check whether the JobTracker can be started. - */ - private JobTracker startJobTracker(JobConf conf, JTRunner runner) - throws IOException, LoginException { - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - JobTracker jt = null; - try { - jt = JobTracker.startTracker(conf); - runner.setJobTracker(jt); - runner.start(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:" + jt.getTrackerPort()); - conf.set(JTConfig.JT_HTTP_ADDRESS, - "0.0.0.0:" + jt.getInfoPort()); - } catch(InterruptedException e) { - throw new IOException(e.getLocalizedMessage()); - } - return jt; - } - - private void setDataNodePorts(Configuration conf) { - conf.set("dfs.datanode.address", THIS_HOST); - conf.set("dfs.datanode.http.address", THIS_HOST); - conf.set("dfs.datanode.ipc.address", THIS_HOST); - } - - /** - * Check whether the JobTracker can be started. - */ - private boolean canStartJobTracker(JobConf conf) - throws IOException, InterruptedException, LoginException { - JobTracker jt = null; - try { - jt = JobTracker.startTracker(conf); - } catch(IOException e) { - if (e instanceof java.net.BindException) - return false; - throw e; - } - jt.fs.close(); - jt.stopTracker(); - return true; - } - - /** - * Check whether the TaskTracker can be started. - */ - private boolean canStartTaskTracker(JobConf conf) - throws IOException, InterruptedException { - TaskTracker tt = null; - try { - tt = new TaskTracker(conf); - } catch(IOException e) { - if (e instanceof java.net.BindException) - return false; - throw e; - } - tt.shutdown(); - return true; - } - - /** - * Verify JobTracker port usage. - */ - public void testJobTrackerPorts() throws Exception { - NameNode nn = null; - DataNode dn = null; - try { - nn = hdfs.startNameNode(); - setDataNodePorts(hdfs.getConfig()); - dn = hdfs.startDataNode(1, hdfs.getConfig()); - - // start job tracker on the same port as name-node - JobConf conf2 = new JobConf(hdfs.getConfig()); - conf2.set(JTConfig.JT_IPC_ADDRESS, - FileSystem.getDefaultUri(hdfs.getConfig()).toString()); - conf2.set(JTConfig.JT_HTTP_ADDRESS, THIS_HOST); - boolean started = canStartJobTracker(conf2); - assertFalse(started); // should fail - - // bind http server to the same port as name-node - conf2.set(JTConfig.JT_IPC_ADDRESS, THIS_HOST); - conf2.set(JTConfig.JT_HTTP_ADDRESS, - hdfs.getConfig().get("dfs.http.address")); - started = canStartJobTracker(conf2); - assertFalse(started); // should fail again - - // both ports are different from the name-node ones - conf2.set(JTConfig.JT_IPC_ADDRESS, THIS_HOST); - conf2.set(JTConfig.JT_HTTP_ADDRESS, THIS_HOST); - started = canStartJobTracker(conf2); - assertTrue(started); // should start now - - } finally { - hdfs.stopDataNode(dn); - hdfs.stopNameNode(nn); - } - } - - /** - * Verify JobTracker port usage. - */ - public void testTaskTrackerPorts() throws Exception { - NameNode nn = null; - DataNode dn = null; - JobTracker jt = null; - JTRunner runner = null; - try { - nn = hdfs.startNameNode(); - setDataNodePorts(hdfs.getConfig()); - dn = hdfs.startDataNode(2, hdfs.getConfig()); - - JobConf conf2 = new JobConf(hdfs.getConfig()); - runner = new JTRunner(); - jt = startJobTracker(conf2, runner); - - // start job tracker on the same port as name-node - conf2.set(TTConfig.TT_REPORT_ADDRESS, - FileSystem.getDefaultUri(hdfs.getConfig()).toString()); - conf2.set(TTConfig.TT_HTTP_ADDRESS, THIS_HOST); - boolean started = canStartTaskTracker(conf2); - assertFalse(started); // should fail - - // bind http server to the same port as name-node - conf2.set(TTConfig.TT_REPORT_ADDRESS, THIS_HOST); - conf2.set(TTConfig.TT_HTTP_ADDRESS, - hdfs.getConfig().get("dfs.http.address")); - started = canStartTaskTracker(conf2); - assertFalse(started); // should fail again - - // both ports are different from the name-node ones - conf2.set(TTConfig.TT_REPORT_ADDRESS, THIS_HOST); - conf2.set(TTConfig.TT_HTTP_ADDRESS, THIS_HOST); - started = canStartTaskTracker(conf2); - assertTrue(started); // should start now - } finally { - if (jt != null) { - jt.fs.close(); - jt.stopTracker(); - runner.interrupt(); - runner.join(); - } - hdfs.stopDataNode(dn); - hdfs.stopNameNode(nn); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java deleted file mode 100644 index 1b2d8f846cf..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.RandomWriter; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.hdfs.MiniDFSCluster; - -public class TestMapredHeartbeat extends TestCase { - public void testJobDirCleanup() throws IOException { - MiniMRCluster mr = null; - try { - // test the default heartbeat interval - int taskTrackers = 2; - JobConf conf = new JobConf(); - mr = new MiniMRCluster(taskTrackers, "file:///", 3, - null, null, conf); - JobClient jc = new JobClient(mr.createJobConf()); - while(jc.getClusterStatus().getTaskTrackers() != taskTrackers) { - UtilsForTests.waitFor(100); - } - assertEquals(JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT, - mr.getJobTrackerRunner().getJobTracker().getNextHeartbeatInterval()); - mr.shutdown(); - - // test configured heartbeat interval - taskTrackers = 5; - conf.setInt(JTConfig.JT_HEARTBEATS_IN_SECOND, 1); - mr = new MiniMRCluster(taskTrackers, "file:///", 3, - null, null, conf); - jc = new JobClient(mr.createJobConf()); - while(jc.getClusterStatus().getTaskTrackers() != taskTrackers) { - UtilsForTests.waitFor(100); - } - assertEquals(taskTrackers * 1000, - mr.getJobTrackerRunner().getJobTracker().getNextHeartbeatInterval()); - mr.shutdown(); - - // test configured heartbeat interval is capped with min value - taskTrackers = 5; - conf.setInt(JTConfig.JT_HEARTBEATS_IN_SECOND, 100); - mr = new MiniMRCluster(taskTrackers, "file:///", 3, - null, null, conf); - jc = new JobClient(mr.createJobConf()); - while(jc.getClusterStatus().getTaskTrackers() != taskTrackers) { - UtilsForTests.waitFor(100); - } - assertEquals(JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT, - mr.getJobTrackerRunner().getJobTracker().getNextHeartbeatInterval()); - } finally { - if (mr != null) { mr.shutdown(); } - } - } - - public void testOutOfBandHeartbeats() throws Exception { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - try { - Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 4, true, null); - - int taskTrackers = 1; - JobConf jobConf = new JobConf(); - jobConf.setFloat(JTConfig.JT_HEARTBEATS_SCALING_FACTOR, 30.0f); - jobConf.setBoolean(TTConfig.TT_OUTOFBAND_HEARBEAT, true); - mr = new MiniMRCluster(taskTrackers, - dfs.getFileSystem().getUri().toString(), 3, - null, null, jobConf); - long start = System.currentTimeMillis(); - TestMiniMRDFSSort.runRandomWriter(mr.createJobConf(), new Path("rw")); - long end = System.currentTimeMillis(); - - final int expectedRuntimeSecs = 120; - final int runTimeSecs = (int)((end-start) / 1000); - System.err.println("Runtime is " + runTimeSecs); - assertEquals("Actual runtime " + runTimeSecs + "s not less than expected " + - "runtime of " + expectedRuntimeSecs + "s!", - true, (runTimeSecs <= 120)); - } finally { - if (mr != null) { mr.shutdown(); } - if (dfs != null) { dfs.shutdown(); } - } - } - -} - - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java deleted file mode 100644 index e7b2a73be01..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.*; - -/** - * Test if JobTracker is resilient to garbage in {@link JTConfig#JT_SYSTEM_DIR} - */ -public class TestMapredSystemDir extends TestCase { - private static final Log LOG = LogFactory.getLog(TestMapredSystemDir.class); - - // dfs ugi - private static final UserGroupInformation DFS_UGI = - TestMiniMRWithDFSWithDistinctUsers.createUGI("dfs", true); - // mapred ugi - private static final UserGroupInformation MR_UGI = - TestMiniMRWithDFSWithDistinctUsers.createUGI("mr", false); - private static final FsPermission SYSTEM_DIR_PARENT_PERMISSION = - FsPermission.createImmutable((short) 0755); // rwxr-xr-x - private static final FsPermission SYSTEM_DIR_PERMISSION = - FsPermission.createImmutable((short) 0700); // rwx------ - - public void testGarbledMapredSystemDir() throws Exception { - Configuration conf = new Configuration(); - final MiniDFSCluster dfs = new MiniDFSCluster(conf, 1, true, null); - MiniMRCluster mr = null; - try { - // start dfs - conf.set("dfs.permissions.supergroup", "supergroup"); - FileSystem fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { - public FileSystem run() throws IOException { - return dfs.getFileSystem(); - } - }); - - - // create Configs.SYSTEM_DIR's parent with restrictive permissions. - // So long as the JT has access to the system dir itself it should - // be able to start. - Path mapredSysDir = new Path(conf.get(JTConfig.JT_SYSTEM_DIR)); - Path parentDir = mapredSysDir.getParent(); - fs.mkdirs(parentDir); - fs.setPermission(parentDir, - new FsPermission(SYSTEM_DIR_PARENT_PERMISSION)); - fs.mkdirs(mapredSysDir); - fs.setPermission(mapredSysDir, new FsPermission(SYSTEM_DIR_PERMISSION)); - fs.setOwner(mapredSysDir, "mr", "mrgroup"); - - // start mr (i.e jobtracker) - Configuration mrConf = new Configuration(conf); - mr = new MiniMRCluster(0, 0, 0, dfs.getFileSystem().getUri().toString(), - 1, null, null, MR_UGI, new JobConf(mrConf)); - JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - // add garbage to Configs.SYSTEM_DIR - Path garbage = new Path(jobtracker.getSystemDir(), "garbage"); - fs.mkdirs(garbage); - fs.setPermission(garbage, new FsPermission(SYSTEM_DIR_PERMISSION)); - fs.setOwner(garbage, "test", "test-group"); - - // stop the jobtracker - mr.stopJobTracker(); - mr.getJobTrackerConf().setBoolean(JTConfig.JT_RESTART_ENABLED, - false); - // start jobtracker but dont wait for it to be up - mr.startJobTracker(false); - - // check 5 times .. each time wait for 2 secs to check if the jobtracker - // has crashed or not. - for (int i = 0; i < 5; ++i) { - LOG.info("Check #" + i); - if (!mr.getJobTrackerRunner().isActive()) { - return; - } - UtilsForTests.waitFor(2000); - } - - assertFalse("JobTracker did not bail out (waited for 10 secs)", - mr.getJobTrackerRunner().isActive()); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown();} - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java deleted file mode 100644 index 1b860437880..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.DataOutputStream; -import java.io.IOException; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapred.lib.NullOutputFormat; -import org.apache.hadoop.mapreduce.FileSystemCounter; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.examples.RandomWriter; -import org.apache.hadoop.examples.Sort; - -/** - * A JUnit test to test the Map-Reduce framework's sort - * with a Mini Map-Reduce Cluster with a Mini HDFS Clusters. - */ -public class TestMiniMRDFSSort extends TestCase { - // Input/Output paths for sort - private static final Path SORT_INPUT_PATH = new Path("/sort/input"); - private static final Path SORT_OUTPUT_PATH = new Path("/sort/output"); - - // Knobs to control randomwriter; and hence sort - private static final int NUM_HADOOP_SLAVES = 3; - // make it big enough to cause a spill in the map - private static final int RW_BYTES_PER_MAP = 3 * 1024 * 1024; - private static final int RW_MAPS_PER_HOST = 2; - - private static MiniMRCluster mrCluster = null; - private static MiniDFSCluster dfsCluster = null; - private static FileSystem dfs = null; - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestMiniMRDFSSort.class)) { - protected void setUp() throws Exception { - Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null); - dfs = dfsCluster.getFileSystem(); - mrCluster = new MiniMRCluster(NUM_HADOOP_SLAVES, - dfs.getUri().toString(), 1); - } - protected void tearDown() throws Exception { - if (dfsCluster != null) { dfsCluster.shutdown(); } - if (mrCluster != null) { mrCluster.shutdown(); } - } - }; - return setup; - } - - public static void runRandomWriter(JobConf job, Path sortInput) - throws Exception { - // Scale down the default settings for RandomWriter for the test-case - // Generates NUM_HADOOP_SLAVES * RW_MAPS_PER_HOST * RW_BYTES_PER_MAP - job.setInt(RandomWriter.BYTES_PER_MAP, RW_BYTES_PER_MAP); - job.setInt(RandomWriter.MAPS_PER_HOST, RW_MAPS_PER_HOST); - String[] rwArgs = {sortInput.toString()}; - - // Run RandomWriter - assertEquals(ToolRunner.run(job, new RandomWriter(), rwArgs), 0); - } - - private static void runSort(JobConf job, Path sortInput, Path sortOutput) - throws Exception { - - job.setInt(JobContext.JVM_NUMTASKS_TORUN, -1); - job.setInt(JobContext.IO_SORT_MB, 1); - job.setNumMapTasks(12); - - // Setup command-line arguments to 'sort' - String[] sortArgs = {sortInput.toString(), sortOutput.toString()}; - - // Run Sort - Sort sort = new Sort(); - assertEquals(ToolRunner.run(job, sort, sortArgs), 0); - org.apache.hadoop.mapreduce.Counters counters = sort.getResult().getCounters(); - long mapInput = counters.findCounter(FileInputFormatCounter.BYTES_READ) - .getValue(); - long hdfsRead = counters.findCounter("hdfs", FileSystemCounter.BYTES_READ) - .getValue(); - // the hdfs read should be between 100% and 110% of the map input bytes - assertTrue("map input = " + mapInput + ", hdfs read = " + hdfsRead, - (hdfsRead < (mapInput * 1.1)) && - (hdfsRead >= mapInput)); - } - - private static void runSortValidator(JobConf job, - Path sortInput, Path sortOutput) - throws Exception { - String[] svArgs = {"-sortInput", sortInput.toString(), - "-sortOutput", sortOutput.toString()}; - - // Run Sort-Validator - assertEquals(ToolRunner.run(job, new SortValidator(), svArgs), 0); - } - - private static class ReuseDetector extends MapReduceBase - implements Mapper { - static int instances = 0; - Reporter reporter = null; - - @Override - public void map(BytesWritable key, BytesWritable value, - OutputCollector output, - Reporter reporter) throws IOException { - this.reporter = reporter; - } - - public void close() throws IOException { - reporter.incrCounter("jvm", "use", ++instances); - } - } - - private static void runJvmReuseTest(JobConf job, - boolean reuse) throws IOException { - // setup a map-only job that reads the input and only sets the counters - // based on how many times the jvm was reused. - job.setInt(JobContext.JVM_NUMTASKS_TORUN, reuse ? -1 : 1); - FileInputFormat.setInputPaths(job, SORT_INPUT_PATH); - job.setInputFormat(SequenceFileInputFormat.class); - job.setOutputFormat(NullOutputFormat.class); - job.setMapperClass(ReuseDetector.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Text.class); - job.setNumMapTasks(24); - job.setNumReduceTasks(0); - RunningJob result = JobClient.runJob(job); - long uses = result.getCounters().findCounter("jvm", "use").getValue(); - int maps = job.getNumMapTasks(); - if (reuse) { - assertTrue("maps = " + maps + ", uses = " + uses, maps < uses); - } else { - assertEquals("uses should be number of maps", job.getNumMapTasks(), uses); - } - } - - public void testMapReduceSort() throws Exception { - // Run randomwriter to generate input for 'sort' - runRandomWriter(mrCluster.createJobConf(), SORT_INPUT_PATH); - - // Run sort - runSort(mrCluster.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH); - - // Run sort-validator to check if sort worked correctly - runSortValidator(mrCluster.createJobConf(), SORT_INPUT_PATH, - SORT_OUTPUT_PATH); - } - - public void testJvmReuse() throws Exception { - runJvmReuseTest(mrCluster.createJobConf(), true); - } - - public void testNoJvmReuse() throws Exception { - runJvmReuseTest(mrCluster.createJobConf(), false); - } - - private static class BadPartitioner - implements Partitioner { - boolean low; - public void configure(JobConf conf) { - low = conf.getBoolean("test.testmapred.badpartition", true); - } - public int getPartition(LongWritable k, Text v, int numPartitions) { - return low ? -1 : numPartitions; - } - } - - public void testPartitioner() throws Exception { - JobConf conf = mrCluster.createJobConf(); - conf.setPartitionerClass(BadPartitioner.class); - conf.setNumReduceTasks(3); - FileSystem fs = FileSystem.get(conf); - Path testdir = - new Path("blah").makeQualified(fs.getUri(), fs.getWorkingDirectory()); - Path inFile = new Path(testdir, "blah"); - DataOutputStream f = fs.create(inFile); - f.writeBytes("blah blah blah\n"); - f.close(); - FileInputFormat.setInputPaths(conf, inFile); - FileOutputFormat.setOutputPath(conf, new Path(testdir, "out")); - conf.setMapperClass(IdentityMapper.class); - conf.setReducerClass(IdentityReducer.class); - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - conf.setMaxMapAttempts(1); - - // partition too low - conf.setBoolean("test.testmapred.badpartition", true); - boolean pass = true; - try { - JobClient.runJob(conf); - } catch (IOException e) { - pass = false; - } - assertFalse("should fail for partition < 0", pass); - - // partition too high - conf.setBoolean("test.testmapred.badpartition", false); - pass = true; - try { - JobClient.runJob(conf); - } catch (IOException e) { - pass = false; - } - assertFalse("should fail for partition >= numPartitions", pass); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRLocalFS.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRLocalFS.java deleted file mode 100644 index 2788a373402..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRLocalFS.java +++ /dev/null @@ -1,348 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.util.Iterator; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.SecondarySort; -import org.apache.hadoop.examples.WordCount; -import org.apache.hadoop.examples.SecondarySort.FirstGroupingComparator; -import org.apache.hadoop.examples.SecondarySort.FirstPartitioner; -import org.apache.hadoop.examples.SecondarySort.IntPair; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.mapred.MRCaching.TestResult; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskCounter; -import org.apache.hadoop.mapreduce.TestMapReduceLocal; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.util.Progressable; - -/** - * A JUnit test to test min map-reduce cluster with local file system. - */ -public class TestMiniMRLocalFS extends TestCase { - private static String TEST_ROOT_DIR = - new File(System.getProperty("test.build.data","/tmp")) - .toURI().toString().replace(' ', '+'); - - public void testWithLocal() - throws IOException, InterruptedException, ClassNotFoundException { - MiniMRCluster mr = null; - try { - mr = new MiniMRCluster(2, "file:///", 3); - // make cleanup inline sothat validation of existence of these directories - // can be done - mr.setInlineCleanupThreads(); - - TestMiniMRWithDFS.runPI(mr, mr.createJobConf()); - - // run the wordcount example with caching - JobConf job = mr.createJobConf(); - TestResult ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input", - TEST_ROOT_DIR + "/wc/output", - TEST_ROOT_DIR + "/cachedir", - job, - "The quick brown fox\n" - + "has many silly\n" - + "red fox sox\n"); - // assert the number of lines read during caching - assertTrue("Failed test archives not matching", ret.isOutputOk); - // test the task report fetchers - JobClient client = new JobClient(job); - JobID jobid = ret.job.getID(); - TaskReport[] reports; - reports = client.getSetupTaskReports(jobid); - assertEquals("number of setups", 2, reports.length); - reports = client.getMapTaskReports(jobid); - assertEquals("number of maps", 1, reports.length); - reports = client.getReduceTaskReports(jobid); - assertEquals("number of reduces", 1, reports.length); - reports = client.getCleanupTaskReports(jobid); - assertEquals("number of cleanups", 2, reports.length); - Counters counters = ret.job.getCounters(); - assertEquals("number of map inputs", 3, - counters.getCounter(TaskCounter.MAP_INPUT_RECORDS)); - assertEquals("number of reduce outputs", 9, - counters.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS)); - runCustomFormats(mr); - runSecondarySort(mr.createJobConf()); - } finally { - if (mr != null) { mr.shutdown(); } - } - } - - private void runCustomFormats(MiniMRCluster mr) throws IOException { - JobConf job = mr.createJobConf(); - FileSystem fileSys = FileSystem.get(job); - Path testDir = new Path(TEST_ROOT_DIR + "/test_mini_mr_local"); - Path outDir = new Path(testDir, "out"); - System.out.println("testDir= " + testDir); - fileSys.delete(testDir, true); - - job.setInputFormat(MyInputFormat.class); - job.setOutputFormat(MyOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(IntWritable.class); - - job.setMapperClass(MyMapper.class); - job.setReducerClass(MyReducer.class); - job.setNumMapTasks(100); - job.setNumReduceTasks(1); - // explicitly do not use "normal" job.setOutputPath to make sure - // that it is not hardcoded anywhere in the framework. - job.set("non.std.out", outDir.toString()); - try { - JobClient.runJob(job); - String result = - MapReduceTestUtil.readOutput(outDir, job); - assertEquals("output", ("aunt annie\t1\n" + - "bumble boat\t4\n" + - "crocodile pants\t0\n" + - "duck-dog\t5\n"+ - "eggs\t2\n" + - "finagle the agent\t3\n"), result); - } finally { - fileSys.delete(testDir, true); - } - - } - - private static class MyInputFormat - implements InputFormat { - - static final String[] data = new String[]{ - "crocodile pants", - "aunt annie", - "eggs", - "finagle the agent", - "bumble boat", - "duck-dog", - }; - - private static class MySplit implements InputSplit { - int first; - int length; - - public MySplit() { } - - public MySplit(int first, int length) { - this.first = first; - this.length = length; - } - - public String[] getLocations() { - return new String[0]; - } - - public long getLength() { - return length; - } - - public void write(DataOutput out) throws IOException { - WritableUtils.writeVInt(out, first); - WritableUtils.writeVInt(out, length); - } - - public void readFields(DataInput in) throws IOException { - first = WritableUtils.readVInt(in); - length = WritableUtils.readVInt(in); - } - } - - static class MyRecordReader implements RecordReader { - int index; - int past; - int length; - - MyRecordReader(int index, int length) { - this.index = index; - this.past = index + length; - this.length = length; - } - - public boolean next(IntWritable key, Text value) throws IOException { - if (index < past) { - key.set(index); - value.set(data[index]); - index += 1; - return true; - } - return false; - } - - public IntWritable createKey() { - return new IntWritable(); - } - - public Text createValue() { - return new Text(); - } - - public long getPos() throws IOException { - return index; - } - - public void close() throws IOException {} - - public float getProgress() throws IOException { - return 1.0f - (past-index)/length; - } - } - - public InputSplit[] getSplits(JobConf job, - int numSplits) throws IOException { - return new MySplit[]{new MySplit(0, 1), new MySplit(1, 3), - new MySplit(4, 2)}; - } - - public RecordReader getRecordReader(InputSplit split, - JobConf job, - Reporter reporter) - throws IOException { - MySplit sp = (MySplit) split; - return new MyRecordReader(sp.first, sp.length); - } - - } - - static class MyMapper extends MapReduceBase - implements Mapper { - - public void map(WritableComparable key, Writable value, - OutputCollector out, - Reporter reporter) throws IOException { - System.out.println("map: " + key + ", " + value); - out.collect((WritableComparable) value, key); - InputSplit split = reporter.getInputSplit(); - if (split.getClass() != MyInputFormat.MySplit.class) { - throw new IOException("Got wrong split in MyMapper! " + - split.getClass().getName()); - } - } - } - - static class MyReducer extends MapReduceBase - implements Reducer { - public void reduce(WritableComparable key, Iterator values, - OutputCollector output, - Reporter reporter) throws IOException { - try { - InputSplit split = reporter.getInputSplit(); - throw new IOException("Got an input split of " + split); - } catch (UnsupportedOperationException e) { - // expected result - } - while (values.hasNext()) { - Writable value = values.next(); - System.out.println("reduce: " + key + ", " + value); - output.collect(key, value); - } - } - } - - static class MyOutputFormat implements OutputFormat { - static class MyRecordWriter implements RecordWriter { - private DataOutputStream out; - - public MyRecordWriter(Path outputFile, JobConf job) throws IOException { - out = outputFile.getFileSystem(job).create(outputFile); - } - - public void write(Object key, Object value) throws IOException { - out.writeBytes(key.toString() + "\t" + value.toString() + "\n"); - } - - public void close(Reporter reporter) throws IOException { - out.close(); - } - } - - public RecordWriter getRecordWriter(FileSystem ignored, JobConf job, - String name, - Progressable progress - ) throws IOException { - return new MyRecordWriter(new Path(job.get("non.std.out")), job); - } - - public void checkOutputSpecs(FileSystem ignored, - JobConf job) throws IOException { - } - } - - private void runSecondarySort(Configuration conf) throws IOException, - InterruptedException, - ClassNotFoundException { - FileSystem localFs = FileSystem.getLocal(conf); - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - TestMapReduceLocal.writeFile - ("in/part1", "-1 -4\n-3 23\n5 10\n-1 -2\n-1 300\n-1 10\n4 1\n" + - "4 2\n4 10\n4 -1\n4 -10\n10 20\n10 30\n10 25\n"); - Job job = Job.getInstance(conf, "word count"); - job.setJarByClass(WordCount.class); - job.setNumReduceTasks(2); - job.setMapperClass(SecondarySort.MapClass.class); - job.setReducerClass(SecondarySort.Reduce.class); - // group and partition by the first int in the pair - job.setPartitionerClass(FirstPartitioner.class); - job.setGroupingComparatorClass(FirstGroupingComparator.class); - - // the map output is IntPair, IntWritable - job.setMapOutputKeyClass(IntPair.class); - job.setMapOutputValueClass(IntWritable.class); - - // the reduce output is Text, IntWritable - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(IntWritable.class); - - FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); - FileOutputFormat.setOutputPath(job, new Path(TEST_ROOT_DIR + "/out")); - assertTrue(job.waitForCompletion(true)); - String out = TestMapReduceLocal.readFile("out/part-r-00000"); - assertEquals("------------------------------------------------\n" + - "4\t-10\n4\t-1\n4\t1\n4\t2\n4\t10\n" + - "------------------------------------------------\n" + - "10\t20\n10\t25\n10\t30\n", out); - out = TestMapReduceLocal.readFile("out/part-r-00001"); - assertEquals("------------------------------------------------\n" + - "-3\t23\n" + - "------------------------------------------------\n" + - "-1\t-4\n-1\t-2\n-1\t10\n-1\t300\n" + - "------------------------------------------------\n" + - "5\t10\n", out); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java deleted file mode 100644 index 2e16f844bc8..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRWithDFS.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.BufferedReader; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.FileSystemCounter; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskCounter; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; - -/** - * A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS. - */ -public class TestMiniMRWithDFS extends TestCase { - private static final Log LOG = - LogFactory.getLog(TestMiniMRWithDFS.class.getName()); - - static final int NUM_MAPS = 10; - static final int NUM_SAMPLES = 100000; - - public static class TestResult { - public String output; - public RunningJob job; - TestResult(RunningJob job, String output) { - this.job = job; - this.output = output; - } - } - public static TestResult launchWordCount(JobConf conf, - Path inDir, - Path outDir, - String input, - int numMaps, - int numReduces) throws IOException { - FileSystem inFs = inDir.getFileSystem(conf); - FileSystem outFs = outDir.getFileSystem(conf); - outFs.delete(outDir, true); - if (!inFs.mkdirs(inDir)) { - throw new IOException("Mkdirs failed to create " + inDir.toString()); - } - { - DataOutputStream file = inFs.create(new Path(inDir, "part-0")); - file.writeBytes(input); - file.close(); - } - conf.setJobName("wordcount"); - conf.setInputFormat(TextInputFormat.class); - - // the keys are words (strings) - conf.setOutputKeyClass(Text.class); - // the values are counts (ints) - conf.setOutputValueClass(IntWritable.class); - - conf.setMapperClass(WordCount.MapClass.class); - conf.setCombinerClass(WordCount.Reduce.class); - conf.setReducerClass(WordCount.Reduce.class); - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReduces); - RunningJob job = JobClient.runJob(conf); - return new TestResult(job, MapReduceTestUtil.readOutput(outDir, conf)); - } - - /** - * Make sure that there are exactly the directories that we expect to find. - * - *
- *
- * - * For e.g., if we want to check the existence of *only* the directories for - * user1's tasks job1-attempt1, job1-attempt2, job2-attempt1, we pass user1 as - * user, {job1, job1, job2, job3} as jobIds and {attempt1, attempt2, attempt1, - * attempt3} as taskDirs. - * - * @param mr the map-reduce cluster - * @param user the name of the job-owner - * @param jobIds the list of jobs - * @param taskDirs the task ids that should be present - */ - static void checkTaskDirectories(MiniMRCluster mr, String user, - String[] jobIds, String[] taskDirs) { - - mr.waitUntilIdle(); - int trackers = mr.getNumTaskTrackers(); - - List observedJobDirs = new ArrayList(); - List observedFilesInsideJobDir = new ArrayList(); - - for (int i = 0; i < trackers; ++i) { - - // Verify that mapred-local-dir and it's direct contents are valid - File localDir = new File(mr.getTaskTrackerLocalDir(i)); - assertTrue("Local dir " + localDir + " does not exist.", localDir - .isDirectory()); - LOG.info("Verifying contents of " + MRConfig.LOCAL_DIR + " " - + localDir.getAbsolutePath()); - - // Verify contents(user-dir) of tracker-sub-dir - File trackerSubDir = new File(localDir, TaskTracker.SUBDIR); - if (trackerSubDir.isDirectory()) { - - // Verify contents of user-dir and populate the job-dirs/attempt-dirs - // lists - File userDir = new File(trackerSubDir, user); - if (userDir.isDirectory()) { - LOG.info("Verifying contents of user-dir " - + userDir.getAbsolutePath()); - verifyContents(new String[] { TaskTracker.JOBCACHE, - TaskTracker.DISTCACHEDIR }, userDir.list()); - - File jobCacheDir = - new File(localDir, TaskTracker.getJobCacheSubdir(user)); - String[] jobDirs = jobCacheDir.list(); - observedJobDirs.addAll(Arrays.asList(jobDirs)); - - for (String jobDir : jobDirs) { - String[] attemptDirs = new File(jobCacheDir, jobDir).list(); - observedFilesInsideJobDir.addAll(Arrays.asList(attemptDirs)); - } - } - } - } - - // Now verify that only expected job-dirs and attempt-dirs are present. - LOG.info("Verifying the list of job directories"); - verifyContents(jobIds, observedJobDirs.toArray(new String[observedJobDirs - .size()])); - LOG.info("Verifying the list of task directories"); - // All taskDirs should be present in the observed list. Other files like - // job.xml etc may be present too, we are not checking them here. - for (int j = 0; j < taskDirs.length; j++) { - assertTrue( - "Expected task-directory " + taskDirs[j] + " is not present!", - observedFilesInsideJobDir.contains(taskDirs[j])); - } - } - - /** - * Check the list of expectedFiles against the list of observedFiles and make - * sure they both are the same. Duplicates can be present in either of the - * lists and all duplicate entries are treated as a single entity. - * - * @param expectedFiles - * @param observedFiles - */ - private static void verifyContents(String[] expectedFiles, - String[] observedFiles) { - boolean[] foundExpectedFiles = new boolean[expectedFiles.length]; - boolean[] validObservedFiles = new boolean[observedFiles.length]; - for (int j = 0; j < observedFiles.length; ++j) { - for (int k = 0; k < expectedFiles.length; ++k) { - if (expectedFiles[k].equals(observedFiles[j])) { - foundExpectedFiles[k] = true; - validObservedFiles[j] = true; - } - } - } - for (int j = 0; j < foundExpectedFiles.length; j++) { - assertTrue("Expected file " + expectedFiles[j] + " not found", - foundExpectedFiles[j]); - } - for (int j = 0; j < validObservedFiles.length; j++) { - assertTrue("Unexpected file " + observedFiles[j] + " found", - validObservedFiles[j]); - } - } - - public static void runPI(MiniMRCluster mr, JobConf jobconf) - throws IOException, InterruptedException, ClassNotFoundException { - LOG.info("runPI"); - double estimate = org.apache.hadoop.examples.QuasiMonteCarlo.estimatePi( - NUM_MAPS, NUM_SAMPLES, jobconf).doubleValue(); - double error = Math.abs(Math.PI - estimate); - assertTrue("Error in PI estimation "+error+" exceeds 0.01", (error < 0.01)); - String userName = UserGroupInformation.getLoginUser().getUserName(); - checkTaskDirectories(mr, userName, new String[] {}, new String[] {}); - } - - public static void runWordCount(MiniMRCluster mr, JobConf jobConf) - throws IOException { - LOG.info("runWordCount"); - // Run a word count example - // Keeping tasks that match this pattern - String pattern = - TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null); - jobConf.setKeepTaskFilesPattern(pattern); - TestResult result; - final Path inDir = new Path("./wc/input"); - final Path outDir = new Path("./wc/output"); - String input = "The quick brown fox\nhas many silly\nred fox sox\n"; - result = launchWordCount(jobConf, inDir, outDir, input, 3, 1); - assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + - "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output); - JobID jobid = result.job.getID(); - TaskAttemptID taskid = new TaskAttemptID( - new TaskID(jobid, TaskType.MAP, 1),0); - String userName = UserGroupInformation.getLoginUser().getUserName(); - - checkTaskDirectories(mr, userName, new String[] { jobid.toString() }, - new String[] { taskid.toString() }); - // test with maps=0 - jobConf = mr.createJobConf(); - input = "owen is oom"; - result = launchWordCount(jobConf, inDir, outDir, input, 0, 1); - assertEquals("is\t1\noom\t1\nowen\t1\n", result.output); - Counters counters = result.job.getCounters(); - long hdfsRead = counters.findCounter("HDFS", - FileSystemCounter.BYTES_READ).getValue(); - long hdfsWrite = counters.findCounter("HDFS", - FileSystemCounter.BYTES_WRITTEN).getValue(); - long rawSplitBytesRead = - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getCounter(); - assertEquals(result.output.length(), hdfsWrite); - // add the correction factor of 234 as the input split is also streamed - assertEquals(input.length() + rawSplitBytesRead, hdfsRead); - - // Run a job with input and output going to localfs even though the - // default fs is hdfs. - { - FileSystem localfs = FileSystem.getLocal(jobConf); - String TEST_ROOT_DIR = - new File(System.getProperty("test.build.data","/tmp")) - .toString().replace(' ', '+'); - Path localIn = localfs.makeQualified - (new Path(TEST_ROOT_DIR + "/local/in")); - Path localOut = localfs.makeQualified - (new Path(TEST_ROOT_DIR + "/local/out")); - result = launchWordCount(jobConf, localIn, localOut, - "all your base belong to us", 1, 1); - assertEquals("all\t1\nbase\t1\nbelong\t1\nto\t1\nus\t1\nyour\t1\n", - result.output); - assertTrue("outputs on localfs", localfs.exists(localOut)); - - } - } - - public void testWithDFS() - throws IOException, InterruptedException, ClassNotFoundException { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int taskTrackers = 4; - - Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 4, true, null); - fileSys = dfs.getFileSystem(); - mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); - // make cleanup inline sothat validation of existence of these directories - // can be done - mr.setInlineCleanupThreads(); - - runPI(mr, mr.createJobConf()); - runWordCount(mr, mr.createJobConf()); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); - } - } - } - - public void testWithDFSWithDefaultPort() throws IOException { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int taskTrackers = 4; - - Configuration conf = new Configuration(); - // start a dfs with the default port number - dfs = new MiniDFSCluster( - NameNode.DEFAULT_PORT, conf, 4, true, true, null, null); - fileSys = dfs.getFileSystem(); - mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); - - JobConf jobConf = mr.createJobConf(); - TestResult result; - final Path inDir = new Path("./wc/input"); - final Path outDir = new Path("hdfs://" + - dfs.getNameNode().getNameNodeAddress().getHostName() + - ":" + NameNode.DEFAULT_PORT +"/./wc/output"); - String input = "The quick brown fox\nhas many silly\nred fox sox\n"; - result = launchWordCount(jobConf, inDir, outDir, input, 3, 1); - assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + - "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output); - final Path outDir2 = new Path("hdfs:/test/wc/output2"); - jobConf.set("fs.default.name", "hdfs://localhost:" + NameNode.DEFAULT_PORT); - result = launchWordCount(jobConf, inDir, outDir2, input, 3, 1); - assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + - "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output); - } catch (java.net.BindException be) { - LOG.info("Skip the test this time because can not start namenode on port " - + NameNode.DEFAULT_PORT, be); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeHealthService.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeHealthService.java deleted file mode 100644 index 30449094ea7..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeHealthService.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.TimerTask; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.TaskTrackerStatus.TaskTrackerHealthStatus; - -import junit.framework.TestCase; - -public class TestNodeHealthService extends TestCase { - - private static volatile Log LOG = LogFactory - .getLog(TestNodeHealthService.class); - - private static final String nodeHealthConfigPath = System.getProperty( - "test.build.extraconf", "build/test/extraconf"); - - final static File nodeHealthConfigFile = new File(nodeHealthConfigPath, - "mapred-site.xml"); - - private String testRootDir = new File(System.getProperty("test.build.data", - "/tmp")).getAbsolutePath(); - - private File nodeHealthscriptFile = new File(testRootDir, "failingscript.sh"); - - @Override - protected void tearDown() throws Exception { - if (nodeHealthConfigFile.exists()) { - nodeHealthConfigFile.delete(); - } - if (nodeHealthscriptFile.exists()) { - nodeHealthscriptFile.delete(); - } - super.tearDown(); - } - - private Configuration getConfForNodeHealthScript() { - Configuration conf = new Configuration(); - conf.set(NodeHealthCheckerService.HEALTH_CHECK_SCRIPT_PROPERTY, - nodeHealthscriptFile.getAbsolutePath()); - conf.setLong(NodeHealthCheckerService.HEALTH_CHECK_INTERVAL_PROPERTY, 500); - conf.setLong( - NodeHealthCheckerService.HEALTH_CHECK_FAILURE_INTERVAL_PROPERTY, 1000); - return conf; - } - - private void writeNodeHealthScriptFile(String scriptStr, boolean setExecutable) - throws IOException { - PrintWriter pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile)); - pw.println(scriptStr); - pw.flush(); - pw.close(); - nodeHealthscriptFile.setExecutable(setExecutable); - } - - public void testNodeHealthScriptShouldRun() throws IOException { - // Node health script should not start if there is no property called - // node health script path. - assertFalse("Health checker should not have started", - NodeHealthCheckerService.shouldRun(new Configuration())); - Configuration conf = getConfForNodeHealthScript(); - // Node health script should not start if the node health script does not - // exists - assertFalse("Node health script should start", NodeHealthCheckerService - .shouldRun(conf)); - // Create script path. - conf.writeXml(new FileOutputStream(nodeHealthConfigFile)); - writeNodeHealthScriptFile("", false); - // Node health script should not start if the node health script is not - // executable. - assertFalse("Node health script should start", NodeHealthCheckerService - .shouldRun(conf)); - writeNodeHealthScriptFile("", true); - assertTrue("Node health script should start", NodeHealthCheckerService - .shouldRun(conf)); - } - - public void testNodeHealthScript() throws Exception { - TaskTrackerHealthStatus healthStatus = new TaskTrackerHealthStatus(); - String errorScript = "echo ERROR\n echo \"Tracker not healthy\""; - String normalScript = "echo \"I am all fine\""; - String timeOutScript = "sleep 4\n echo\"I am fine\""; - Configuration conf = getConfForNodeHealthScript(); - conf.writeXml(new FileOutputStream(nodeHealthConfigFile)); - - NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService( - conf); - TimerTask timer = nodeHealthChecker.getTimer(); - writeNodeHealthScriptFile(normalScript, true); - timer.run(); - - nodeHealthChecker.setHealthStatus(healthStatus); - LOG.info("Checking initial healthy condition"); - // Check proper report conditions. - assertTrue("Node health status reported unhealthy", healthStatus - .isNodeHealthy()); - assertTrue("Node health status reported unhealthy", healthStatus - .getHealthReport().isEmpty()); - - // write out error file. - // Healthy to unhealthy transition - writeNodeHealthScriptFile(errorScript, true); - // Run timer - timer.run(); - // update health status - nodeHealthChecker.setHealthStatus(healthStatus); - LOG.info("Checking Healthy--->Unhealthy"); - assertFalse("Node health status reported healthy", healthStatus - .isNodeHealthy()); - assertFalse("Node health status reported healthy", healthStatus - .getHealthReport().isEmpty()); - - // Check unhealthy to healthy transitions. - writeNodeHealthScriptFile(normalScript, true); - timer.run(); - nodeHealthChecker.setHealthStatus(healthStatus); - LOG.info("Checking UnHealthy--->healthy"); - // Check proper report conditions. - assertTrue("Node health status reported unhealthy", healthStatus - .isNodeHealthy()); - assertTrue("Node health status reported unhealthy", healthStatus - .getHealthReport().isEmpty()); - - // Healthy to timeout transition. - writeNodeHealthScriptFile(timeOutScript, true); - timer.run(); - nodeHealthChecker.setHealthStatus(healthStatus); - LOG.info("Checking Healthy--->timeout"); - assertFalse("Node health status reported healthy even after timeout", - healthStatus.isNodeHealthy()); - assertEquals("Node time out message not propogated", healthStatus - .getHealthReport(), - NodeHealthCheckerService.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java deleted file mode 100644 index d0e51dec025..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java +++ /dev/null @@ -1,490 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; - -/** - * Test node decommissioning and recommissioning via refresh. Also check if the - * nodes are decommissioned upon refresh. - */ -public class TestNodeRefresh extends TestCase { - private String namenode = null; - private MiniDFSCluster dfs = null; - private MiniMRCluster mr = null; - private JobTracker jt = null; - private String[] hosts = null; - private String[] trackerHosts = null; - private UserGroupInformation owner, user1, user2, user3, user4, user5; - private static final Log LOG = - LogFactory.getLog(TestNodeRefresh.class); - - private String getHostname(int i) { - return "host" + i + ".com"; - } - - private void startCluster(int numHosts, int numTrackerPerHost, - int numExcluded, UserGroupInformation clusterUgi, - Configuration conf) - throws IOException { - try { - // create fake mapping for the groups - owner = UserGroupInformation.getLoginUser(); - user1= UserGroupInformation.createUserForTesting("user1", - new String[] {"user1"}); - user2= UserGroupInformation.createUserForTesting("user2", - new String[] {"user2"}); - user3= UserGroupInformation.createUserForTesting("user3", - new String[] {"abc"}); - user4= UserGroupInformation.createUserForTesting("user4", - new String[] {"supergroup"}); - user5= UserGroupInformation.createUserForTesting("user5", - new String[] {"user5"}); - conf.setBoolean("dfs.replication.considerLoad", false); - - // prepare hosts info - hosts = new String[numHosts]; - for (int i = 1; i <= numHosts; ++i) { - hosts[i - 1] = getHostname(i); - } - - // start dfs - dfs = new MiniDFSCluster(conf, 1, true, null, hosts); - dfs.waitActive(); - dfs.startDataNodes(conf, numHosts, true, null, null, hosts, null); - dfs.waitActive(); - FileSystem.mkdirs(dfs.getFileSystem(), new Path("/"), - new FsPermission((short) 0777)); - - namenode = (dfs.getFileSystem()).getUri().getHost() + ":" + - (dfs.getFileSystem()).getUri().getPort(); - - // create tracker hosts - trackerHosts = new String[numHosts * numTrackerPerHost]; - for (int i = 1; i <= (numHosts * numTrackerPerHost); ++i) { - trackerHosts[i - 1] = getHostname(i); - } - - // start mini mr - JobConf jtConf = new JobConf(conf); - mr = new MiniMRCluster(0, 0, numHosts * numTrackerPerHost, namenode, 1, - null, trackerHosts, clusterUgi, jtConf, - numExcluded * numTrackerPerHost); - - jt = mr.getJobTrackerRunner().getJobTracker(); - - // check if trackers from all the desired hosts have connected - Set hostsSeen = new HashSet(); - for (TaskTrackerStatus status : jt.taskTrackers()) { - hostsSeen.add(status.getHost()); - } - assertEquals("Not all hosts are up", numHosts - numExcluded, - hostsSeen.size()); - } catch (IOException ioe) { - stopCluster(); - } - } - - private void stopCluster() { - hosts = null; - trackerHosts = null; - if (dfs != null) { - dfs.shutdown(); - dfs = null; - namenode = null; - } - if (mr != null) { - mr.shutdown(); - mr = null; - jt = null; - } - } - - private AdminOperationsProtocol getClient(Configuration conf, - UserGroupInformation ugi) - throws IOException { - return (AdminOperationsProtocol)RPC.getProxy(AdminOperationsProtocol.class, - AdminOperationsProtocol.versionID, JobTracker.getAddress(conf), ugi, - conf, NetUtils.getSocketFactory(conf, AdminOperationsProtocol.class)); - } - - /** - * Check default value of HOSTS_EXCLUDE. Also check if only - * owner is allowed to this command. - */ - public void testMRRefreshDefault() throws IOException { - // start a cluster with 2 hosts and no exclude-hosts file - Configuration conf = new Configuration(); - conf.set(JTConfig.JT_HOSTS_EXCLUDE_FILENAME, ""); - startCluster(2, 1, 0, UserGroupInformation.getLoginUser(),conf); - - conf = mr.createJobConf(new JobConf(conf)); - - // refresh with wrong user - AdminOperationsProtocol client = getClient(conf, user1); - boolean success = false; - try { - // Also try tool runner - client.refreshNodes(); - success = true; - } catch (IOException ioe) {} - assertFalse("Invalid user performed privileged refresh operation", success); - - // refresh with correct user - success = false; - client = getClient(conf, owner); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertTrue("Privileged user denied permission for refresh operation", - success); - - // refresh with invalid user - success = false; - client = getClient(conf, user4); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertFalse("Invalid user performed privileged refresh operation", - success); - - // check the cluster status and tracker size - assertEquals("Trackers are lost upon refresh with empty hosts.exclude", - 2, jt.getClusterStatus(false).getTaskTrackers()); - assertEquals("Excluded node count is incorrect", - 0, jt.getClusterStatus(false).getNumExcludedNodes()); - - // check if the host is disallowed - Set hosts = new HashSet(); - for (TaskTrackerStatus status : jt.taskTrackers()) { - hosts.add(status.getHost()); - } - assertEquals("Host is excluded upon refresh with empty hosts.exclude", - 2, hosts.size()); - - stopCluster(); - } - - /** - * Check refresh with a specific user/group set in the conf - */ - public void testMRSuperUsers() throws IOException { - // start a cluster with 1 host and specified cluster administrators - Configuration conf = new Configuration(); - // set the supergroup - conf.set(MRConfig.MR_SUPERGROUP, "supergroup"); - // set the admin acl - conf.set(MRConfig.MR_ADMINS, "user5 abc"); - startCluster(2, 1, 0, UserGroupInformation.createRemoteUser("user1"), conf); - - conf = mr.createJobConf(new JobConf(conf)); - - // refresh with wrong user - AdminOperationsProtocol client = getClient(conf, user2); - boolean success = false; - try { - // Also try tool runner - client.refreshNodes(); - success = true; - } catch (IOException ioe) {} - assertFalse("Invalid user performed privileged refresh operation", success); - - // refresh with correct user - success = false; - client = getClient(conf, user1); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertTrue("Privileged user denied permission for refresh operation", - success); - - // refresh with admin group - success = false; - client = getClient(conf, user3); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertTrue("Admin group member denied permission for refresh operation", - success); - - // refresh with admin user - success = false; - client = getClient(conf, user5); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertTrue("Admin user denied permission for refresh operation", - success); - - // refresh with deprecated super group member - success = false; - client = getClient(conf, user4); - try { - client.refreshNodes(); - success = true; - } catch (IOException ioe){} - assertTrue("Deprecated Super group member denied permission for refresh" + - " operation", success); - - stopCluster(); - } - - /** - * Check node refresh for decommissioning. Check if an allowed host is - * disallowed upon refresh. Also check if only owner/cluster administrator is - * allowed to fire this command. - */ - public void testMRRefreshDecommissioning() throws IOException { - // start a cluster with 2 hosts and empty exclude-hosts file - Configuration conf = new Configuration(); - File file = new File("hosts.exclude"); - file.delete(); - startCluster(2, 1, 0, UserGroupInformation.getLoginUser(), conf); - String hostToDecommission = getHostname(1); - conf = mr.createJobConf(new JobConf(conf)); - - // change the exclude-hosts file to include one host - FileOutputStream out = new FileOutputStream(file); - LOG.info("Writing excluded nodes to log file " + file.toString()); - BufferedWriter writer = null; - try { - writer = new BufferedWriter(new OutputStreamWriter(out)); - writer.write( hostToDecommission + "\n"); // decommission first host - } finally { - if (writer != null) { - writer.close(); - } - out.close(); - } - file.deleteOnExit(); - - AdminOperationsProtocol client = getClient(conf, owner); - try { - client.refreshNodes(); - } catch (IOException ioe){} - - // check the cluster status and tracker size - assertEquals("Tracker is not lost upon host decommissioning", - 1, jt.getClusterStatus(false).getTaskTrackers()); - assertEquals("Excluded node count is incorrect", - 1, jt.getClusterStatus(false).getNumExcludedNodes()); - - // check if the host is disallowed - for (TaskTrackerStatus status : jt.taskTrackers()) { - assertFalse("Tracker from decommissioned host still exist", - status.getHost().equals(hostToDecommission)); - } - - stopCluster(); - } - - /** - * Check node refresh for recommissioning. Check if an disallowed host is - * allowed upon refresh. - */ - public void testMRRefreshRecommissioning() throws IOException { - String hostToInclude = getHostname(1); - - // start a cluster with 2 hosts and exclude-hosts file having one hostname - Configuration conf = new Configuration(); - - // create a exclude-hosts file to include one host - File file = new File("hosts.exclude"); - file.delete(); - FileOutputStream out = new FileOutputStream(file); - LOG.info("Writing excluded nodes to log file " + file.toString()); - BufferedWriter writer = null; - try { - writer = new BufferedWriter(new OutputStreamWriter(out)); - writer.write(hostToInclude + "\n"); // exclude first host - } finally { - if (writer != null) { - writer.close(); - } - out.close(); - } - - startCluster(2, 1, 1, UserGroupInformation.getLoginUser(), conf); - - file.delete(); - - // change the exclude-hosts file to include no hosts - // note that this will also test hosts file with no content - out = new FileOutputStream(file); - LOG.info("Clearing hosts.exclude file " + file.toString()); - writer = null; - try { - writer = new BufferedWriter(new OutputStreamWriter(out)); - writer.write("\n"); - } finally { - if (writer != null) { - writer.close(); - } - out.close(); - } - file.deleteOnExit(); - - conf = mr.createJobConf(new JobConf(conf)); - - AdminOperationsProtocol client = getClient(conf, owner); - try { - client.refreshNodes(); - } catch (IOException ioe){} - - // start a tracker - mr.startTaskTracker(hostToInclude, null, 2, 1); - - // wait for the tracker to join the jt - while (jt.taskTrackers().size() < 2) { - UtilsForTests.waitFor(100); - } - - assertEquals("Excluded node count is incorrect", - 0, jt.getClusterStatus(false).getNumExcludedNodes()); - - // check if the host is disallowed - boolean seen = false; - for (TaskTrackerStatus status : jt.taskTrackers()) { - if(status.getHost().equals(hostToInclude)) { - seen = true; - break; - } - } - assertTrue("Tracker from excluded host doesnt exist", seen); - - stopCluster(); - } - - // Mapper that fails once for the first time - static class FailOnceMapper extends MapReduceBase implements - Mapper { - - private boolean shouldFail = false; - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - - if (shouldFail) { - throw new RuntimeException("failing map"); - } - } - - @Override - public void configure(JobConf conf) { - TaskAttemptID id = TaskAttemptID.forName(conf.get("mapred.task.id")); - shouldFail = id.getId() == 0 && id.getTaskID().getId() == 0; - } - } - - /** - * Check refreshNodes for decommissioning blacklisted nodes. - */ - public void testBlacklistedNodeDecommissioning() throws Exception { - LOG.info("Testing blacklisted node decommissioning"); - - Configuration conf = new Configuration(); - conf.set(JTConfig.JT_MAX_TRACKER_BLACKLISTS, "1"); - - startCluster(2, 1, 0, UserGroupInformation.getLoginUser(), conf); - - assertEquals("Trackers not up", 2, - mr.getJobTrackerRunner().getJobTracker().getActiveTrackers().length); - // validate the total tracker count - assertEquals("Active tracker count mismatch", - 2, jt.getClusterStatus(false).getTaskTrackers()); - // validate blacklisted count - assertEquals("Blacklisted tracker count mismatch", - 0, jt.getClusterStatus(false).getBlacklistedTrackers()); - - // run a failing job to blacklist the tracker - JobConf jConf = mr.createJobConf(); - jConf.set(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, "1"); - jConf.setJobName("test-job-fail-once"); - jConf.setMapperClass(FailOnceMapper.class); - jConf.setReducerClass(IdentityReducer.class); - jConf.setNumMapTasks(1); - jConf.setNumReduceTasks(0); - - RunningJob job = - UtilsForTests.runJob(jConf, new Path("in"), new Path("out")); - job.waitForCompletion(); - - // check if the tracker is lost - // validate the total tracker count - assertEquals("Active tracker count mismatch", - 1, jt.getClusterStatus(false).getTaskTrackers()); - // validate blacklisted count - assertEquals("Blacklisted tracker count mismatch", - 1, jt.getClusterStatus(false).getBlacklistedTrackers()); - - // find the tracker to decommission - String hostToDecommission = - JobInProgress.convertTrackerNameToHostName( - jt.getBlacklistedTrackers()[0].getTaskTrackerName()); - LOG.info("Decommissioning host " + hostToDecommission); - - Set decom = new HashSet(1); - decom.add(hostToDecommission); - jt.decommissionNodes(decom); - - // check the cluster status and tracker size - assertEquals("Tracker is not lost upon host decommissioning", - 1, jt.getClusterStatus(false).getTaskTrackers()); - assertEquals("Blacklisted tracker count incorrect in cluster status after " - + "decommissioning", - 0, jt.getClusterStatus(false).getBlacklistedTrackers()); - assertEquals("Tracker is not lost upon host decommissioning", - 1, jt.taskTrackers().size()); - - stopCluster(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java deleted file mode 100644 index 4e812ad1f55..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java +++ /dev/null @@ -1,253 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory; -import org.apache.hadoop.mapred.JobInProgress.KillInterruptedException; -import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -public class TestParallelInitialization extends TestCase { - - private static int jobCounter; - private static final int NUM_JOBS = 3; - IntWritable numJobsCompleted = new IntWritable(); - - static void resetCounters() { - jobCounter = 0; - } - - class FakeJobInProgress extends JobInProgress { - - public FakeJobInProgress(JobConf jobConf, - FakeTaskTrackerManager taskTrackerManager, - JobTracker jt) throws IOException { - super(new JobID("test", ++jobCounter), jobConf, jt); - this.startTime = System.currentTimeMillis(); - this.status = new JobStatus(getJobID(), 0f, 0f, JobStatus.PREP, - jobConf.getUser(), - jobConf.getJobName(), "", ""); - this.status.setJobPriority(JobPriority.NORMAL); - this.status.setStartTime(startTime); - this.jobHistory = new FakeJobHistory(); - } - - @Override - public synchronized void initTasks() throws IOException { - try { - int jobNumber = this.getJobID().getId(); - synchronized (numJobsCompleted) { - while (numJobsCompleted.get() != (NUM_JOBS - jobNumber)) { - numJobsCompleted.wait(); - } - numJobsCompleted.set(numJobsCompleted.get() + 1); - numJobsCompleted.notifyAll(); - LOG.info("JobNumber " + jobNumber + " succeeded"); - } - } catch (InterruptedException ie) {}; - this.status.setRunState(JobStatus.SUCCEEDED); - } - - @Override - synchronized void fail() { - this.status.setRunState(JobStatus.FAILED); - } - } - - static class FakeTaskTrackerManager implements TaskTrackerManager { - - int maps = 0; - int reduces = 0; - int maxMapTasksPerTracker = 2; - int maxReduceTasksPerTracker = 2; - List listeners = - new ArrayList(); - QueueManager queueManager; - - private Map trackers = - new HashMap(); - - public FakeTaskTrackerManager() { - JobConf conf = new JobConf(); - queueManager = new QueueManager(conf); - trackers.put("tt1", new TaskTrackerStatus("tt1", "tt1.host", 1, - new ArrayList(), 0, - maxMapTasksPerTracker, maxReduceTasksPerTracker)); - } - - public ClusterStatus getClusterStatus() { - int numTrackers = trackers.size(); - return new ClusterStatus(numTrackers, 0, - 10 * 60 * 1000, - maps, reduces, - numTrackers * maxMapTasksPerTracker, - numTrackers * maxReduceTasksPerTracker, - JobTrackerStatus.RUNNING); - } - - public int getNumberOfUniqueHosts() { - return 0; - } - - public Collection taskTrackers() { - return trackers.values(); - } - - public void addJobInProgressListener(JobInProgressListener listener) { - listeners.add(listener); - } - - public void removeJobInProgressListener(JobInProgressListener listener) { - listeners.remove(listener); - } - - - public QueueManager getQueueManager() { - return queueManager; - } - - public int getNextHeartbeatInterval() { - return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT; - } - - public void killJob(JobID jobid) { - return; - } - - public JobInProgress getJob(JobID jobid) { - return null; - } - - public boolean killTask(TaskAttemptID attemptId, boolean shouldFail) { - return true; - } - - public void initJob(JobInProgress job) { - try { - JobStatus prevStatus = (JobStatus)job.getStatus().clone(); - job.initTasks(); - if (job.isJobEmpty()) { - completeEmptyJob(job); - } else if (!job.isSetupCleanupRequired()) { - job.completeSetup(); - } - JobStatus newStatus = (JobStatus)job.getStatus().clone(); - if (prevStatus.getRunState() != newStatus.getRunState()) { - JobStatusChangeEvent event = - new JobStatusChangeEvent(job, EventType.RUN_STATE_CHANGED, prevStatus, - newStatus); - for (JobInProgressListener listener : listeners) { - listener.jobUpdated(event); - } - } - } catch (KillInterruptedException kie) { - killJob(job.getJobID()); - } catch (IOException ioe) { - failJob(job); - } - } - - private synchronized void completeEmptyJob(JobInProgress job) { - job.completeEmptyJob(); - } - - public synchronized void failJob(JobInProgress job) { - JobStatus prevStatus = (JobStatus)job.getStatus().clone(); - job.fail(); - JobStatus newStatus = (JobStatus)job.getStatus().clone(); - if (prevStatus.getRunState() != newStatus.getRunState()) { - JobStatusChangeEvent event = - new JobStatusChangeEvent(job, EventType.RUN_STATE_CHANGED, prevStatus, - newStatus); - for (JobInProgressListener listener : listeners) { - listener.jobUpdated(event); - } - } - } - - // Test methods - - public void submitJob(JobInProgress job) throws IOException { - for (JobInProgressListener listener : listeners) { - listener.jobAdded(job); - } - } - } - - protected JobConf jobConf; - protected TaskScheduler scheduler; - private FakeTaskTrackerManager taskTrackerManager; - - @Override - protected void setUp() throws Exception { - resetCounters(); - jobConf = new JobConf(); - taskTrackerManager = new FakeTaskTrackerManager(); - scheduler = createTaskScheduler(); - scheduler.setConf(jobConf); - scheduler.setTaskTrackerManager(taskTrackerManager); - scheduler.start(); - } - - @Override - protected void tearDown() throws Exception { - if (scheduler != null) { - scheduler.terminate(); - } - } - - protected TaskScheduler createTaskScheduler() { - return new JobQueueTaskScheduler(); - } - - public void testParallelInitJobs() throws IOException { - FakeJobInProgress[] jobs = new FakeJobInProgress[NUM_JOBS]; - - // Submit NUM_JOBS jobs in order. The init code will ensure - // that the jobs get inited in descending order of Job ids - // i.e. highest job id first and the smallest last. - // If we were not doing parallel init, the first submitted job - // will be inited first and that will hang - - for (int i = 0; i < NUM_JOBS; i++) { - jobs[i] = new FakeJobInProgress(jobConf, taskTrackerManager, - UtilsForTests.getJobTracker()); - jobs[i].getStatus().setRunState(JobStatus.PREP); - taskTrackerManager.submitJob(jobs[i]); - } - - try { - Thread.sleep(1000); - } catch (InterruptedException ie) {} - - for (int i = 0; i < NUM_JOBS; i++) { - assertTrue(jobs[i].getStatus().getRunState() == JobStatus.SUCCEEDED); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueAclsForCurrentUser.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueAclsForCurrentUser.java deleted file mode 100644 index 6518c9327b0..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueAclsForCurrentUser.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import junit.framework.TestCase; - -import org.apache.hadoop.mapreduce.MRConfig; - -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; -import org.apache.hadoop.security.UserGroupInformation; - -/** - * Unit test class to test queue acls - * - */ -public class TestQueueAclsForCurrentUser extends TestCase { - - private QueueManager queueManager; - private JobConf conf = null; - UserGroupInformation currentUGI = null; - String submitAcl = QueueACL.SUBMIT_JOB.getAclName(); - String adminAcl = QueueACL.ADMINISTER_JOBS.getAclName(); - - @Override - protected void tearDown() { - deleteQueuesConfigFile(); - } - - // No access for queues for the user currentUGI - private void setupConfForNoAccess() throws Exception { - currentUGI = UserGroupInformation.getLoginUser(); - String userName = currentUGI.getUserName(); - - String[] queueNames = {"qu1", "qu2"}; - // Only user u1 has access for queue qu1 - // Only group g2 has acls for the queue qu2 - createQueuesConfigFile( - queueNames, new String[]{"u1", " g2"}, new String[]{"u1", " g2"}); - - conf = new JobConf(); - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - - queueManager = new QueueManager(conf); - } - - /** - * sets up configuration for acls test. - * @return - */ - private void setupConf(boolean aclSwitch) throws Exception{ - currentUGI = UserGroupInformation.getLoginUser(); - String userName = currentUGI.getUserName(); - StringBuilder groupNames = new StringBuilder(""); - String[] ugiGroupNames = currentUGI.getGroupNames(); - int max = ugiGroupNames.length-1; - for(int j=0;j< ugiGroupNames.length;j++) { - groupNames.append(ugiGroupNames[j]); - if(j - * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; -import static org.apache.hadoop.mapred.QueueConfigurationParser.*; -import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; -import static org.junit.Assert.*; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.security.UserGroupInformation; -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.After; -import org.junit.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; -import java.util.Set; - - -public class TestQueueManager { - - private static final Log LOG = LogFactory.getLog( - TestQueueManager.class); - - @After - public void tearDown() throws Exception { - deleteQueuesConfigFile(); - } - - // create UGI with the given user name and the fixed group name "myGroup" - private UserGroupInformation createUGI(String userName) { - return UserGroupInformation.createUserForTesting( - userName, new String[]{"myGroup"}); - } - - @Test - public void testDefault() throws Exception { - deleteQueuesConfigFile(); - QueueManager qm = new QueueManager(); - Queue root = qm.getRoot(); - assertEquals(root.getChildren().size(), 1); - assertEquals(root.getChildren().iterator().next().getName(), "default"); - assertNull(root.getChildren().iterator().next().getChildren()); - } - - @Test - public void testXMLParsing() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - Set rootQueues = qm.getRoot().getChildren(); - List names = new ArrayList(); - for (Queue q : rootQueues) { - names.add(q.getName()); - } - - //Size of root. - assertEquals(rootQueues.size(), 2); - - //check root level queues - assertTrue(names.contains("q1")); - assertTrue(names.contains("p1")); - - - //check for leaf names - Set leafNames = qm.getLeafQueueNames(); - Queue p = qm.getQueue("p1"); - Set children = p.getChildren(); - assertTrue(children.size() == 2); - - //check leaf level queues - assertTrue( - leafNames.contains( - "p1" + NAME_SEPARATOR + "p11")); - assertTrue( - leafNames.contains( - "p1" + NAME_SEPARATOR + "p12")); - - - Queue q = qm.getQueue( - "p1" + NAME_SEPARATOR + "p12"); - - assertTrue( - q.getAcls().get( - toFullPropertyName( - q.getName(), ACL_SUBMIT_JOB_TAG)).isUserAllowed( - createUGI("u1"))); - - assertTrue( - q.getAcls().get( - toFullPropertyName( - q.getName(), - ACL_ADMINISTER_JOB_TAG)) - .isUserAllowed(createUGI("u2"))); - assertTrue(q.getState().equals(QueueState.STOPPED)); - } - - @Test - public void testhasAccess() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocumentWithAcls(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - - UserGroupInformation ugi; - // test for acls access when acls are set with * - ugi = createUGI("u1"); - assertTrue(qm.hasAccess("p1" + NAME_SEPARATOR + "p12", - QueueACL.SUBMIT_JOB, ugi)); - ugi = createUGI("u2"); - assertTrue(qm.hasAccess("p1" + NAME_SEPARATOR + "p12", - QueueACL.ADMINISTER_JOBS, ugi)); - - // test for acls access when acls are not set with * - ugi = createUGI("u1"); - assertTrue(qm.hasAccess("p1" + NAME_SEPARATOR + "p11", - QueueACL.SUBMIT_JOB, ugi)); - ugi = createUGI("u2"); - assertTrue(qm.hasAccess("p1" + NAME_SEPARATOR + "p11", - QueueACL.ADMINISTER_JOBS, ugi)); - - // Test for acls access when acls are not specified but acls are enabled. - // By default, the queue acls for any queue are empty. - ugi = createUGI("u1"); - assertFalse(qm.hasAccess("p1" + NAME_SEPARATOR + "p13", - QueueACL.SUBMIT_JOB, ugi)); - ugi = createUGI("u2"); - assertFalse(qm.hasAccess("p1" + NAME_SEPARATOR + "p13", - QueueACL.ADMINISTER_JOBS, ugi)); - - assertTrue(qm.isRunning("p1" + NAME_SEPARATOR + "p13")); - } - - @Test - public void testQueueView() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - - for (Queue queue : qm.getRoot().getChildren()) { - checkHierarchy(queue, qm); - } - } - - private void checkHierarchy(Queue queue, QueueManager queueManager) { - JobQueueInfo jobQueueInfo = queueManager.getJobQueueInfo(queue.getName()); - assertEquals(queue.getName(),jobQueueInfo.getQueueName()); - assertEquals(queue.getState(),jobQueueInfo.getState()); - if (queue.getChildren() !=null && queue.getChildren().size() > 0) { - for (Queue childQueue : queue.getChildren()) { - checkHierarchy(childQueue, queueManager); - } - } - } - - @Test - public void testhasAccessForParent() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - - UserGroupInformation ugi = createUGI("u1"); - assertFalse(qm.hasAccess("p1", QueueACL.SUBMIT_JOB, ugi)); - } - - @Test - public void testValidation() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - Element queues = createQueuesNode(doc); - Element q1 = createQueue(doc, "q1"); - - q1.appendChild(createAcls(doc, "acl-submit-job", "u1")); - q1.appendChild(createAcls(doc, "acl-administer-jobs", "u2")); - q1.appendChild(createQueue(doc, "p15")); - q1.appendChild(createQueue(doc, "p16")); - - queues.appendChild(q1); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - try { - new QueueManager(QUEUES_CONFIG_FILE_PATH, false); - fail("Should throw an exception as configuration is wrong "); - } catch (RuntimeException re) { - LOG.info(re.getMessage()); - } - } - - @Test - public void testInvalidName() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - Element queues = createQueuesNode(doc); - Element q1 = createQueue(doc, ""); - queues.appendChild(q1); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - try { - new QueueManager(QUEUES_CONFIG_FILE_PATH, false); - fail("Should throw an exception as configuration is wrong "); - } catch (Exception re) { - re.printStackTrace(); - LOG.info(re.getMessage()); - } - deleteQueuesConfigFile(); - doc = createDocument(); - queues = createQueuesNode(doc); - q1 = doc.createElement("queue"); - queues.appendChild(q1); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - try { - new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - fail("Should throw an exception as configuration is wrong "); - } catch (RuntimeException re) { - re.printStackTrace(); - LOG.info(re.getMessage()); - } - } - - @Test - public void testMissingConfigFile() throws Exception { - deleteQueuesConfigFile(); // deletes file - - try { - new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - fail("Should throw an exception for missing file when " + - "explicitly passed."); - } catch (RuntimeException re) { - } - - // If we just want to pick up the queues from the class loader - // it should fall through to the default. The class loader is set to - // load CONFIG for the "mapred-queues.xml" resource, but it's missing - // so should fall through to mapred-queues-default.xml - QueueManager qm = new QueueManager(); - List rootQueues = - qm.getRoot().getJobQueueInfo().getChildren(); - assertEquals(1, rootQueues.size()); - assertEquals("default", rootQueues.get(0).getQueueName()); - } - - @Test - public void testEmptyProperties() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - Element queues = createQueuesNode(doc); - Element q1 = createQueue(doc, "q1"); - Element p = createProperties(doc, null); - q1.appendChild(p); - queues.appendChild(q1); - } - - @Test - public void testEmptyFile() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - try { - new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - fail("Should throw an exception as configuration is wrong "); - } catch (Exception re) { - re.printStackTrace(); - LOG.info(re.getMessage()); - } - } - - @Test - public void testJobQueueInfoGeneration() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - - List rootQueues = - qm.getRoot().getJobQueueInfo().getChildren(); - assertEquals(rootQueues.size(), 2); - List names = new ArrayList(); - for (JobQueueInfo q : rootQueues) { - names.add(q.getQueueName()); - if (q.getQueueName().equals("q1")) { - Properties p = q.getProperties(); - assertEquals(p.getProperty("capacity"), "10"); - assertEquals(p.getProperty("maxCapacity"), "35"); - - assertTrue(q.getChildren().isEmpty()); - } else if (q.getQueueName().equals("p1")) { - List children = q.getChildren(); - assertEquals(children.size(), 2); - for (JobQueueInfo child : children) { - if (child.getQueueName().equals( - "p1" + NAME_SEPARATOR + "p12")) { - assertEquals( - child.getQueueState(), QueueState.STOPPED.getStateName()); - } else if (child.getQueueName().equals( - "p1" + NAME_SEPARATOR + "p11")) { - assertEquals( - child.getQueueState(), QueueState.RUNNING.getStateName()); - } else { - fail("Only 2 children"); - } - } - } else { - fail("Only 2 queues with q1 and p1 "); - } - } - } - - /** - * Test the refresh of queues. - * - * @throws Exception - */ - @Test - public void testRefresh() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, true); - Queue beforeRefreshRoot = qm.getRoot(); - //remove the file and create new one. - Set rootQueues = beforeRefreshRoot.getChildren(); - for (Queue qs : rootQueues) { - if (qs.getName().equals("q1")) { - - assertEquals(qs.getProperties().getProperty("capacity"), "10"); - assertEquals(qs.getProperties().getProperty("maxCapacity"), "35"); - - } else if (qs.getName().equals("p1")) { - - Set children = qs.getChildren(); - for (Queue child : children) { - if (child.getName().equals( - "p1" + NAME_SEPARATOR + "p12")) { - assertTrue( - child.getAcls().get( - toFullPropertyName( - child.getName(), ACL_SUBMIT_JOB_TAG)) - .isUserAllowed(createUGI("u1"))); - - assertTrue( - child.getAcls().get( - toFullPropertyName( - child.getName(), - ACL_ADMINISTER_JOB_TAG)) - .isUserAllowed(createUGI("u2"))); - assertTrue(child.getState().equals(QueueState.STOPPED)); - } else { - assertTrue(child.getState().equals(QueueState.RUNNING)); - } - } - } - } - deleteQueuesConfigFile(); - doc = createDocument(); - refreshSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueConfigurationParser cp = new QueueConfigurationParser(QUEUES_CONFIG_FILE_PATH, true); - qm.getRoot().isHierarchySameAs(cp.getRoot()); - qm.setQueues( - cp.getRoot().getChildren().toArray( - new Queue[cp.getRoot().getChildren().size()])); - Queue afterRefreshRoot = qm.getRoot(); - //remove the file and create new one. - rootQueues = afterRefreshRoot.getChildren(); - for (Queue qs : rootQueues) { - if (qs.getName().equals("q1")) { - - assertEquals(qs.getProperties().getProperty("capacity"), "70"); - assertEquals(qs.getProperties().getProperty("maxCapacity"), "35"); - - } else if (qs.getName().equals("p1")) { - - Set children = qs.getChildren(); - for (Queue child : children) { - if (child.getName().equals( - "p1" + NAME_SEPARATOR + "p12")) { - assertTrue( - child.getAcls().get( - toFullPropertyName( - child.getName(), - ACL_SUBMIT_JOB_TAG)) - .isUserAllowed(createUGI("u3"))); - - assertTrue( - child.getAcls().get( - toFullPropertyName( - child.getName(), - ACL_ADMINISTER_JOB_TAG)) - .isUserAllowed(createUGI("u4"))); - assertTrue(child.getState().equals(QueueState.RUNNING)); - } else { - assertTrue(child.getState().equals(QueueState.STOPPED)); - } - } - } - } - } - - @Test - public void testRefreshWithInvalidFile() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - QueueManager qm = new QueueManager(QUEUES_CONFIG_FILE_PATH, false); - - deleteQueuesConfigFile(); - doc = createDocument(); - Element queues = createQueuesNode(doc); - Element q1 = createQueue(doc, ""); - queues.appendChild(q1); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - try { - QueueConfigurationParser cp = new QueueConfigurationParser(QUEUES_CONFIG_FILE_PATH, false); - - fail("Should throw an exception as configuration is wrong "); - } catch (Throwable re) { - re.printStackTrace(); - LOG.info(re.getMessage()); - } - } - - /** - * Class to store the array of queues retrieved by parsing the string - * that is dumped in Json format - */ - static class JsonQueueTree { - boolean acls_enabled; - - JsonQueue[] queues; - - public JsonQueue[] getQueues() { - return queues; - } - - public void setQueues(JsonQueue[] queues) { - this.queues = queues; - } - - public boolean isAcls_enabled() { - return acls_enabled; - } - - public void setAcls_enabled(boolean aclsEnabled) { - acls_enabled = aclsEnabled; - } - } - - /** - * Class to store the contents of each queue that is dumped in JSON format. - */ - static class JsonQueue { - String name; - String state; - String acl_submit_job; - String acl_administer_jobs; - JsonProperty[] properties; - JsonQueue[] children; - public String getName() { - return name; - } - public String getState() { - return state; - } - public JsonProperty[] getProperties() { - return properties; - } - public JsonQueue[] getChildren() { - return children; - } - public void setName(String name) { - this.name = name; - } - public void setState(String state) { - this.state = state; - } - public void setProperties(JsonProperty[] properties) { - this.properties = properties; - } - public void setChildren(JsonQueue[] children) { - this.children = children; - } - public String getAcl_submit_job() { - return acl_submit_job; - } - public void setAcl_submit_job(String aclSubmitJob) { - acl_submit_job = aclSubmitJob; - } - public String getAcl_administer_jobs() { - return acl_administer_jobs; - } - public void setAcl_administer_jobs(String aclAdministerJobs) { - acl_administer_jobs = aclAdministerJobs; - } - } - - /** - * Class to store the contents of attribute "properties" in Json dump - */ - static class JsonProperty { - String key; - String value; - public String getKey() { - return key; - } - public void setKey(String key) { - this.key = key; - } - public String getValue() { - return value; - } - public void setValue(String value) { - this.value = value; - } - } - - /** - * checks the format of the dump in JSON format when - * QueueManager.dumpConfiguration(Writer) is called. - * @throws Exception - */ - @Test - public void testDumpConfiguration() throws Exception { - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - - StringWriter out = new StringWriter(); - Configuration conf = new Configuration(false); - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - QueueManager.dumpConfiguration(out, QUEUES_CONFIG_FILE_PATH, conf); - - ObjectMapper mapper = new ObjectMapper(); - // parse the Json dump - JsonQueueTree queueTree = - mapper.readValue(out.toString(), JsonQueueTree.class); - - // check for the number of top-level queues - assertEquals(2, queueTree.getQueues().length); - - HashMap topQueues = new HashMap(); - for (JsonQueue topQueue : queueTree.getQueues()) { - topQueues.put(topQueue.getName(), topQueue); - } - - // check for consistency in number of children - assertEquals(2, topQueues.get("p1").getChildren().length); - - HashMap childQueues = new HashMap(); - for (JsonQueue child : topQueues.get("p1").getChildren()) { - childQueues.put(child.getName(), child); - } - - // check for consistency in state - assertEquals("stopped", childQueues.get("p1:p12").getState()); - - // check for consistency in properties - HashMap q1_properties = - new HashMap(); - for (JsonProperty prop : topQueues.get("q1").getProperties()) { - q1_properties.put(prop.getKey(), prop); - } - assertEquals("10", q1_properties.get("capacity").getValue()); - assertEquals("35", q1_properties.get("maxCapacity").getValue()); - - // check for acls - assertEquals("u1 ", childQueues.get("p1:p12").getAcl_submit_job()); - assertEquals("u2 ", childQueues.get("p1:p12").getAcl_administer_jobs()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java deleted file mode 100644 index b46c75839cd..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import static junit.framework.Assert.assertTrue; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.fail; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; - -import org.junit.After; -import org.junit.Test; - -/** - * Test the refresh feature of QueueManager. - */ -public class TestQueueManagerRefresh { - - private static final Log LOG = - LogFactory.getLog(TestQueueManagerRefresh.class); - - /** - * Remove the configuration file after the test's done. - */ - @After - public void tearDown() { - deleteQueuesConfigFile(); - } - - /** - * @return a simple hierarchy of JobQueueInfos - */ - static JobQueueInfo[] getSimpleQueueHierarchy() { - int numQs = 3; - JobQueueInfo[] queues = new JobQueueInfo[numQs]; - queues[0] = - newJobQueueInfo(new ArrayList(), null, "q1", - QueueState.UNDEFINED, null); - queues[1] = - newJobQueueInfo(new ArrayList(), null, "q1:q2", - QueueState.RUNNING, null); - queues[2] = - newJobQueueInfo(new ArrayList(), null, "q1:q3", - QueueState.RUNNING, null); - queues[0].addChild(queues[1]); - queues[0].addChild(queues[2]); - return queues; - } - - /** - * Test to verify that the refresh of queue properties fails if a new queue is - * added. - * - * @throws Exception - */ - @Test - public void testRefreshWithAddedQueues() - throws Exception { - - JobQueueInfo[] queues = getSimpleQueueHierarchy(); - - // write the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - QueueManager qManager = new QueueManager(); - - JobQueueInfo newQueue = - newJobQueueInfo(new ArrayList(), null, "q4", - QueueState.UNDEFINED, null); - queues[0].addChild(newQueue); - - // Rewrite the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - testRefreshFailureWithChangeOfHierarchy(qManager); - - } - - /** - * Test to verify that the refresh of queue properties fails if queues are - * removed. - * - * @throws Exception - */ - @Test - public void testRefreshWithRemovedQueues() - throws Exception { - - JobQueueInfo[] queues = getSimpleQueueHierarchy(); - - // write the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - QueueManager qManager = new QueueManager(); - - // Remove queue[2] - JobQueueInfo q2 = queues[2]; - queues[0].removeChild(q2); - - // Rewrite the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - testRefreshFailureWithChangeOfHierarchy(qManager); - } - - /** - * @param originalQManager - * @throws Exception - */ - private void testRefreshFailureWithChangeOfHierarchy( - QueueManager originalQManager) - throws Exception { - - // Make sure that isHierarchySame returns false. - QueueManager modifiedQueueManager = new QueueManager(); - assertFalse("Hierarchy changed after refresh!", - originalQManager.getRoot().isHierarchySameAs( - modifiedQueueManager.getRoot())); - - // Refresh the QueueManager and make sure it fails. - try { - originalQManager.refreshQueues(null, null); - fail("Queue-refresh should have failed!"); - } catch (Exception e) { - // Refresh failed as expected. Check the error message. - assertTrue( - "Exception message should point to a change in queue hierarchy!", - e.getMessage().contains( - QueueManager.MSG_REFRESH_FAILURE_WITH_CHANGE_OF_HIERARCHY)); - } - - // Make sure that the old configuration is retained. - List rootQueues = - originalQManager.getRoot().getJobQueueInfo().getChildren(); - assertTrue(rootQueues.size() == 1); - } - - /** - * Test to verify that the refresh of queue properties fails if scheduler - * fails to reload itself. - * - * @throws Exception - */ - // @Test - public void testRefreshWithSchedulerFailure() - throws Exception { - JobQueueInfo[] queues = getSimpleQueueHierarchy(); - - // write the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - QueueManager qManager = new QueueManager(); - - // No change in configuration. Just Refresh the QueueManager and make sure - // it fails. - try { - qManager.refreshQueues(null, - new MyTaskScheduler().new MyFailingQueueRefresher()); - fail("Queue-refresh should have failed!"); - } catch (Exception e) { - // Refresh failed as expected. Check the error message. - assertTrue( - "Exception message should point to a refresh-failure in scheduler!", - e.getMessage().contains( - QueueManager.MSG_REFRESH_FAILURE_WITH_SCHEDULER_FAILURE)); - } - } - - /** - * Test to verify that the refresh of scheduler properties passes smoothly. - * - * @throws Exception - */ - @Test - public void testRefreshOfSchedulerProperties() - throws Exception { - JobQueueInfo[] queues = getSimpleQueueHierarchy(); - - // Set some scheduler properties - for (JobQueueInfo jqi : queues) { - Properties props = new Properties(); - props.setProperty("testing.property", "testing.value." - + jqi.getQueueName()); - jqi.setProperties(props); - } - - // write the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - QueueManager qManager = new QueueManager(); - - MyTaskScheduler myScheduler = new MyTaskScheduler(); - - qManager.refreshQueues(null, myScheduler.new MyQueueRefresher()); - - // Verify that the scheduler props are set correctly by scheduler-refresh. - Map schedProps = myScheduler.getSchedulerProperties(); - for (JobQueueInfo jqi : queues) { - String expectedVal = "testing.value." + jqi.getQueueName(); - Properties qProperties = schedProps.get(jqi.getQueueName()); - assertNotNull("Properties should not be null for the SchedulerQueue " - + jqi.getQueueName(), qProperties); - String observedVal = qProperties.getProperty("testing.property"); - assertEquals("Properties for the SchedulerQueue " + jqi.getQueueName() - + " are not reloaded properly!", expectedVal, observedVal); - } - } - - /** - * Test to verify that the scheduling information per queue in the - * {@link QueueManager} is retained across queue-refresh. - * - * @throws Exception - */ - @Test - public void testSchedulingInfoAfterRefresh() - throws Exception { - - JobQueueInfo[] queues = getSimpleQueueHierarchy(); - - // write the configuration file - writeQueueConfigurationFile( - QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] }); - - QueueManager qManager = new QueueManager(); - - // Set some scheduling information for the queues in the QueueManager. - for (String qName : qManager.getLeafQueueNames()) { - qManager.setSchedulerInfo(qName, new String( - "scheduling-information-for-queue-" + qName)); - } - - qManager.refreshQueues(null, null); - - // Verify that the scheduling information is retained across refresh. - for (String qName : qManager.getLeafQueueNames()) { - assertEquals("scheduling-information-for-queue-" + qName, - qManager.getSchedulerInfo(qName)); - } - } - - static class MyTaskScheduler extends TaskScheduler { - - Map schedulerPropsMap = - new HashMap(); - - Map getSchedulerProperties() { - return schedulerPropsMap; - } - - class MyQueueRefresher extends QueueRefresher { - - private void updateSchedulerProps(JobQueueInfo jqi) { - LOG.info("Updating properties for SchedulerQueue " - + jqi.getQueueName()); - LOG.info("Putting " + jqi.getProperties() + " in " - + jqi.getQueueName()); - schedulerPropsMap.put(jqi.getQueueName(), jqi.getProperties()); - for (JobQueueInfo child : jqi.getChildren()) { - updateSchedulerProps(child); - } - } - - @Override - void refreshQueues(List newRootQueues) { - LOG.info("Refreshing scheduler's properties"); - for (JobQueueInfo jqi : newRootQueues) { - updateSchedulerProps(jqi); - } - } - } - - class MyFailingQueueRefresher extends QueueRefresher { - @Override - void refreshQueues(List newRootQueues) - throws Throwable { - throw new IOException("Scheduler cannot refresh the queues!"); - } - } - - @Override - public List assignTasks(TaskTracker taskTracker) { - return null; - } - - @Override - public Collection getJobs(String queueName) { - return null; - } - } - - static JobQueueInfo newJobQueueInfo(List children, - Properties props, String queueName, QueueState state, - String schedulingInfo) { - JobQueueInfo jqi = new JobQueueInfo(); - jqi.setChildren(children); - if (props != null) { - jqi.setProperties(props); - } - jqi.setQueueName(queueName); - jqi.setQueueState(state.getStateName()); - jqi.setSchedulingInfo(schedulingInfo); - return jqi; - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java deleted file mode 100644 index 4e60a040a90..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TreeSet; - -import javax.security.auth.login.LoginException; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.security.UserGroupInformation; -import static org.apache.hadoop.mapred.DeprecatedQueueConfigurationParser.*; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; -import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; - -public class TestQueueManagerWithDeprecatedConf extends TestCase { - - static final Log LOG = LogFactory.getLog( - TestQueueManagerWithDeprecatedConf.class); - - String submitAcl = QueueACL.SUBMIT_JOB.getAclName(); - String adminAcl = QueueACL.ADMINISTER_JOBS.getAclName(); - - - public void testMultipleQueues() { - JobConf conf = new JobConf(); - conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, - "q1,q2,Q3"); - QueueManager qMgr = new QueueManager(conf); - Set expQueues = new TreeSet(); - expQueues.add("q1"); - expQueues.add("q2"); - expQueues.add("Q3"); - verifyQueues(expQueues, qMgr.getLeafQueueNames()); - } - - public void testSchedulerInfo() { - JobConf conf = new JobConf(); - conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, - "qq1,qq2"); - QueueManager qMgr = new QueueManager(conf); - qMgr.setSchedulerInfo("qq1", "queueInfoForqq1"); - qMgr.setSchedulerInfo("qq2", "queueInfoForqq2"); - assertEquals(qMgr.getSchedulerInfo("qq2"), "queueInfoForqq2"); - assertEquals(qMgr.getSchedulerInfo("qq1"), "queueInfoForqq1"); - } - - - public void testQueueManagerWithDeprecatedConf() throws IOException { - String queueConfigPath = - System.getProperty("test.build.extraconf", "build/test/extraconf"); - - File hadoopConfigFile = new File(queueConfigPath, "mapred-site.xml"); - try { - // queue properties with which the cluster is started. - Properties hadoopConfProps = new Properties(); - hadoopConfProps.put(DeprecatedQueueConfigurationParser. - MAPRED_QUEUE_NAMES_KEY, "default,q1,q2"); - hadoopConfProps.put(MRConfig.MR_ACLS_ENABLED, "true"); - - UserGroupInformation ugi = - UserGroupInformation.createRemoteUser("unknownUser"); - hadoopConfProps.put(toFullPropertyName( - "default", submitAcl), ugi.getUserName()); - hadoopConfProps.put(toFullPropertyName( - "q1", submitAcl), "u1"); - hadoopConfProps.put(toFullPropertyName( - "q2", submitAcl), "*"); - hadoopConfProps.put(toFullPropertyName( - "default", adminAcl), ugi.getUserName()); - hadoopConfProps.put(toFullPropertyName( - "q1", adminAcl), "u2"); - hadoopConfProps.put(toFullPropertyName( - "q2", adminAcl), "*"); - - UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile); - - Configuration conf = new JobConf(); - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - QueueManager queueManager = new QueueManager(conf); - //Testing job submission access to queues. - assertTrue("User Job Submission failed.", - queueManager.hasAccess("default", QueueACL. - SUBMIT_JOB, ugi)); - assertFalse("User Job Submission failed.", - queueManager.hasAccess("q1", QueueACL. - SUBMIT_JOB, ugi)); - assertTrue("User Job Submission failed.", - queueManager.hasAccess("q2", QueueACL. - SUBMIT_JOB, ugi)); - //Testing the administer-jobs acls - assertTrue("User Job Submission failed.", - queueManager.hasAccess("default", - QueueACL.ADMINISTER_JOBS, ugi)); - assertFalse("User Job Submission failed.", - queueManager.hasAccess("q1", QueueACL. - ADMINISTER_JOBS, ugi)); - assertTrue("User Job Submission failed.", - queueManager.hasAccess("q2", QueueACL. - ADMINISTER_JOBS, ugi)); - - } finally { - //Cleanup the configuration files in all cases - if(hadoopConfigFile.exists()) { - hadoopConfigFile.delete(); - } - } - } - - private void verifyQueues(Set expectedQueues, - Set actualQueues) { - assertEquals(expectedQueues.size(), actualQueues.size()); - for (String queue : expectedQueues) { - assertTrue(actualQueues.contains(queue)); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java deleted file mode 100644 index 2bf615069f2..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import static org.apache.hadoop.mapred.QueueConfigurationParser.NAME_SEPARATOR; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.QUEUES_CONFIG_FILE_PATH; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.deleteQueuesConfigFile; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createAcls; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createDocument; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createProperties; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueue; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesNode; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createSimpleDocumentWithAcls; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createState; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.miniMRCluster; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.submitSleepJob; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.writeToFile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.tools.MRAdmin; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.QueueState; -import org.apache.hadoop.mapreduce.JobStatus.State; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.AfterClass; -import org.junit.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -public class TestQueueManagerWithJobTracker { - - private static Configuration conf; - - @AfterClass - public static void tearDown() throws Exception { - deleteQueuesConfigFile(); - } - - String adminUser = "adminUser"; - String adminGroup = "adminGroup"; - String deprecatedSuperGroup = "superGroup"; - - private void startCluster(boolean aclsEnabled) - throws Exception { - - deleteQueuesConfigFile(); - Document doc = createDocument(); - createSimpleDocumentWithAcls(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - conf = new Configuration(); - conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false"); - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled); - conf.set(MRConfig.MR_SUPERGROUP, deprecatedSuperGroup); - conf.set(MRConfig.MR_ADMINS, adminUser + " " + adminGroup); - - JobConf jobConf = new JobConf(conf); - String namenode = "file:///"; - miniMRCluster = new MiniMRCluster(0, namenode, 3, null, null, jobConf); - - } - - /** - * Test to check that jobs cannot be submitted to a queue in STOPPED state - * @throws Exception - */ - @Test(expected = IOException.class) - public void testSubmitJobForStoppedQueue() throws Exception { - startCluster(true); - - submitSleepJob(10, 10, 100, 100, false, null, - "p1" + NAME_SEPARATOR + "p14", conf); - fail("queue p1:p14 is in stopped state and should not accept jobs"); - } - - /** - * Test to check that jobs cannot be submitted to a container queue - * @throws Exception - */ - @Test(expected = IOException.class) - public void testSubmitJobForContainerQueue() throws Exception { - startCluster(true); - - submitSleepJob(10, 10, 100, 100, false, null, "p1", conf); - fail("queue p1 is a container queue and cannot have jobs"); - } - - /** - * Tests the submission of job with specified acls - * @throws Exception - */ - @Test - public void testAclsForSubmitJob() throws Exception { - startCluster(true); - - Job job; - try { - // submit job to queue p1:p13 with unspecified acls - job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR - + "p13", conf); - fail("user u1 cannot submit jobs to queue p1:p13"); - } catch (Exception e) { - } - - // check access to admins - job = submitSleepJob(0, 0, 0, 0, true, adminUser+ ",g1", - "p1" + NAME_SEPARATOR + "p13", conf); - assertTrue("Admin user cannot submit jobs to queue p1:p13", - job.isSuccessful()); - job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ adminGroup, - "p1" + NAME_SEPARATOR + "p13", conf); - assertTrue("Admin group member cannot submit jobs to queue p1:p13", - job.isSuccessful()); - job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ deprecatedSuperGroup, - "p1" + NAME_SEPARATOR + "p13", conf); - assertTrue("Deprecated super group member cannot submit jobs to queue" + - " p1:p13", job.isSuccessful()); - - // check for access to submit the job - try { - job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR - + "p11", conf); - fail("user u2 cannot submit jobs to queue p1:p11"); - } catch (Exception e) { - } - // submit job to queue p1:p11 with acl-submit-job as u1 - job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" - + NAME_SEPARATOR + "p11", conf); - assertTrue("Job submission for u1 failed in queue : p1:p11.", - job.isSuccessful()); - - // submit job to queue p1:p12 with acl-submit-job as * - job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" - + NAME_SEPARATOR + "p12", conf); - assertTrue("Job submission for u2 failed in queue : p1:p12.", - job.isSuccessful()); - } - - /** - * Tests the accessibility to kill a job - * @throws Exception - */ - @Test - public void testAccessToKillJob() throws Exception { - startCluster(true); - - Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", "p1" - + NAME_SEPARATOR + "p11", conf); - final JobConf jobConf = miniMRCluster.createJobConf(); - Cluster cluster = null; - JobID jobID = job.getStatus().getJobID(); - //Ensure that the jobinprogress is initied before we issue a kill - //signal to the job. - JobTracker tracker = miniMRCluster.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = tracker.getJob(org.apache.hadoop.mapred.JobID - .downgrade(jobID)); - tracker.initJob(jip); - try { - final Configuration userConf = - new Configuration(miniMRCluster.createJobConf()); - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting("someRandomUser", - new String[] { "someRandomGroup" }); - cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(userConf); - } - }); - cluster.getJob(jobID).killJob(); - fail("user 'someRandomeUser' is neither u1 nor in the administer group list"); - } catch (Exception e) { - final Configuration userConf = new Configuration(miniMRCluster.createJobConf()); - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting("u1",new String[]{"g1"}); - cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(userConf); - } - }); - cluster.getJob(jobID).killJob(); - // kill the running job - assertEquals("job submitted for u1 and queue p1:p11 is not killed.", - cluster.getJob(jobID).getStatus().getState(), (State.KILLED)); - } - - job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", "p1" + NAME_SEPARATOR - + "p12", conf); - jobID = job.getStatus().getJobID(); - //Ensure that the jobinprogress is initied before we issue a kill - //signal to the job. - jip = tracker.getJob(org.apache.hadoop.mapred.JobID.downgrade(jobID)); - tracker.initJob(jip); - tracker.killJob(job.getJobID()); - // kill the job by the user who submitted the job - assertEquals("job submitted for u1 and queue p1:p11 is not killed.", - cluster.getJob(jobID).getStatus().getState(), (State.KILLED)); - - final Configuration userConf = new Configuration(miniMRCluster.createJobConf()); - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting("u1",new String[]{"g1"}); - cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(userConf); - } - }); - job = submitSleepJob(1, 1, 10, 10, false, "u1,g1", "p1" + NAME_SEPARATOR - + "p11", conf); - jobID = job.getStatus().getJobID(); - //Ensure that the jobinprogress is initied before we issue a kill - //signal to the job. - jip = tracker.getJob(org.apache.hadoop.mapred.JobID.downgrade(jobID)); - tracker.initJob(jip); - ugi = - UserGroupInformation.createUserForTesting("u3",new String[]{"g3"}); - cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(jobConf); - } - }); - // try killing job with user not in administer list - try { - cluster.getJob(jobID).killJob(); - fail("u3 not in administer list"); - } catch (Exception e) { - ugi = - UserGroupInformation.createUserForTesting("u1",new String[]{"g1"}); - cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(jobConf); - } - }); - assertFalse(cluster.getJob(jobID).isComplete()); - cluster.getJob(jobID).killJob(); - // kill the running job - assertEquals("job submitted for u1 and queue p1:p11 is not killed.", - cluster.getJob(jobID).getStatus().getState(), (State.KILLED)); - } - // check kill access to admins - ugi = - UserGroupInformation.createUserForTesting("adminUser", new String[]{"g3"}); - checkAccessToKill(tracker, jobConf, ugi); - - ugi = - UserGroupInformation.createUserForTesting("u3", new String[]{adminGroup}); - checkAccessToKill(tracker, jobConf, ugi); - - ugi = - UserGroupInformation.createUserForTesting("u3", - new String[]{deprecatedSuperGroup}); - checkAccessToKill(tracker, jobConf, ugi); - - } - - private void checkAccessToKill(JobTracker tracker, final JobConf mrConf, - UserGroupInformation killer) throws IOException, InterruptedException, - ClassNotFoundException { - Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", - "p1" + NAME_SEPARATOR + "p11", conf); - JobID jobID = job.getStatus().getJobID(); - //Ensure that the jobinprogress is initied before we issue a kill - //signal to the job. - JobInProgress jip = tracker.getJob( - org.apache.hadoop.mapred.JobID.downgrade(jobID)); - tracker.initJob(jip); - Cluster cluster = killer.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(mrConf); - } - }); - cluster.getJob(jobID).killJob(); - assertEquals("job not killed by " + killer, - cluster.getJob(jobID).getStatus().getState(), (State.KILLED)); - } - - /** - * Tests job submission after refresh - * @throws Exception - */ - @Test - public void testSubmitJobsAfterRefresh() throws Exception { - startCluster(true); - - // test for refresh - deleteQueuesConfigFile(); - Document doc = createDocument(); - refreshDocument(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf()); - admin.run(new String[] { "-refreshQueues" }); - try { - submitSleepJob(10, 10, 100, 100, false, "u1,g1", "p1" - + NAME_SEPARATOR + "p11", conf); - fail("user u1 is not in the submit jobs' list"); - } catch (Exception e) { - } - deleteQueuesConfigFile(); - doc = createDocument(); - createSimpleDocumentWithAcls(doc); - writeToFile(doc, QUEUES_CONFIG_FILE_PATH); - admin.run(new String[] { "-refreshQueues" }); - } - - private void refreshDocument(Document doc) { - Element queues = createQueuesNode(doc); - - // Create parent level queue q1. - Element q1 = createQueue(doc, "q1"); - Properties props = new Properties(); - props.setProperty("capacity", "10"); - props.setProperty("maxCapacity", "35"); - q1.appendChild(createProperties(doc, props)); - queues.appendChild(q1); - - // Create another parent level p1 - Element p1 = createQueue(doc, "p1"); - - // append child p11 to p1 - Element p11 = createQueue(doc, "p11"); - p11.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, " ")); - p11.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "u2")); - p1.appendChild(p11); - - Element p12 = createQueue(doc, "p12"); - - p12.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_SUBMIT_JOB_TAG, "*")); - p12.appendChild(createAcls(doc, - QueueConfigurationParser.ACL_ADMINISTER_JOB_TAG, "*")); - - // append p12 to p1. - p1.appendChild(p12); - // append child p13 to p1 - Element p13 = createQueue(doc, "p13"); - p13.appendChild(createState(doc, QueueState.RUNNING.getStateName())); - p1.appendChild(p13); - // append child p14 to p1 - Element p14 = createQueue(doc, "p14"); - p14.appendChild(createState(doc, QueueState.STOPPED.getStateName())); - p1.appendChild(p14); - queues.appendChild(p1); - } - - /** - * Tests job submission when acls are disabled - * @throws Exception - */ - @Test - public void testAclsDisabled() throws Exception { - startCluster(false); - - // submit job to queue p1:p11 by any user not in acls-submit-job - Job job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" + NAME_SEPARATOR - + "p11", conf); - assertTrue("Job submitted for u2 in queue p1:p11 is not successful.", - job.isSuccessful()); - - // submit job to queue p1:p11 by user in acls-submit-job - job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR - + "p11", conf); - assertTrue("Job submitted for u2 in queue p1:p11 is not successful.", - job.isSuccessful()); - - job = submitSleepJob(1, 1, 0, 0, false, "u1,g1", "p1" + NAME_SEPARATOR - + "p11", conf); - // kill the job by any user - final JobConf jobConf = miniMRCluster.createJobConf(); - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting("u3",new String[]{"g3"}); - Cluster cluster = ugi.doAs(new PrivilegedExceptionAction() { - public Cluster run() throws IOException { - return new Cluster(jobConf); - } - }); - JobID jobID = job.getStatus().getJobID(); - //Ensure that the jobinprogress is initied before we issue a kill - //signal to the job. - JobInProgress jip = miniMRCluster.getJobTrackerRunner().getJobTracker() - .getJob(org.apache.hadoop.mapred.JobID.downgrade(jobID)); - miniMRCluster.getJobTrackerRunner().getJobTracker().initJob(jip); - cluster.getJob(jobID).killJob(); - assertEquals("job submitted for u1 and queue p1:p11 is not killed.", - cluster.getJob(jobID).getStatus().getState(), (State.KILLED)); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRackAwareTaskPlacement.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRackAwareTaskPlacement.java deleted file mode 100644 index 82e5b56f97e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRackAwareTaskPlacement.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTrackerMetricsInst; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobCounter; -import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.split.JobSplit; -import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; -import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.net.StaticMapping; -import org.mortbay.log.Log; - -/** - * A JUnit test to test configured task limits. - */ -public class TestRackAwareTaskPlacement extends TestCase { - - static String trackers[] = new String[] {"tracker_tracker1.r1.com:1000", - "tracker_tracker2.r1.com:1000", "tracker_tracker3.r2.com:1000", - "tracker_tracker4.r3.com:1000"}; - - static String[] allHosts = - new String[] {"tracker1.r1.com", "tracker2.r1.com", "tracker3.r2.com", - "tracker4.r3.com"}; - - static String[] allRacks = - new String[] { "/r1", "/r1", "/r2", "/r3"}; - - static FakeJobTracker jobTracker; - static String jtIdentifier = "test"; - private static int jobCounter; - private static FakeJobTrackerMetricsInst fakeInst; - - public static Test suite() { - TestSetup setup = - new TestSetup(new TestSuite(TestRackAwareTaskPlacement.class)) { - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setClass("topology.node.switch.mapping.impl", - StaticMapping.class, DNSToSwitchMapping.class); - conf.set(JTConfig.JT_INSTRUMENTATION, - FakeJobTrackerMetricsInst.class.getName()); - jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); - fakeInst = (FakeJobTrackerMetricsInst) jobTracker.getInstrumentation(); - // Set up the Topology Information - for (int i = 0; i < allHosts.length; i++) { - StaticMapping.addNodeToRack(allHosts[i], allRacks[i]); - } - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - } - }; - return setup; - } - - static class MyFakeJobInProgress extends JobInProgress { - static JobID jobid; - int numMaps; - - MyFakeJobInProgress(JobConf jc, JobTracker jt) throws IOException { - super((jobid = new JobID(jtIdentifier, jobCounter ++)), jc, jt); - Path jobFile = new Path("Dummy"); - this.profile = new JobProfile(jc.getUser(), jobid, - jobFile.toString(), null, jc.getJobName(), - jc.getQueueName()); - this.jobHistory = new FakeJobHistory(); - } - - @Override - public void initTasks() throws IOException { - TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(jobId); - numMapTasks = taskSplitMetaInfo.length; - createMapTasks(null, taskSplitMetaInfo); - nonRunningMapCache = createCache(taskSplitMetaInfo, maxLevel); - tasksInited.set(true); - this.status.setRunState(JobStatus.RUNNING); - - } - - @Override - protected TaskSplitMetaInfo [] createSplits( - org.apache.hadoop.mapreduce.JobID jobId) throws IOException { - TaskSplitMetaInfo[] splits = new TaskSplitMetaInfo[numMaps]; - // Hand code for now. - // M0,2,3 reside in Host1 - // M1 resides in Host3 - // M4 resides in Host4 - String[] splitHosts0 = new String[] { allHosts[0] }; - - String[] splitHosts1 = new String[] { allHosts[2] }; - String[] splitHosts2 = new String[] { allHosts[3] }; - for (int i = 0; i < numMaps; i++) { - if (i == 0 || i == 2 || i == 3) { - splits[i] = new TaskSplitMetaInfo(splitHosts0, 0, 0); - } else if (i == 1) { - splits[i] = new TaskSplitMetaInfo(splitHosts1, 0, 0); - } else if (i == 4) { - splits[i] = new TaskSplitMetaInfo(splitHosts2, 0, 0); - } - } - - return splits; - } - } - @SuppressWarnings("deprecation") - public void testTaskPlacement() throws IOException { - JobConf conf = new JobConf(); - conf.setNumReduceTasks(0); - conf.setJobName("TestTaskPlacement"); - - MyFakeJobInProgress jip = new MyFakeJobInProgress(conf, jobTracker); - jip.numMaps = 5; - jip.initTasks(); - - // Tracker1 should get a rack local - TaskTrackerStatus tts = new TaskTrackerStatus(trackers[1], allHosts[1]); - jip.obtainNewMapTask(tts, 4, 4); - - // Tracker0 should get a data local - tts = new TaskTrackerStatus(trackers[0], allHosts[0]); - jip.obtainNewMapTask(tts, 4, 4); - - // Tracker2 should get a data local - tts = new TaskTrackerStatus(trackers[2], allHosts[2]); - jip.obtainNewMapTask(tts, 4, 4); - - // Tracker0 should get a data local - tts = new TaskTrackerStatus(trackers[0], allHosts[0]); - jip.obtainNewMapTask(tts, 4, 4); - - // Tracker1 should not get any locality at all - tts = new TaskTrackerStatus(trackers[1], allHosts[1]); - jip.obtainNewMapTask(tts, 4, 4); - - - Counters counters = jip.getCounters(); - assertEquals("Number of data local maps", 3, - counters.getCounter(JobCounter.DATA_LOCAL_MAPS)); - - assertEquals("Number of Rack-local maps", 1 , - counters.getCounter(JobCounter.RACK_LOCAL_MAPS)); - - assertEquals("Number of Other-local maps", 0, - counters.getCounter(JobCounter.OTHER_LOCAL_MAPS)); - // Also verify jobtracker instrumentation - assertEquals("Number of data local maps", 3, fakeInst.numDataLocalMaps); - assertEquals("Number of rack local maps", 1, fakeInst.numRackLocalMaps); - - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java deleted file mode 100644 index c446af4ee89..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.JobTracker.RecoveryManager; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.MRConfig; - -import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesConfigFile; -import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.UserGroupInformation; - -/** - * Test whether the {@link RecoveryManager} is able to tolerate job-recovery - * failures and the jobtracker is able to tolerate {@link RecoveryManager} - * failure. - */ -public class TestRecoveryManager extends TestCase { - private static final Log LOG = - LogFactory.getLog(TestRecoveryManager.class); - private static final Path TEST_DIR = - new Path(System.getProperty("test.build.data", "/tmp"), - "test-recovery-manager"); - - /** - * Tests the {@link JobTracker} against the exceptions thrown in - * {@link JobTracker.RecoveryManager}. It does the following : - * - submits 3 jobs - * - kills the jobtracker - * - Garble job.xml for one job causing it to fail in constructor - * and job.split for another causing it to fail in init. - * - delete the job temp/submit dir - * - restarts the jobtracker - * - checks if the jobtraker starts normally - */ - public void testJobTracker() throws Exception { - LOG.info("Testing jobtracker restart with faulty job"); - String signalFile = new Path(TEST_DIR, "signal").toString(); - JobConf conf = new JobConf(); - - FileSystem fs = FileSystem.get(new Configuration()); - fs.delete(TEST_DIR, true); // cleanup - - conf.set(JTConfig.JT_JOBHISTORY_BLOCK_SIZE, "1024"); - - MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); - - JobConf job1 = mr.createJobConf(); - - UtilsForTests.configureWaitingJobConf(job1, - new Path(TEST_DIR, "input"), new Path(TEST_DIR, "output1"), 2, 0, - "test-recovery-manager", signalFile, signalFile); - - // submit the faulty job - RunningJob rJob1 = (new JobClient(job1)).submitJob(job1); - LOG.info("Submitted job " + rJob1.getID()); - - while (rJob1.mapProgress() < 0.5f) { - LOG.info("Waiting for job " + rJob1.getID() + " to be 50% done"); - UtilsForTests.waitFor(100); - } - - JobConf job2 = mr.createJobConf(); - - UtilsForTests.configureWaitingJobConf(job2, - new Path(TEST_DIR, "input"), new Path(TEST_DIR, "output2"), 30, 0, - "test-recovery-manager", signalFile, signalFile); - - // submit the faulty job - RunningJob rJob2 = (new JobClient(job2)).submitJob(job2); - LOG.info("Submitted job " + rJob2.getID()); - - while (rJob2.mapProgress() < 0.5f) { - LOG.info("Waiting for job " + rJob2.getID() + " to be 50% done"); - UtilsForTests.waitFor(100); - } - - // kill the jobtracker - LOG.info("Stopping jobtracker"); - String sysDir = mr.getJobTrackerRunner().getJobTracker().getSystemDir(); - mr.stopJobTracker(); - - // delete the job.xml of job #1 causing the job to fail in submit Job - //while recovery itself - Path jobFile = - new Path(sysDir, rJob1.getID().toString() + "/" + JobTracker.JOB_INFO_FILE); - LOG.info("Deleting job token file : " + jobFile.toString()); - fs.delete(jobFile, false); // delete the job.xml file - - // create the job token file with 1 byte - FSDataOutputStream out = fs.create(jobFile); - out.write(1); - out.close(); - - // make sure that the jobtracker is in recovery mode - mr.getJobTrackerConf().setBoolean(JTConfig.JT_RESTART_ENABLED, true); - // start the jobtracker - LOG.info("Starting jobtracker"); - mr.startJobTracker(); - ClusterStatus status = - mr.getJobTrackerRunner().getJobTracker().getClusterStatus(false); - - // check if the jobtracker came up or not - assertEquals("JobTracker crashed!", - JobTrackerStatus.RUNNING, status.getJobTrackerStatus()); - - // assert the no of recovered jobs - assertEquals("No of recovered jobs not correct", - 1, mr.getJobTrackerRunner().getJobTracker(). - recoveryManager.getRecovered()); - - mr.shutdown(); - } - - /** - * Tests the {@link JobTracker.RecoveryManager} against the exceptions thrown - * during recovery. It does the following : - * - submits a job with HIGH priority and x tasks - * - allows it to complete 50% - * - submits another job with normal priority and y tasks - * - kills the jobtracker - * - restarts the jobtracker with max-tasks-per-job such that - * y < max-tasks-per-job < x - * - checks if the jobtraker starts normally and job#2 is recovered while - * job#1 is failed. - */ - public void testRecoveryManager() throws Exception { - LOG.info("Testing recovery-manager"); - String signalFile = new Path(TEST_DIR, "signal").toString(); - - // clean up - FileSystem fs = FileSystem.get(new Configuration()); - fs.delete(TEST_DIR, true); - - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_JOBHISTORY_BLOCK_SIZE, "1024"); - - MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); - JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - JobConf job1 = mr.createJobConf(); - // set the high priority - job1.setJobPriority(JobPriority.HIGH); - - UtilsForTests.configureWaitingJobConf(job1, - new Path(TEST_DIR, "input"), new Path(TEST_DIR, "output3"), 30, 0, - "test-recovery-manager", signalFile, signalFile); - - // submit the faulty job - JobClient jc = new JobClient(job1); - RunningJob rJob1 = jc.submitJob(job1); - LOG.info("Submitted first job " + rJob1.getID()); - - while (rJob1.mapProgress() < 0.5f) { - LOG.info("Waiting for job " + rJob1.getID() + " to be 50% done"); - UtilsForTests.waitFor(100); - } - - // now submit job2 - JobConf job2 = mr.createJobConf(); - - String signalFile1 = new Path(TEST_DIR, "signal1").toString(); - UtilsForTests.configureWaitingJobConf(job2, - new Path(TEST_DIR, "input"), new Path(TEST_DIR, "output4"), 20, 0, - "test-recovery-manager", signalFile1, signalFile1); - - // submit the job - RunningJob rJob2 = (new JobClient(job2)).submitJob(job2); - LOG.info("Submitted job " + rJob2.getID()); - - // wait for it to init - JobInProgress jip = jobtracker.getJob(rJob2.getID()); - - while (!jip.inited()) { - LOG.info("Waiting for job " + jip.getJobID() + " to be inited"); - UtilsForTests.waitFor(100); - } - - // now submit job3 with inappropriate acls - final JobConf job3 = mr.createJobConf(); - UserGroupInformation ugi3 = - UserGroupInformation.createUserForTesting("abc", new String[]{"users"}); - - UtilsForTests.configureWaitingJobConf(job3, - new Path(TEST_DIR, "input"), new Path(TEST_DIR, "output5"), 1, 0, - "test-recovery-manager", signalFile, signalFile); - - // submit the job - RunningJob rJob3 = ugi3.doAs(new PrivilegedExceptionAction() { - public RunningJob run() throws IOException { - return (new JobClient(job3)).submitJob(job3); - } - }); - - LOG.info("Submitted job " + rJob3.getID() + " with different user"); - - jip = jobtracker.getJob(rJob3.getID()); - assertEquals("Restart count is not correct", - 0, jip.getNumRestarts()); - - while (!jip.inited()) { - LOG.info("Waiting for job " + jip.getJobID() + " to be inited"); - UtilsForTests.waitFor(100); - } - - // kill the jobtracker - LOG.info("Stopping jobtracker"); - mr.stopJobTracker(); - - // make sure that the jobtracker is in recovery mode - mr.getJobTrackerConf().setBoolean(JTConfig.JT_RESTART_ENABLED, - true); - mr.getJobTrackerConf().setInt(JTConfig.JT_TASKS_PER_JOB, 25); - - mr.getJobTrackerConf().setBoolean(MRConfig.MR_ACLS_ENABLED, true); - - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - mr.getJobTrackerConf().set(toFullPropertyName( - "default", QueueACL.SUBMIT_JOB.getAclName()), ugi.getUserName()); - - // start the jobtracker - LOG.info("Starting jobtracker"); - mr.startJobTracker(); - UtilsForTests.waitForJobTracker(jc); - - jobtracker = mr.getJobTrackerRunner().getJobTracker(); - - // assert that job2 is recovered by the jobtracker as job1 would fail - assertEquals("Recovery manager failed to tolerate job failures", - 2, jobtracker.getAllJobs().length); - - // assert the no of recovered jobs - assertEquals("No of recovered jobs not correct", - 2, jobtracker.recoveryManager.getRecovered()); - assertEquals("Restart count is not correct", - 1, jobtracker.getJob(rJob2.getID()).getNumRestarts()); - // check if the job#1 has failed - JobStatus status = jobtracker.getJobStatus(rJob1.getID()); - assertEquals("Faulty job not failed", - JobStatus.FAILED, status.getRunState()); - - jip = jobtracker.getJob(rJob2.getID()); - assertFalse("Job should be running", jip.isComplete()); - - status = jobtracker.getJobStatus(rJob3.getID()); - assertNull("Job should be missing", status); - - mr.shutdown(); - } - - /** - * Test if the jobtracker waits for the info file to be created before - * starting. - */ - public void testJobTrackerInfoCreation() throws Exception { - LOG.info("Testing jobtracker.info file"); - MiniDFSCluster dfs = new MiniDFSCluster(new Configuration(), 1, true, null); - String namenode = (dfs.getFileSystem()).getUri().getHost() + ":" - + (dfs.getFileSystem()).getUri().getPort(); - // shut down the data nodes - dfs.shutdownDataNodes(); - - // start the jobtracker - JobConf conf = new JobConf(); - FileSystem.setDefaultUri(conf, namenode); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "127.0.0.1:0"); - - JobTracker jobtracker = new JobTracker(conf); - - // now check if the update restart count works fine or not - boolean failed = false; - try { - jobtracker.recoveryManager.updateRestartCount(); - } catch (IOException ioe) { - failed = true; - } - assertTrue("JobTracker created info files without datanodes!!!", failed); - - Path restartFile = jobtracker.recoveryManager.getRestartCountFile(); - Path tmpRestartFile = jobtracker.recoveryManager.getTempRestartCountFile(); - FileSystem fs = dfs.getFileSystem(); - assertFalse("Info file exists after update failure", - fs.exists(restartFile)); - assertFalse("Temporary restart-file exists after update failure", - fs.exists(restartFile)); - - // start 1 data node - dfs.startDataNodes(conf, 1, true, null, null, null, null); - dfs.waitActive(); - - failed = false; - try { - jobtracker.recoveryManager.updateRestartCount(); - } catch (IOException ioe) { - failed = true; - } - assertFalse("JobTracker failed to create info files with datanodes!!!", failed); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java deleted file mode 100644 index 2b47a38e577..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import org.apache.hadoop.mapreduce.split.JobSplit; - -import junit.framework.TestCase; - -public class TestResourceEstimation extends TestCase { - - - public void testResourceEstimator() throws Exception { - final int maps = 100; - final int reduces = 2; - final int singleMapOutputSize = 1000; - JobConf jc = new JobConf(); - JobID jid = new JobID("testJT", 0); - jc.setNumMapTasks(maps); - jc.setNumReduceTasks(reduces); - - JobInProgress jip = new JobInProgress(jid, jc, - UtilsForTests.getJobTracker()); - //unfortunately, we can't set job input size from here. - ResourceEstimator re = new ResourceEstimator(jip); - - for(int i = 0; i < maps / 10 ; ++i) { - - long estOutSize = re.getEstimatedMapOutputSize(); - System.out.println(estOutSize); - assertEquals(0, estOutSize); - - TaskStatus ts = new MapTaskStatus(); - ts.setOutputSize(singleMapOutputSize); - JobSplit.TaskSplitMetaInfo split = - new JobSplit.TaskSplitMetaInfo(new String[0], 0, 0); - TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1); - re.updateWithCompletedTask(ts, tip); - } - assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize()); - assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize()); - - } - - public void testWithNonZeroInput() throws Exception { - final int maps = 100; - final int reduces = 2; - final int singleMapOutputSize = 1000; - final int singleMapInputSize = 500; - JobConf jc = new JobConf(); - JobID jid = new JobID("testJT", 0); - jc.setNumMapTasks(maps); - jc.setNumReduceTasks(reduces); - - JobInProgress jip = new JobInProgress(jid, jc, - UtilsForTests.getJobTracker()) { - long getInputLength() { - return singleMapInputSize*desiredMaps(); - } - }; - ResourceEstimator re = new ResourceEstimator(jip); - - for(int i = 0; i < maps / 10 ; ++i) { - - long estOutSize = re.getEstimatedMapOutputSize(); - System.out.println(estOutSize); - assertEquals(0, estOutSize); - - TaskStatus ts = new MapTaskStatus(); - ts.setOutputSize(singleMapOutputSize); - JobSplit.TaskSplitMetaInfo split = - new JobSplit.TaskSplitMetaInfo(new String[0], 0, - singleMapInputSize); - TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1); - re.updateWithCompletedTask(ts, tip); - } - - assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize()); - assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize()); - - //add one more map task with input size as 0 - TaskStatus ts = new MapTaskStatus(); - ts.setOutputSize(singleMapOutputSize); - JobSplit.TaskSplitMetaInfo split = - new JobSplit.TaskSplitMetaInfo(new String[0], 0, 0); - TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1); - re.updateWithCompletedTask(ts, tip); - - long expectedTotalMapOutSize = (singleMapOutputSize*11) * - ((maps*singleMapInputSize)+maps)/((singleMapInputSize+1)*10+1); - assertEquals(2* expectedTotalMapOutSize/maps, re.getEstimatedMapOutputSize()); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java deleted file mode 100644 index 2c4d9998c46..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; - -/** - * Tests various failures in setup/cleanup of job, like - * throwing exception, command line kill and lost tracker - */ -public class TestSetupAndCleanupFailure extends TestCase { - - final Path inDir = new Path("./input"); - final Path outDir = new Path("./output"); - static Path setupSignalFile = new Path("/setup-signal"); - static Path cleanupSignalFile = new Path("/cleanup-signal"); - - // Commiter with setupJob throwing exception - static class CommitterWithFailSetup extends FileOutputCommitter { - @Override - public void setupJob(JobContext context) throws IOException { - throw new IOException(); - } - } - - // Commiter with commitJob throwing exception - static class CommitterWithFailCommit extends FileOutputCommitter { - @Override - public void commitJob(JobContext context) throws IOException { - throw new IOException(); - } - } - - // Committer waits for a file to be created on dfs. - static class CommitterWithLongSetupAndCommit extends FileOutputCommitter { - - private void waitForSignalFile(FileSystem fs, Path signalFile) - throws IOException { - while (!fs.exists(signalFile)) { - try { - Thread.sleep(100); - } catch (InterruptedException ie) { - break; - } - } - } - - @Override - public void setupJob(JobContext context) throws IOException { - waitForSignalFile(FileSystem.get(context.getJobConf()), setupSignalFile); - super.setupJob(context); - } - - @Override - public void commitJob(JobContext context) throws IOException { - waitForSignalFile(FileSystem.get(context.getJobConf()), cleanupSignalFile); - super.commitJob(context); - } - } - - // Among these tips only one of the tasks will be running, - // get the taskid for that task - private TaskAttemptID getRunningTaskID(TaskInProgress[] tips) { - TaskAttemptID taskid = null; - while (taskid == null) { - for (TaskInProgress tip :tips) { - TaskStatus[] statuses = tip.getTaskStatuses(); - for (TaskStatus status : statuses) { - if (status.getRunState() == TaskStatus.State.RUNNING) { - taskid = status.getTaskID(); - break; - } - } - if (taskid != null) break; - } - try { - Thread.sleep(10); - } catch (InterruptedException ie) {} - } - return taskid; - } - - // Tests the failures in setup/cleanup job. Job should cleanly fail. - private void testFailCommitter(Class theClass, - JobConf jobConf) - throws IOException { - jobConf.setOutputCommitter(theClass); - RunningJob job = UtilsForTests.runJob(jobConf, inDir, outDir); - // wait for the job to finish. - job.waitForCompletion(); - assertEquals(JobStatus.FAILED, job.getJobState()); - } - - // launch job with CommitterWithLongSetupAndCleanup as committer - // and wait till the job is inited. - private RunningJob launchJobWithWaitingSetupAndCleanup(MiniMRCluster mr) - throws IOException { - // launch job with waiting setup/cleanup - JobConf jobConf = mr.createJobConf(); - jobConf.setOutputCommitter(CommitterWithLongSetupAndCommit.class); - RunningJob job = UtilsForTests.runJob(jobConf, inDir, outDir); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = jt.getJob(job.getID()); - while (!jip.inited()) { - try { - Thread.sleep(10); - } catch (InterruptedException ie) {} - } - return job; - } - - /** - * Tests setup and cleanup attempts getting killed from command-line - * and lost tracker - * - * @param mr - * @param dfs - * @param commandLineKill if true, test with command-line kill - * else, test with lost tracker - * @throws IOException - */ - private void testSetupAndCleanupKill(MiniMRCluster mr, - MiniDFSCluster dfs, - boolean commandLineKill) - throws Exception { - // launch job with waiting setup/cleanup - RunningJob job = launchJobWithWaitingSetupAndCleanup(mr); - - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = jt.getJob(job.getID()); - // get the running setup task id - TaskAttemptID setupID = getRunningTaskID(jip.getTasks(TaskType.JOB_SETUP)); - if (commandLineKill) { - killTaskFromCommandLine(job, setupID, jt); - } else { - killTaskWithLostTracker(mr, setupID); - } - // signal the setup to complete - UtilsForTests.writeFile(dfs.getNameNode(), - dfs.getFileSystem().getConf(), - setupSignalFile, (short)3); - // wait for maps and reduces to complete - while (job.reduceProgress() != 1.0f) { - try { - Thread.sleep(100); - } catch (InterruptedException ie) {} - } - // get the running cleanup task id - TaskAttemptID cleanupID = - getRunningTaskID(jip.getTasks(TaskType.JOB_CLEANUP)); - if (commandLineKill) { - killTaskFromCommandLine(job, cleanupID, jt); - } else { - killTaskWithLostTracker(mr, cleanupID); - } - // signal the cleanup to complete - UtilsForTests.writeFile(dfs.getNameNode(), - dfs.getFileSystem().getConf(), - cleanupSignalFile, (short)3); - // wait for the job to finish. - job.waitForCompletion(); - assertEquals(JobStatus.SUCCEEDED, job.getJobState()); - assertEquals(TaskStatus.State.KILLED, - jt.getTaskStatus(setupID).getRunState()); - assertEquals(TaskStatus.State.KILLED, - jt.getTaskStatus(cleanupID).getRunState()); - } - - // kill the task from command-line - // wait till it kill is reported back - private void killTaskFromCommandLine(RunningJob job, - TaskAttemptID taskid, - JobTracker jt) - throws IOException { - job.killTask(taskid, false); - // wait till the kill happens - while (jt.getTaskStatus(taskid).getRunState() != - TaskStatus.State.KILLED) { - try { - Thread.sleep(10); - } catch (InterruptedException ie) {} - } - - } - // kill the task by losing the tracker - private void killTaskWithLostTracker(MiniMRCluster mr, - TaskAttemptID taskid) { - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - String trackerName = jt.getTaskStatus(taskid).getTaskTracker(); - int trackerID = mr.getTaskTrackerID(trackerName); - assertTrue(trackerID != -1); - mr.stopTaskTracker(trackerID); - } - - // Tests the failures in setup/cleanup job. Job should cleanly fail. - // Also Tests the command-line kill for setup/cleanup attempts. - // tests the setup/cleanup attempts getting killed if - // they were running on a lost tracker - public void testWithDFS() throws Exception { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int taskTrackers = 4; - Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 4, true, null); - fileSys = dfs.getFileSystem(); - JobConf jtConf = new JobConf(); - jtConf.setInt(TTConfig.TT_MAP_SLOTS, 1); - jtConf.setInt(TTConfig.TT_REDUCE_SLOTS, 1); - jtConf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 10 * 1000); - mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, - null, null, jtConf); - // test setup/cleanup throwing exceptions - testFailCommitter(CommitterWithFailSetup.class, mr.createJobConf()); - testFailCommitter(CommitterWithFailCommit.class, mr.createJobConf()); - // test the command-line kill for setup/cleanup attempts. - testSetupAndCleanupKill(mr, dfs, true); - // remove setup/cleanup signal files. - fileSys.delete(setupSignalFile , true); - fileSys.delete(cleanupSignalFile , true); - // test the setup/cleanup attempts getting killed if - // they were running on a lost tracker - testSetupAndCleanupKill(mr, dfs, false); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); - } - } - } - - public static void main(String[] argv) throws Exception { - TestSetupAndCleanupFailure td = new TestSetupAndCleanupFailure(); - td.testWithDFS(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java deleted file mode 100644 index 059d692eb82..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java +++ /dev/null @@ -1,334 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeTaskInProgress; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.split.JobSplit; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.TaskType; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import junit.framework.TestCase; - -public class TestSetupTaskScheduling extends TestCase { - - public static final Log LOG = - LogFactory.getLog(TestSetupTaskScheduling.class); - - static String[] trackers = new String[] { "tracker_tracker1:1000", - "tracker_tracker2:1000", "tracker_tracker3:1000" }; - private static FakeJobTracker jobTracker; - - /** - * Fake JobInProgress that can return a hardcoded setup or - * cleanup task depending on the slot type passed in. - */ - static class FakeJobWithSetupTask - extends FakeObjectUtilities.FakeJobInProgress { - - FakeJobWithSetupTask(JobConf jobConf, - JobTracker tracker) throws IOException { - super(jobConf, tracker); - } - - /** - * Initialize tasks, including setup. - */ - @Override - public synchronized void initTasks() throws IOException { - super.initTasks(); - JobSplit.TaskSplitMetaInfo emptySplit = new JobSplit.TaskSplitMetaInfo(); - setup = new TaskInProgress[2]; - setup[0] = new TaskInProgress(getJobID(), "test", emptySplit, - jobtracker, getJobConf(), this, numMapTasks + 1, 1); - setup[1] = new TaskInProgress(getJobID(), "test", numMapTasks, - numReduceTasks + 1, jobtracker, getJobConf(), this, 1); - } - - /** - * Obtain a setup task on a map slot or reduce slot - * depending on what is free. - * - * Every call to this will return either a map or reduce - * setup task. No check is done to see if the task is already - * returned - */ - @Override - public Task obtainJobSetupTask(TaskTrackerStatus tts, - int clusterSize, - int numUniqueHosts, - boolean isMapSlot) - throws IOException{ - TaskInProgress tip = null; - if (isMapSlot) { - tip = setup[0]; - } else { - tip = setup[1]; - } - Task t = tip.getTaskToRun(tts.getHost()); - t.setJobSetupTask(); - return t; - } - } - - static class FakeJobWithTaskCleanupTask - extends FakeObjectUtilities.FakeJobInProgress { - - FakeJobWithTaskCleanupTask(JobConf jobConf, - JobTracker tracker) throws IOException { - super(jobConf, tracker); - } - - /** - * Initialize tasks(1 map and 1 reduce task each needs 2 slots, similar to - * tasks of a high RAM job). - */ - @Override - public synchronized void initTasks() throws IOException { - super.initTasks(); - - final int numSlotsPerTask = 2; - maps = new TaskInProgress[1]; - reduces = new TaskInProgress[1]; - - maps[0] = new FakeTaskInProgress(getJobID(), "test", - JobSplit.EMPTY_TASK_SPLIT, - jobtracker, getJobConf(), this, 0, numSlotsPerTask); - TaskAttemptID attemptId = new TaskAttemptID(maps[0].getTIPId(), 0); - - // make this task a taskCleanup task of a map task - mapCleanupTasks.add(attemptId); - TaskStatus stat = new MapTaskStatus(attemptId, 0.01f, 2, - TaskStatus.State.FAILED_UNCLEAN, "", "", trackers[0], - TaskStatus.Phase.MAP, new Counters()); - maps[0].updateStatus(stat); - - //similarly for reduce task's taskCleanup task - reduces[0] = new FakeTaskInProgress(getJobID(), "test", 1, - 0, jobtracker, getJobConf(), this, numSlotsPerTask); - attemptId = new TaskAttemptID(reduces[0].getTIPId(), 0); - - // make this task a taskCleanup task of a reduce task - reduceCleanupTasks.add(attemptId); - stat = new ReduceTaskStatus(attemptId, 0.01f, 2, - TaskStatus.State.FAILED_UNCLEAN, "", "", trackers[0], - TaskStatus.Phase.REDUCE, new Counters()); - reduces[0].updateStatus(stat); - } - } - - public void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - jobTracker = new FakeJobTracker(conf, new Clock(), trackers); - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - } - - // create a job for testing setup tasks and reservations - FakeJobInProgress createJob(TaskType taskType) throws IOException { - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setNumMapTasks(2); - conf.setNumReduceTasks(2); - conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70"); - conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70"); - FakeJobInProgress job = null; - if (taskType == null) { - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - job = new FakeJobInProgress(conf, jobTracker); - } else if (taskType == TaskType.JOB_SETUP) { - job = new FakeJobWithSetupTask(conf, jobTracker); - } else if (taskType == TaskType.TASK_CLEANUP) { - job = new FakeJobWithTaskCleanupTask(conf, jobTracker); - } - job.setClusterSize(trackers.length); - job.initTasks(); - return job; - } - - // create a new TaskStatus and add to a list of status objects. - // useMapSlot param is needed only when taskType is TASK_CLEANUP. - void addNewTaskStatus(FakeJobInProgress job, TaskType taskType, - boolean useMapSlot, String tracker, List reports) - throws IOException { - TaskAttemptID task = null; - TaskStatus status = null; - if (taskType == TaskType.MAP) { - task = job.findMapTask(tracker); - status = new MapTaskStatus(task, 0.01f, 2, - TaskStatus.State.RUNNING, "", "", tracker, - TaskStatus.Phase.MAP, new Counters()); - } else if (taskType == TaskType.TASK_CLEANUP) { - if (useMapSlot) { - status = job.maps[0].taskStatuses.get( - new TaskAttemptID(job.maps[0].getTIPId(), 0)); - } else { - status = job.reduces[0].taskStatuses.get( - new TaskAttemptID(job.reduces[0].getTIPId(), 0)); - } - } else { - task = job.findReduceTask(tracker); - status = new ReduceTaskStatus(task, 0.01f, 2, - TaskStatus.State.RUNNING, "", "", tracker, - TaskStatus.Phase.REDUCE, new Counters()); - } - reports.add(status); - } - - // create a TaskTrackerStatus - TaskTrackerStatus createTaskTrackerStatus(String tracker, - List reports) { - TaskTrackerStatus ttStatus = - new TaskTrackerStatus(tracker, - JobInProgress.convertTrackerNameToHostName(tracker), - 0, reports, 0, 2, 2); - return ttStatus; - } - - /** - * Test that a setup task can be run against a map slot - * if it is free. - * @throws IOException - */ - public void testSetupTaskReturnedForFreeMapSlots() throws IOException { - // create a job with a setup task. - FakeJobInProgress job = createJob(TaskType.JOB_SETUP); - jobTracker.jobs.put(job.getJobID(), job); - - // create a status simulating a free tasktracker - List reports = new ArrayList(); - TaskTrackerStatus ttStatus - = createTaskTrackerStatus(trackers[2], reports); - - // verify that a setup task can be assigned to a map slot. - List tasks = jobTracker.getSetupAndCleanupTasks(ttStatus); - assertEquals(1, tasks.size()); - assertTrue(tasks.get(0).isJobSetupTask()); - assertTrue(tasks.get(0).isMapTask()); - jobTracker.jobs.clear(); - } - - /** - * Test to check that map slots are counted when returning - * a setup task. - * @throws IOException - */ - public void testMapSlotsCountedForSetup() throws IOException { - // create a job with a setup task. - FakeJobInProgress job = createJob(TaskType.JOB_SETUP); - jobTracker.jobs.put(job.getJobID(), job); - - // create another job for reservation - FakeJobInProgress job1 = createJob(null); - jobTracker.jobs.put(job1.getJobID(), job1); - - // create TT status for testing getSetupAndCleanupTasks - List taskStatuses = new ArrayList(); - addNewTaskStatus(job, TaskType.MAP, true, trackers[0], taskStatuses); - TaskTrackerStatus ttStatus - = createTaskTrackerStatus(trackers[0], taskStatuses); - - // test that there should be no map setup task returned. - List tasks = jobTracker.getSetupAndCleanupTasks(ttStatus); - assertEquals(1, tasks.size()); - assertTrue(tasks.get(0).isJobSetupTask()); - assertFalse(tasks.get(0).isMapTask()); - jobTracker.jobs.clear(); - } - - /** - * Test to check that reduce slots are also counted when returning - * a setup task. - * @throws IOException - */ - public void testReduceSlotsCountedForSetup() throws IOException { - // create a job with a setup task. - FakeJobInProgress job = createJob(TaskType.JOB_SETUP); - jobTracker.jobs.put(job.getJobID(), job); - - // create another job for reservation - FakeJobInProgress job1 = createJob(null); - jobTracker.jobs.put(job1.getJobID(), job1); - - // create TT status for testing getSetupAndCleanupTasks - List reports = new ArrayList(); - // because free map slots are checked first in code, - // we fill up map slots also. - addNewTaskStatus(job1, TaskType.MAP, true, trackers[1], reports); - addNewTaskStatus(job1, TaskType.REDUCE, false,trackers[1], reports); - TaskTrackerStatus ttStatus - = createTaskTrackerStatus(trackers[1], reports); - - // test that there should be no setup task returned, - // as both map and reduce slots are occupied. - List tasks = jobTracker.getSetupAndCleanupTasks(ttStatus); - assertNull(tasks); - jobTracker.jobs.clear(); - } - - void validateNumSlotsUsedForTaskCleanup(TaskTrackerStatus ttStatus) - throws IOException { - List tasks = jobTracker.getSetupAndCleanupTasks(ttStatus); - - assertEquals("Actual number of taskCleanup tasks is not same as expected", 1, tasks.size()); - LOG.info("taskCleanup task is " + tasks.get(0)); - assertTrue(tasks.get(0).isTaskCleanupTask()); - - // slots needed for taskCleanup task should be 1(even for high RAM jobs) - assertEquals("TaskCleanup task should not need more than 1 slot.", - 1, tasks.get(0).getNumSlotsRequired()); - } - - /** - * Test to check that map slots are counted when returning - * a taskCleanup task. - * @throws IOException - */ - public void testNumSlotsUsedForTaskCleanup() throws IOException { - // Create a high RAM job with a map task's cleanup task and a reduce task's - // cleanup task. Make this Fake job a high RAM job by setting the slots - // required for map/reduce task to 2. - FakeJobInProgress job = createJob(TaskType.TASK_CLEANUP); - jobTracker.jobs.put(job.getJobID(), job); - - // create TT status for testing getSetupAndCleanupTasks - List taskStatuses = new ArrayList(); - TaskTrackerStatus ttStatus = - createTaskTrackerStatus(trackers[0], taskStatuses);//create dummy status - - // validate mapTaskCleanup task - validateNumSlotsUsedForTaskCleanup(ttStatus); - - // validate reduceTaskCleanup task - validateNumSlotsUsedForTaskCleanup(ttStatus); - - jobTracker.jobs.clear(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupWorkDir.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupWorkDir.java deleted file mode 100644 index c861d4d27e4..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupWorkDir.java +++ /dev/null @@ -1,235 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import junit.framework.TestCase; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.filecache.DistributedCache; -import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager; - -/** - * Verifies if TaskRunner.SetupWorkDir() is cleaning up files/dirs pointed - * to by symlinks under work dir. - */ -public class TestSetupWorkDir extends TestCase { - - /** - * Creates 1 subdirectory and 1 file under dir2. Creates 1 subdir, 1 file, - * 1 symlink to a dir and a symlink to a file under dir1. - * Creates dir1/subDir, dir1/file, dir2/subDir, dir2/file, - * dir1/symlinkSubDir->dir2/subDir, dir1/symlinkFile->dir2/file. - */ - static void createSubDirsAndSymLinks(JobConf jobConf, Path dir1, Path dir2) - throws IOException { - FileSystem fs = FileSystem.getLocal(jobConf); - createSubDirAndFile(fs, dir1); - createSubDirAndFile(fs, dir2); - // now create symlinks under dir1 that point to file/dir under dir2 - FileUtil.symLink(dir2+"/subDir", dir1+"/symlinkSubDir"); - FileUtil.symLink(dir2+"/file", dir1+"/symlinkFile"); - } - - static void createSubDirAndFile(FileSystem fs, Path dir) throws IOException { - Path subDir = new Path(dir, "subDir"); - fs.mkdirs(subDir); - createFile(fs, dir, "file"); - } - - /** - * Create a file - * - * @param fs filesystem - * @param dir directory location of the file - * @param fileName filename - * @throws IOException - */ - static void createFile(FileSystem fs, Path dir, String fileName) - throws IOException { - Path p = new Path(dir, fileName); - DataOutputStream out = fs.create(p); - out.writeBytes("dummy input"); - out.close(); - } - - void createEmptyDir(FileSystem fs, Path dir) throws IOException { - if (fs.exists(dir)) { - fs.delete(dir, true); - } - if (!fs.mkdirs(dir)) { - throw new IOException("Unable to create directory " + dir); - } - } - - /** - * Validates if TaskRunner.setupWorkDir() is properly cleaning up the - * contents of workDir and creating tmp dir under it (even though workDir - * contains symlinks to files/directories). - */ - public void testSetupWorkDir() throws IOException { - Path rootDir = new Path(System.getProperty("test.build.data", "/tmp"), - "testSetupWorkDir"); - Path myWorkDir = new Path(rootDir, "./work"); - Path myTargetDir = new Path(rootDir, "./tmp"); - JobConf jConf = new JobConf(); - FileSystem fs = FileSystem.getLocal(jConf); - createEmptyDir(fs, myWorkDir); - createEmptyDir(fs, myTargetDir); - - // create subDirs and symlinks under work dir - createSubDirsAndSymLinks(jConf, myWorkDir, myTargetDir); - - assertTrue("Did not create symlinks/files/dirs properly. Check " - + myWorkDir + " and " + myTargetDir, - (fs.listStatus(myWorkDir).length == 4) && - (fs.listStatus(myTargetDir).length == 2)); - - // let us disable creation of symlinks in setupWorkDir() - jConf.set(MRJobConfig.CACHE_SYMLINK, "no"); - - // Deletion of myWorkDir should not affect contents of myTargetDir. - // myTargetDir is like $user/jobcache/distcache - TaskRunner.setupWorkDir(jConf, new File(myWorkDir.toUri().getPath())); - - // Contents of myWorkDir should be cleaned up and a tmp dir should be - // created under myWorkDir - assertTrue(myWorkDir + " is not cleaned up properly.", - fs.exists(myWorkDir) && (fs.listStatus(myWorkDir).length == 1)); - - // Make sure that the dir under myWorkDir is tmp - assertTrue(fs.listStatus(myWorkDir)[0].getPath().toUri().getPath() - .toString().equals(myWorkDir.toString() + "/tmp")); - - // Make sure that myTargetDir is not changed/deleted - assertTrue("Dir " + myTargetDir + " seem to be modified.", - fs.exists(myTargetDir) && (fs.listStatus(myTargetDir).length == 2)); - - // cleanup - fs.delete(rootDir, true); - } - - /** - * Validates distributed cache symlink getting created fine - * - * @throws IOException, URISyntaxException - */ - public void testSetupWorkDirDistCacheSymlinkValid() - throws IOException, URISyntaxException { - JobConf jConf = new JobConf(); - FileSystem fs = FileSystem.getLocal(jConf); - - Path rootDir = new Path(System.getProperty("test.build.data", "/tmp"), - "testSetupWorkDirSymlinkFailure"); - - // create file for DistributedCache and set it - Path myTargetDir = new Path(rootDir, "./tmp"); - createEmptyDir(fs, myTargetDir); - createFile(fs, myTargetDir, "cacheFile.txt"); - TrackerDistributedCacheManager.setLocalFiles(jConf, - (myTargetDir.toString()+Path.SEPARATOR+"cacheFile.txt")); - assertTrue("Did not create cache file in " + myTargetDir, - (fs.listStatus(myTargetDir).length == 1)); - - // let us enable creation of symlinks in setupWorkDir() - jConf.set(MRJobConfig.CACHE_SYMLINK, "yes"); - - // add a valid symlink - Path myWorkDir = new Path(rootDir, "./work"); - createEmptyDir(fs, myWorkDir); - DistributedCache.addCacheFile(new URI(myWorkDir.toString() + - Path.SEPARATOR + "file.txt#valid"), jConf); - - // setupWorkDir should create symlinks - TaskRunner.setupWorkDir(jConf, new File(myWorkDir.toUri().getPath())); - - // myWorkDir should have 2 entries, a tmp dir and the symlink valid - assertTrue(myWorkDir + " does not have cache symlink.", - fs.exists(myWorkDir) && (fs.listStatus(myWorkDir).length == 2)); - - // make sure work dir has symlink valid - boolean foundValid = false; - for (FileStatus fstat : fs.listStatus(myWorkDir)) { - if (fstat.getPath().toUri() != null && - fstat.getPath().toUri().getPath().toString() - .equals(myWorkDir.toString() + Path.SEPARATOR+ "valid")) { - foundValid = true; - } - } - - assertTrue("Valid symlink not created", foundValid); - - // cleanup - fs.delete(rootDir, true); - } - - /** - * Invalid distributed cache files errors out with IOException - * - * @throws IOException, URISyntaxException - */ - public void testSetupWorkDirDistCacheSymlinkInvalid() - throws IOException, URISyntaxException { - JobConf jConf = new JobConf(); - FileSystem fs = FileSystem.getLocal(jConf); - - Path rootDir = new Path(System.getProperty("test.build.data", "/tmp"), - "testSetupWorkDirSymlinkFailure"); - - // create file for DistributedCache and set it - Path myTargetDir = new Path(rootDir, "./tmp"); - createEmptyDir(fs, myTargetDir); - createFile(fs, myTargetDir, "cacheFile.txt"); - TrackerDistributedCacheManager.setLocalFiles(jConf, (myTargetDir.toString() + - Path.SEPARATOR+"cacheFile.txt")); - assertTrue("Did not create cache file in " + myTargetDir, - (fs.listStatus(myTargetDir).length == 1)); - - // let us enable creation of symlinks in setupWorkDir() - jConf.set(MRJobConfig.CACHE_SYMLINK, "yes"); - - // add an invalid symlink - Path myWorkDir = new Path(rootDir, "./work"); - createEmptyDir(fs, myWorkDir); - DistributedCache.addCacheFile(new URI(myWorkDir.toString() + - Path.SEPARATOR+"file.txt#invalid/abc"), jConf); - - // setupWorkDir should throw exception - try { - TaskRunner.setupWorkDir(jConf, new File(myWorkDir.toUri().getPath())); - assertFalse("TaskRunner.setupWorkDir() did not throw exception when" + - " given invalid cache file", true); - } catch(IOException e) { - // this is correct behavior - assertTrue(myWorkDir + " does not have cache symlink.", - fs.exists(myWorkDir) && (fs.listStatus(myWorkDir).length == 0)); - } - - // cleanup - fs.delete(rootDir, true); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java deleted file mode 100644 index fbb2867c63e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java +++ /dev/null @@ -1,443 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.security.PrivilegedExceptionAction; -import java.util.Iterator; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.TestJobInProgressListener.MyListener; -import org.apache.hadoop.mapred.UtilsForTests.FailMapper; -import org.apache.hadoop.mapred.UtilsForTests.KillMapper; -import org.apache.hadoop.mapred.lib.NullOutputFormat; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * This is a test case that tests several miscellaneous functionality. - * This is intended for a fast test and encompasses the following: - * TestJobName - * TestJobClient - * TestJobDirCleanup - * TestJobKillAndFail - * TestUserDefinedCounters - * TestJobInProgressListener - * TestJobHistory - * TestMiniMRClassPath - * TestMiniMRWithDFSWithDistinctUsers - */ - -@SuppressWarnings("deprecation") -public class TestSeveral { - - static final UserGroupInformation DFS_UGI = - TestMiniMRWithDFSWithDistinctUsers.createUGI("dfs", true); - static final UserGroupInformation TEST1_UGI = - TestMiniMRWithDFSWithDistinctUsers.createUGI("pi", false); - static final UserGroupInformation TEST2_UGI = - TestMiniMRWithDFSWithDistinctUsers.createUGI("wc", false); - - private static MiniMRCluster mrCluster = null; - private static MiniDFSCluster dfs = null; - private static FileSystem fs = null; - private static MyListener myListener = null; - - private int numReduces = 5; - private static final int numTT = 5; - - @Before - public void setUp() throws Exception { - - Configuration conf = new Configuration(); - conf.setInt("dfs.replication", 1); - dfs = new MiniDFSCluster(conf, numTT, true, null); - fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { - public FileSystem run() throws IOException { - return dfs.getFileSystem(); - } - }); - - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", - "mapred", (short)01777); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", - "mapred", (short)01777); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, - conf.get(JTConfig.JT_STAGING_AREA_ROOT), - "mapred", "mapred", (short)01777); - - UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); - - // Create a TestJobInProgressListener.MyListener and associate - // it with the MiniMRCluster - - myListener = new MyListener(); - conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); - mrCluster = new MiniMRCluster(0, 0, - numTT, fs.getUri().toString(), - 1, null, null, MR_UGI, new JobConf()); - // make cleanup inline sothat validation of existence of these directories - // can be done - mrCluster.setInlineCleanupThreads(); - - mrCluster.getJobTrackerRunner().getJobTracker() - .addJobInProgressListener(myListener); - } - - @After - public void tearDown() throws Exception { - if (fs != null) { fs.close(); } - if (dfs != null) { dfs.shutdown(); } - if (mrCluster != null) { mrCluster.shutdown(); } - } - - /** - * Utility class to create input for the jobs - * @param inDir - * @param conf - * @throws IOException - */ - private void makeInput(Path inDir, JobConf conf) throws IOException { - FileSystem inFs = inDir.getFileSystem(conf); - - if (inFs.exists(inDir)) { - inFs.delete(inDir, true); - } - inFs.mkdirs(inDir); - Path inFile = new Path(inDir, "part-0"); - DataOutputStream file = inFs.create(inFile); - for (int i = 0; i < numReduces; i++) { - file.writeBytes("b a\n"); - } - file.close(); - } - - /** - * Clean the Output directories before running a Job - * @param fs - * @param outDir - */ - private void clean(FileSystem fs, Path outDir) { - try { - fs.delete(outDir, true); - } catch (Exception e) {} - } - - private void verifyOutput(FileSystem fs, Path outDir) throws IOException { - Path[] outputFiles = FileUtil.stat2Paths( - fs.listStatus(outDir, new Utils.OutputFileUtils.OutputFilesFilter())); - assertEquals(numReduces, outputFiles.length); - InputStream is = fs.open(outputFiles[0]); - BufferedReader reader = new BufferedReader(new InputStreamReader(is)); - String s = reader.readLine().split("\t")[1]; - assertEquals("b a",s); - assertNull(reader.readLine()); - reader.close(); - } - - - @SuppressWarnings("unchecked") - static class DoNothingReducer extends MapReduceBase implements - Reducer { - public void reduce(WritableComparable key, Iterator val, - OutputCollector output, - Reporter reporter) - throws IOException { // Do nothing - } - } - - /** - * Submit a job with a complex name (TestJobName.testComplexName) - * Check the status of the job as successful (TestJobKillAndFail) - * Check that the task tracker directory is cleaned up (TestJobDirCleanup) - * Create some user defined counters and check them (TestUserDefinedCounters) - * Job uses a reducer from an External Jar (TestMiniMRClassPath) - * Check task directories (TestMiniMRWithDFS) - * Check if the listener notifications are received(TestJobInProgressListener) - * Verify if priority changes to the job are reflected (TestJobClient) - * Validate JobHistory file format, content, userlog location (TestJobHistory) - * - * @throws Exception - * - * TODO fix testcase - */ - @Test - @Ignore - public void testSuccessfulJob() throws Exception { - final JobConf conf = mrCluster.createJobConf(); - - // Set a complex Job name (TestJobName) - conf.setJobName("[name][some other value that gets" + - " truncated internally that this test attempts to aggravate]"); - conf.setInputFormat(TextInputFormat.class); - conf.setOutputFormat(TextOutputFormat.class); - - conf.setMapOutputKeyClass(LongWritable.class); - conf.setMapOutputValueClass(Text.class); - - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - - conf.setCompressMapOutput(true); - - // Set the Mapper class to a Counting Mapper that defines user - // defined counters - conf.setMapperClass(TestUserDefinedCounters.CountingMapper.class); - - conf.set("mapred.reducer.class", "testjar.ExternalIdentityReducer"); - - conf.setLong(org.apache.hadoop.mapreduce.lib.input. - FileInputFormat.SPLIT_MINSIZE, 1024*1024); - - conf.setNumReduceTasks(numReduces); - conf.setJobPriority(JobPriority.HIGH); - conf.setJar("build/test/mapred/testjar/testjob.jar"); - - String pattern = - TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null); - conf.setKeepTaskFilesPattern(pattern); - - final Path inDir = new Path("./test/input"); - final Path outDir = new Path("./test/output"); - - TEST1_UGI.doAs(new PrivilegedExceptionAction() { - public Void run() { - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - return null; - } - }); - - clean(fs, outDir); - final RunningJob job = TEST1_UGI.doAs(new PrivilegedExceptionAction() { - public RunningJob run() throws IOException { - makeInput(inDir, conf); - JobClient jobClient = new JobClient(conf); - return jobClient.submitJob(conf); - } - }); - - final JobID jobId = job.getID(); - - while (job.getJobState() != JobStatus.RUNNING) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - - // Check for JobInProgress Listener notification - assertFalse("Missing event notification for a running job", - myListener.contains(jobId, true)); - - job.waitForCompletion(); - - assertTrue(job.isComplete()); - assertEquals(JobStatus.SUCCEEDED,job.getJobState()); - - // check if the job success was notified - assertFalse("Missing event notification for a successful job", - myListener.contains(jobId, false)); - - // Check Task directories - TaskAttemptID taskid = new TaskAttemptID( - new TaskID(jobId, TaskType.MAP, 1),0); - TestMiniMRWithDFS.checkTaskDirectories(mrCluster, TEST1_UGI.getUserName(), - new String[] { jobId.toString() }, new String[] { taskid.toString() }); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int exitCode = TestJobClient.runTool(conf, new JobClient(), - new String[] { "-counter", jobId.toString(), - "org.apache.hadoop.mapreduce.TaskCounter", "MAP_INPUT_RECORDS" }, - out); - assertEquals(0, exitCode); - assertEquals(numReduces, Integer.parseInt(out.toString().trim())); - - // Verify if user defined counters have been updated properly - TestUserDefinedCounters.verifyCounters(job, numTT); - - // Verify job priority change (TestJobClient) - TestJobClient.verifyJobPriority(jobId.toString(), "HIGH", conf); - - // Basic check if the job did run fine - TEST1_UGI.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException { - verifyOutput(outDir.getFileSystem(conf), outDir); - - - //TestJobHistory - TestJobHistory.validateJobHistoryFileFormat( - mrCluster.getJobTrackerRunner().getJobTracker().getJobHistory(), - jobId, conf, "SUCCEEDED", false); - - TestJobHistory.validateJobHistoryFileContent(mrCluster, job, conf); - - // Since we keep setKeepTaskFilesPattern, these files should still be - // present and will not be cleaned up. - for(int i=0; i < numTT; ++i) { - Path jobDirPath = - new Path(mrCluster.getTaskTrackerLocalDir(i), TaskTracker - .getJobCacheSubdir(TEST1_UGI.getUserName())); - boolean b = FileSystem.getLocal(conf).delete(jobDirPath, true); - assertTrue(b); - } - return null; - } - }); - - } - - /** - * Submit a job with BackSlashed name (TestJobName) that will fail - * Test JobHistory User Location to none (TetsJobHistory) - * Verify directory up for the Failed Job (TestJobDirCleanup) - * Verify Event is generated for the failed job (TestJobInProgressListener) - * - * @throws Exception - * - * TODO fix testcase - */ - @Test - @Ignore - public void testFailedJob() throws Exception { - JobConf conf = mrCluster.createJobConf(); - - // Name with regex - conf.setJobName("name \\Evalue]"); - - conf.setInputFormat(TextInputFormat.class); - - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - conf.setMapperClass(FailMapper.class); - conf.setOutputFormat(NullOutputFormat.class); - conf.setJobPriority(JobPriority.HIGH); - - conf.setLong(JobContext.MAP_MAX_ATTEMPTS, 1); - - conf.setNumReduceTasks(0); - - final Path inDir = new Path("./wc/input"); - final Path outDir = new Path("./wc/output"); - - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - - clean(fs, outDir); - makeInput(inDir, conf); - - JobClient jobClient = new JobClient(conf); - RunningJob job = jobClient.submitJob(conf); - JobID jobId = job.getID(); - job.waitForCompletion(); - - assertTrue(job.isComplete()); - assertEquals(JobStatus.FAILED, job.getJobState()); - - // check if the job failure was notified - assertFalse("Missing event notification on failing a running job", - myListener.contains(jobId)); - - TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT, job.getID()); - } - - /** - * Submit a job that will get Killed with a Regex Name (TestJobName) - * Verify Job Directory Cleanup (TestJobDirCleanup) - * Verify Even is generated for Killed Job (TestJobInProgressListener) - * - * @throws Exception - * - * TODO fix testcase - */ - @Test - @Ignore - public void testKilledJob() throws Exception { - JobConf conf = mrCluster.createJobConf(); - - // Name with regex - conf.setJobName("name * abc + Evalue]"); - - conf.setInputFormat(TextInputFormat.class); - - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - conf.setMapperClass(KillMapper.class); - conf.setOutputFormat(NullOutputFormat.class); - conf.setNumReduceTasks(0); - - conf.setLong(JobContext.MAP_MAX_ATTEMPTS, 2); - - final Path inDir = new Path("./wc/input"); - final Path outDir = new Path("./wc/output"); - final Path histDir = new Path("./wc/history"); - - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - - clean(fs, outDir); - makeInput(inDir, conf); - - JobClient jobClient = new JobClient(conf); - RunningJob job = jobClient.submitJob(conf); - - while (job.getJobState() != JobStatus.RUNNING) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - job.killJob(); - - job.waitForCompletion(); - - assertTrue(job.isComplete()); - assertEquals(JobStatus.KILLED, job.getJobState()); - - // check if the job failure was notified - assertFalse("Missing event notification on killing a running job", - myListener.contains(job.getID())); - - TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT, job.getID()); - } - -} - - diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleExceptionCount.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleExceptionCount.java deleted file mode 100644 index 9de261b3136..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleExceptionCount.java +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -import org.apache.hadoop.mapred.TaskTracker.ShuffleServerMetrics; -import org.apache.hadoop.metrics.ContextFactory; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.spi.OutputRecord; -import org.junit.Test; - -public class TestShuffleExceptionCount { - - public static class TestMapOutputServlet extends TaskTracker.MapOutputServlet { - - public void checkException(IOException ie, String exceptionMsgRegex, - String exceptionStackRegex, ShuffleServerMetrics shuffleMetrics) { - super.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - } - - } - - @Test - public void testCheckException() throws IOException, InterruptedException, - ClassNotFoundException, InstantiationException, IllegalAccessException { - TestMapOutputServlet testServlet = new TestMapOutputServlet(); - JobConf conf = new JobConf(); - conf.setUser("testuser"); - conf.setJobName("testJob"); - conf.setSessionId("testSession"); - - // setup metrics context factory - ContextFactory factory = ContextFactory.getFactory(); - factory.setAttribute("mapred.class", - "org.apache.hadoop.metrics.spi.NoEmitMetricsContext"); - - TaskTracker tt = new TaskTracker(); - tt.setConf(conf); - ShuffleServerMetrics shuffleMetrics = tt.new ShuffleServerMetrics(conf); - - // first test with only MsgRegex set but doesn't match - String exceptionMsgRegex = "Broken pipe"; - String exceptionStackRegex = null; - IOException ie = new IOException("EOFException"); - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - - MetricsContext context = factory.getContext("mapred"); - shuffleMetrics.doUpdates(context); - Map> records = context.getAllRecords(); - Collection col = records.get("shuffleOutput"); - OutputRecord outputRecord = col.iterator().next(); - assertEquals(0, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with only MsgRegex set that does match - ie = new IOException("Broken pipe"); - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - - shuffleMetrics.doUpdates(context); - assertEquals(1, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with neither set, make sure incremented - exceptionStackRegex = null; - exceptionMsgRegex = null; - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(2, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with only StackRegex set doesn't match - exceptionStackRegex = ".*\\.doesnt\\$SelectSet\\.wakeup.*"; - exceptionMsgRegex = null; - ie.setStackTrace(constructStackTrace()); - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(2, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with only StackRegex set does match - exceptionStackRegex = ".*\\.SelectorManager\\$SelectSet\\.wakeup.*"; - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(3, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with both regex set and matches - exceptionMsgRegex = "Broken pipe"; - ie.setStackTrace(constructStackTraceTwo()); - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(4, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with both regex set and only msg matches - exceptionStackRegex = ".*[1-9]+BOGUSREGEX"; - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(4, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - - // test with both regex set and only stack matches - exceptionStackRegex = ".*\\.SelectorManager\\$SelectSet\\.wakeup.*"; - exceptionMsgRegex = "EOFException"; - testServlet.checkException(ie, exceptionMsgRegex, exceptionStackRegex, - shuffleMetrics); - shuffleMetrics.doUpdates(context); - assertEquals(4, outputRecord.getMetric("shuffle_exceptions_caught") - .intValue()); - } - - /* - * Construction exception like: java.io.IOException: Broken pipe at - * sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at - * sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:256) at - * sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:175) at - * org.mortbay - * .io.nio.SelectorManager$SelectSet.wakeup(SelectorManager.java:831) at - * org.mortbay - * .io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:709) at - * org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) at - * org - * .mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java - * :124) at - * org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector. - * java:708) at - * org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool - * .java:582) - */ - private StackTraceElement[] constructStackTrace() { - StackTraceElement[] stack = new StackTraceElement[9]; - stack[0] = new StackTraceElement("sun.nio.ch.EPollArrayWrapper", - "interrupt", "", -2); - stack[1] = new StackTraceElement("sun.nio.ch.EPollArrayWrapper", - "interrupt", "EPollArrayWrapper.java", 256); - stack[2] = new StackTraceElement("sun.nio.ch.EPollSelectorImpl", "wakeup", - "EPollSelectorImpl.java", 175); - stack[3] = new StackTraceElement( - "org.mortbay.io.nio.SelectorManager$SelectSet", "wakeup", - "SelectorManager.java", 831); - stack[4] = new StackTraceElement( - "org.mortbay.io.nio.SelectorManager$SelectSet", "doSelect", - "SelectorManager.java", 709); - stack[5] = new StackTraceElement("org.mortbay.io.nio.SelectorManager", - "doSelect", "SelectorManager.java", 192); - stack[6] = new StackTraceElement( - "org.mortbay.jetty.nio.SelectChannelConnector", "accept", - "SelectChannelConnector.java", 124); - stack[7] = new StackTraceElement( - "org.mortbay.jetty.AbstractConnector$Acceptor", "run", - "AbstractConnector.java", 708); - stack[8] = new StackTraceElement( - "org.mortbay.thread.QueuedThreadPool$PoolThread", "run", - "QueuedThreadPool.java", 582); - - return stack; - } - - /* - * java.io.IOException: Broken pipe at - * sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at - * sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:256) at - * sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:175) at - * org.mortbay - * .io.nio.SelectorManager$SelectSet.wakeup(SelectorManager.java:831) at - * org.mortbay - * .io.nio.SelectChannelEndPoint.updateKey(SelectChannelEndPoint.java:335) at - * org - * .mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint - * .java:278) at - * org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator - * .java:545) at - * org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator - * .java:572) at - * org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:1012) at - * org - * .mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:651)at - * org - * .mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) - * at - */ - private StackTraceElement[] constructStackTraceTwo() { - StackTraceElement[] stack = new StackTraceElement[11]; - stack[0] = new StackTraceElement("sun.nio.ch.EPollArrayWrapper", - "interrupt", "", -2); - stack[1] = new StackTraceElement("sun.nio.ch.EPollArrayWrapper", - "interrupt", "EPollArrayWrapper.java", 256); - stack[2] = new StackTraceElement("sun.nio.ch.EPollSelectorImpl", "wakeup", - "EPollSelectorImpl.java", 175); - stack[3] = new StackTraceElement( - "org.mortbay.io.nio.SelectorManager$SelectSet", "wakeup", - "SelectorManager.java", 831); - stack[4] = new StackTraceElement( - "org.mortbay.io.nio.SelectChannelEndPoint", "updateKey", - "SelectChannelEndPoint.java", 335); - stack[5] = new StackTraceElement( - "org.mortbay.io.nio.SelectChannelEndPoint", "blockWritable", - "SelectChannelEndPoint.java", 278); - stack[6] = new StackTraceElement( - "org.mortbay.jetty.AbstractGenerator$Output", "blockForOutput", - "AbstractGenerator.java", 545); - stack[7] = new StackTraceElement( - "org.mortbay.jetty.AbstractGenerator$Output", "flush", - "AbstractGenerator.java", 572); - stack[8] = new StackTraceElement("org.mortbay.jetty.HttpConnection$Output", - "flush", "HttpConnection.java", 1012); - stack[9] = new StackTraceElement( - "org.mortbay.jetty.AbstractGenerator$Output", "write", - "AbstractGenerator.java", 651); - stack[10] = new StackTraceElement( - "org.mortbay.jetty.AbstractGenerator$Output", "write", - "AbstractGenerator.java", 580); - - return stack; - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleJobToken.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleJobToken.java deleted file mode 100644 index 70722dced24..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestShuffleJobToken.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - - -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; -import java.security.GeneralSecurityException; - -import javax.crypto.SecretKey; - -import org.apache.hadoop.http.HttpServer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; -import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; -import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; -import org.apache.hadoop.security.token.Token; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.assertTrue; - -public class TestShuffleJobToken { - private static HttpServer server; - private static URL baseUrl; - private static File dir; - private static final String JOB_ID = "job_20091117075357176_0001"; - private static final String BAD_JOB_ID = "job_20091117075357176_0002"; - - // create fake url - private URL getMapOutputURL(String host) throws MalformedURLException { - // Get the base url - StringBuffer url = new StringBuffer(host); - url.append("mapOutput?"); - url.append("job=" + JOB_ID + "&"); - url.append("reduce=0&"); - url.append("map=attempt"); - - return new URL(url.toString()); - } - - @Before - public void setUp() throws Exception { - dir = new File(System.getProperty("build.webapps", "build/webapps") + "/test"); - System.out.println("dir="+dir.getAbsolutePath()); - if(!dir.exists()) { - assertTrue(dir.mkdirs()); - } - server = new HttpServer("test", "0.0.0.0", 0, true); - server.addServlet("shuffle", "/mapOutput", TaskTracker.MapOutputServlet.class); - server.start(); - int port = server.getPort(); - baseUrl = new URL("http://localhost:" + port + "/"); - } - - @After - public void tearDown() throws Exception { - if(dir.exists()) - dir.delete(); - if(server!=null) - server.stop(); - } - - - /** - * try positive and negative case with invalid urlHash - */ - @Test - public void testInvalidJobToken() - throws IOException, GeneralSecurityException { - - URL url = getMapOutputURL(baseUrl.toString()); - String enc_str = SecureShuffleUtils.buildMsgFrom(url); - URLConnection connectionGood = url.openConnection(); - - TaskTracker tt = new TaskTracker(); - JobTokenSecretManager jtSecretManager = new JobTokenSecretManager(); - // create fake TaskTracker - needed for keys storage - JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(JOB_ID)); - Token jt = new Token(identifier, - jtSecretManager); - SecretKey tokenSecret = JobTokenSecretManager.createSecretKey(jt.getPassword()); - addJobToken(tt, JOB_ID, jt); // fake id - server.setAttribute("task.tracker", tt); - - // encode the url - String urlHashGood = SecureShuffleUtils.generateHash(enc_str.getBytes(), tokenSecret); // valid hash - - // another the key - JobTokenIdentifier badIdentifier = new JobTokenIdentifier(new Text(BAD_JOB_ID)); - Token badToken = new Token(badIdentifier, - jtSecretManager); - SecretKey badSecret = JobTokenSecretManager.createSecretKey(badToken.getPassword()); - String urlHashBad = SecureShuffleUtils.generateHash(enc_str.getBytes(), badSecret); // invalid hash - - // put url hash into http header - connectionGood.addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, urlHashGood); - - // valid url hash should not fail with security error - try { - connectionGood.getInputStream(); - } catch (IOException ie) { - String msg = ie.getLocalizedMessage(); - if(msg.contains("Server returned HTTP response code: 401 for URL:")) { - fail("securtity failure with valid urlHash:"+ie); - } - System.out.println("valid urlhash passed validation"); - } - // invalid url hash - URLConnection connectionBad = url.openConnection(); - connectionBad.addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, urlHashBad); - - try { - connectionBad.getInputStream(); - fail("Connection should've failed because of invalid urlHash"); - } catch (IOException ie) { - String msg = ie.getLocalizedMessage(); - if(!msg.contains("Server returned HTTP response code: 401 for URL:")) { - fail("connection failed with other then validation error:"+ie); - } - System.out.println("validation worked, failed with:"+ie); - } - } - /*Note that this method is there for a unit testcase (TestShuffleJobToken)*/ - void addJobToken(TaskTracker tt, String jobIdStr, Token token) { - JobID jobId = JobID.forName(jobIdStr); - TaskTracker.RunningJob rJob = new TaskTracker.RunningJob(jobId); - synchronized (tt.runningJobs) { - tt.runningJobs.put(jobId, rJob); - } - tt.getJobTokenSecretManager().addTokenForJob(jobIdStr, token); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java deleted file mode 100644 index f7eefe9649b..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java +++ /dev/null @@ -1,333 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker; -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTrackerMetricsInst; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -public class TestSpeculativeExecution extends TestCase { - - FakeJobInProgress job; - static FakeJobTracker jobTracker; - static class SpecFakeClock extends FakeClock { - long SPECULATIVE_LAG = TaskInProgress.SPECULATIVE_LAG; - public void advanceBySpeculativeLag() { - time += SPECULATIVE_LAG; - } - }; - static SpecFakeClock clock; - static final Log LOG = LogFactory.getLog(TestSpeculativeExecution.class); - private static FakeJobTrackerMetricsInst fakeInst; - - - static String trackers[] = new String[] {"tracker_tracker1:1000", - "tracker_tracker2:1000", "tracker_tracker3:1000", - "tracker_tracker4:1000", "tracker_tracker5:1000"}; - - public static Test suite() { - TestSetup setup = - new TestSetup(new TestSuite(TestSpeculativeExecution.class)) { - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.set(JTConfig.JT_INSTRUMENTATION, - FakeJobTrackerMetricsInst.class.getName()); - jobTracker = new FakeJobTracker(conf, (clock = new SpecFakeClock()), - trackers); - fakeInst = (FakeJobTrackerMetricsInst) jobTracker.getInstrumentation(); - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - } - protected void tearDown() throws Exception { - //delete the build/test/logs/ dir - } - }; - return setup; - } - - public void testRunningTaskCountWithSpeculation() throws IOException { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[8]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(true); - conf.setNumMapTasks(3); - conf.setNumReduceTasks(3); - conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - - //Check for runningMap counts first - //schedule maps - taskAttemptID[0] = job.findMapTask(trackers[0]); - taskAttemptID[1] = job.findMapTask(trackers[1]); - taskAttemptID[2] = job.findMapTask(trackers[2]); - - clock.advance(5000); - job.finishTask(taskAttemptID[0]); - clock.advance(1000); - job.finishTask(taskAttemptID[1]); - clock.advanceBySpeculativeLag(); - - //we should get a speculative task now - taskAttemptID[3] = job.findMapTask(trackers[3]); - int oldRunningMap = job.runningMaps(); - LOG.info("No of running maps before fail was " + oldRunningMap); - job.failTask(taskAttemptID[2]); - assertEquals( - "Running maps count should be updated from " + oldRunningMap + " to " + - (oldRunningMap - 1), job.runningMaps(), oldRunningMap - 1); - LOG.info(" Job running maps after fail " + job.runningMaps()); - - clock.advance(5000); - job.finishTask(taskAttemptID[3]); - - //check for runningReduce count. - taskAttemptID[4] = job.findReduceTask(trackers[0]); - taskAttemptID[5] = job.findReduceTask(trackers[1]); - taskAttemptID[6] = job.findReduceTask(trackers[2]); - - clock.advance(5000); - job.finishTask(taskAttemptID[4]); - clock.advance(1000); - job.finishTask(taskAttemptID[5]); - - clock.advanceBySpeculativeLag(); - taskAttemptID[7] = job.findReduceTask(trackers[4]); - - int oldRunningReduces = job.runningReduces(); - job.failTask(taskAttemptID[6]); - LOG.info( - " No of running Reduces before fail " + oldRunningReduces); - LOG.info( - " No of runing reduces after fail " + job.runningReduces()); - assertEquals( - "Running reduces count should be updated from " + oldRunningReduces + - " to " + (oldRunningReduces - 1), job.runningReduces(), - oldRunningReduces - 1); - // Verify total speculative tasks by jobtracker instrumentation - assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps); - assertEquals("Total speculative reduces", 1, - fakeInst.numSpeculativeReduces); - LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); - LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces); - - job.finishTask(taskAttemptID[7]); - } - - public void testIsSlowTracker() throws IOException { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[20]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(true); - conf.setNumMapTasks(10); - conf.setNumReduceTasks(0); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - //schedule some tasks - taskAttemptID[0] = job.findMapTask(trackers[0]); - taskAttemptID[1] = job.findMapTask(trackers[0]); - taskAttemptID[2] = job.findMapTask(trackers[0]); - taskAttemptID[3] = job.findMapTask(trackers[1]); - taskAttemptID[4] = job.findMapTask(trackers[1]); - taskAttemptID[5] = job.findMapTask(trackers[1]); - taskAttemptID[6] = job.findMapTask(trackers[2]); - taskAttemptID[7] = job.findMapTask(trackers[2]); - taskAttemptID[8] = job.findMapTask(trackers[2]); - clock.advance(1000); - //Some tasks finish in 1 second (on trackers[0]) - job.finishTask(taskAttemptID[0]); - job.finishTask(taskAttemptID[1]); - job.finishTask(taskAttemptID[2]); - clock.advance(1000); - //Some tasks finish in 2 second (on trackers[1]) - job.finishTask(taskAttemptID[3]); - job.finishTask(taskAttemptID[4]); - job.finishTask(taskAttemptID[5]); - assertEquals("Tracker "+ trackers[0] + " expected to be not slow ", - job.isSlowTracker(trackers[0]), false); - clock.advance(100000); - //After a long time, some tasks finished on trackers[2] - job.finishTask(taskAttemptID[6]); - job.finishTask(taskAttemptID[7]); - job.finishTask(taskAttemptID[8]); - assertEquals("Tracker "+ trackers[2] + " expected to be slow ", - job.isSlowTracker(trackers[2]), true); - // Verify total speculative tasks by jobtracker instrumentation - assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps); - assertEquals("Total speculative reduces", 1, - fakeInst.numSpeculativeReduces); - LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); - LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces); - } - - public void testTaskToSpeculate() throws IOException { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[6]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(true); - conf.setNumMapTasks(5); - conf.setNumReduceTasks(5); - conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - //schedule maps - taskAttemptID[0] = job.findReduceTask(trackers[0]); - taskAttemptID[1] = job.findReduceTask(trackers[1]); - taskAttemptID[2] = job.findReduceTask(trackers[2]); - taskAttemptID[3] = job.findReduceTask(trackers[3]); - taskAttemptID[4] = job.findReduceTask(trackers[3]); - clock.advance(5000); - job.finishTask(taskAttemptID[0]); - clock.advance(1000); - job.finishTask(taskAttemptID[1]); - clock.advance(20000); - clock.advanceBySpeculativeLag(); - //we should get a speculative task now - taskAttemptID[5] = job.findReduceTask(trackers[4]); - assertEquals(taskAttemptID[5].getTaskID().getId(),2); - clock.advance(5000); - job.finishTask(taskAttemptID[5]); - - taskAttemptID[5] = job.findReduceTask(trackers[4]); - assertEquals(taskAttemptID[5].getTaskID().getId(),3); - - // Verify total speculative tasks by jobtracker instrumentation - assertEquals("Total speculative maps", 1, fakeInst.numSpeculativeMaps); - assertEquals("Total speculative reduces", 3, - fakeInst.numSpeculativeReduces); - LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); - LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces); - } - - /* - * Tests the fact that we choose tasks with lesser progress - * among the possible candidates for speculation - */ - public void testTaskLATEScheduling() throws IOException { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[20]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(true); - conf.setNumMapTasks(5); - conf.setNumReduceTasks(0); - conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - - taskAttemptID[0] = job.findMapTask(trackers[0]); - taskAttemptID[1] = job.findMapTask(trackers[1]); - taskAttemptID[2] = job.findMapTask(trackers[2]); - taskAttemptID[3] = job.findMapTask(trackers[3]); - clock.advance(2000); - job.finishTask(taskAttemptID[0]); - job.finishTask(taskAttemptID[1]); - job.finishTask(taskAttemptID[2]); - clock.advance(250000); - taskAttemptID[4] = job.findMapTask(trackers[3]); - clock.advanceBySpeculativeLag(); - //by doing the above clock adjustments, we bring the progress rate of - //taskID 3 lower than 4. For taskID 3, the rate is 85/317000 - //and for taskID 4, the rate is 20/65000. But when we ask for a spec task - //now, we should get back taskID 4 (since that is expected to complete - //later than taskID 3). - job.progressMade(taskAttemptID[3], 0.85f); - job.progressMade(taskAttemptID[4], 0.20f); - taskAttemptID[5] = job.findMapTask(trackers[4]); - assertEquals(taskAttemptID[5].getTaskID().getId(),4); - // Verify total speculative tasks by jobtracker instrumentation - assertEquals("Total speculative maps", 2, fakeInst.numSpeculativeMaps); - assertEquals("Total speculative reduces", 3, - fakeInst.numSpeculativeReduces); - LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); - LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces); - } - - /* - * Tests the fact that we only launch a limited number of speculative tasks, - * even though we have a lot of tasks in RUNNING state - */ - public void testAtSpeculativeCap() throws IOException { - //The expr which is evaluated for determining whether - //atSpeculativeCap should - //return true or false is - //(#speculative-tasks < max (10, 0.01*#slots, 0.1*#running-tasks) - - //Tests the fact that the max tasks launched is 0.1 * #running-tasks - assertEquals(speculativeCap(1200,800,20), 40); - //Tests the fact that the max tasks launched is 10 - assertEquals(speculativeCap(1200,1150,20), 10); - //Tests the fact that the max tasks launched is 0.01 * #slots - assertEquals(speculativeCap(1200,1150,4000), 20); - // Verify total speculative tasks by jobtracker instrumentation - assertEquals("Total speculative maps", 72, fakeInst.numSpeculativeMaps); - assertEquals("Total speculative reduces", 3, - fakeInst.numSpeculativeReduces); - LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps); - LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces); - } - - private int speculativeCap(int totalTasks, int numEarlyComplete, int slots) - throws IOException { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[1500]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(true); - conf.setNumMapTasks(totalTasks); - conf.setNumReduceTasks(0); - jobTracker.setNumSlots(slots); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.initTasks(); - int i; - for (i = 0; i < totalTasks; i++) { - taskAttemptID[i] = job.findMapTask(trackers[0]); - } - clock.advance(5000); - for (i = 0; i < numEarlyComplete; i++) { - job.finishTask(taskAttemptID[i]); - } - - clock.advanceBySpeculativeLag(); - - for (i = numEarlyComplete; i < totalTasks; i++) { - job.progressMade(taskAttemptID[i], 0.85f); - } - clock.advance(50000); - for (i = 0; i < (totalTasks - numEarlyComplete); i++) { - taskAttemptID[i] = job.findMapTask(trackers[1]); - clock.advance(2000); - if (taskAttemptID[i] != null) { - //add some good progress constantly for the different - //task-attempts so that - //the tasktracker doesn't get into the slow trackers category - job.progressMade(taskAttemptID[i], 0.99f); - } else { - break; - } - } - return i; - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java deleted file mode 100644 index 18792dc7a34..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSubmitJob.java +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.net.URI; -import java.security.PrivilegedExceptionAction; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.protocol.ClientProtocol; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; - -/** - * Test job submission. This test checks if - * - basic : job submission via jobclient - * - cleanup : job client crashes while submitting - * - invalid job config - * - invalid memory config - * - */ -public class TestSubmitJob { - static final Log LOG = LogFactory.getLog(TestSubmitJob.class); - - private static Path TEST_DIR = - new Path(System.getProperty("test.build.data","/tmp"), - "job-submission-testing"); - - - /** - * Test to verify that jobs with invalid memory requirements are killed at the - * JT. - * - * @throws Exception - */ - @SuppressWarnings("deprecation") - @Test - public void testJobWithInvalidMemoryReqs() throws Exception { - MiniMRCluster mrCluster = null; - try { - JobConf jtConf = new JobConf(); - jtConf.setLong(MRConfig.MAPMEMORY_MB, 1 * 1024L); - jtConf.setLong(MRConfig.REDUCEMEMORY_MB, 2 * 1024L); - jtConf.setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 3 * 1024L); - jtConf.setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 4 * 1024L); - - mrCluster = new MiniMRCluster(0, "file:///", 0, null, null, jtConf); - - JobConf clusterConf = mrCluster.createJobConf(); - - // No map-memory configuration - JobConf jobConf = new JobConf(clusterConf); - jobConf.setMemoryForReduceTask(1 * 1024L); - runJobAndVerifyFailure(jobConf, JobConf.DISABLED_MEMORY_LIMIT, 1 * 1024L, - "Invalid job requirements."); - - // No reduce-memory configuration - jobConf = new JobConf(clusterConf); - jobConf.setMemoryForMapTask(1 * 1024L); - runJobAndVerifyFailure(jobConf, 1 * 1024L, JobConf.DISABLED_MEMORY_LIMIT, - "Invalid job requirements."); - - // Invalid map-memory configuration - jobConf = new JobConf(clusterConf); - jobConf.setMemoryForMapTask(4 * 1024L); - jobConf.setMemoryForReduceTask(1 * 1024L); - runJobAndVerifyFailure(jobConf, 4 * 1024L, 1 * 1024L, - "Exceeds the cluster's max-memory-limit."); - - // No reduce-memory configuration - jobConf = new JobConf(clusterConf); - jobConf.setMemoryForMapTask(1 * 1024L); - jobConf.setMemoryForReduceTask(5 * 1024L); - runJobAndVerifyFailure(jobConf, 1 * 1024L, 5 * 1024L, - "Exceeds the cluster's max-memory-limit."); - } finally { - if (mrCluster != null) - mrCluster.shutdown(); - } - } - - @SuppressWarnings("deprecation") - private void runJobAndVerifyFailure(JobConf jobConf, long memForMapTasks, - long memForReduceTasks, String expectedMsg) - throws Exception, - IOException { - String[] args = { "-m", "0", "-r", "0", "-mt", "0", "-rt", "0" }; - boolean throwsException = false; - String msg = null; - try { - ToolRunner.run(jobConf, new SleepJob(), args); - } catch (RemoteException re) { - throwsException = true; - msg = re.unwrapRemoteException().getMessage(); - } - assertTrue(throwsException); - assertNotNull(msg); - - String overallExpectedMsg = - "(" + memForMapTasks + " memForMapTasks " + memForReduceTasks - + " memForReduceTasks): " + expectedMsg; - assertTrue("Observed message - " + msg - + " - doesn't contain expected message - " + overallExpectedMsg, msg - .contains(overallExpectedMsg)); - } - - @SuppressWarnings("deprecation") - static ClientProtocol getJobSubmitClient(JobConf conf, - UserGroupInformation ugi) - throws IOException { - return (ClientProtocol) RPC.getProxy(ClientProtocol.class, - ClientProtocol.versionID, JobTracker.getAddress(conf), ugi, - conf, NetUtils.getSocketFactory(conf, ClientProtocol.class)); - } - - /** - * Submit a job and check if the files are accessible to other users. - */ - @SuppressWarnings("deprecation") - @Test - public void testSecureJobExecution() throws Exception { - LOG.info("Testing secure job submission/execution"); - MiniMRCluster mr = null; - Configuration conf = new Configuration(); - final MiniDFSCluster dfs = new MiniDFSCluster(conf, 1, true, null); - try { - FileSystem fs = - TestMiniMRWithDFSWithDistinctUsers.DFS_UGI.doAs(new PrivilegedExceptionAction() { - public FileSystem run() throws IOException { - return dfs.getFileSystem(); - } - }); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777); - TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT), - "mapred", "mapred", (short)01777); - - UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); - mr = new MiniMRCluster(0, 0, 1, dfs.getFileSystem().getUri().toString(), - 1, null, null, MR_UGI); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - - // cleanup - dfs.getFileSystem().delete(TEST_DIR, true); - - final Path mapSignalFile = new Path(TEST_DIR, "map-signal"); - final Path reduceSignalFile = new Path(TEST_DIR, "reduce-signal"); - - // create a ugi for user 1 - UserGroupInformation user1 = - TestMiniMRWithDFSWithDistinctUsers.createUGI("user1", false); - Path inDir = new Path("/user/input"); - Path outDir = new Path("/user/output"); - final JobConf job = mr.createJobConf(); - - UtilsForTests.configureWaitingJobConf(job, inDir, outDir, 2, 0, - "test-submit-job", mapSignalFile.toString(), - reduceSignalFile.toString()); - job.set(UtilsForTests.getTaskSignalParameter(true), - mapSignalFile.toString()); - job.set(UtilsForTests.getTaskSignalParameter(false), - reduceSignalFile.toString()); - LOG.info("Submit job as the actual user (" + user1.getUserName() + ")"); - final JobClient jClient = - user1.doAs(new PrivilegedExceptionAction() { - public JobClient run() throws IOException { - return new JobClient(job); - } - }); - RunningJob rJob = user1.doAs(new PrivilegedExceptionAction() { - public RunningJob run() throws IOException { - return jClient.submitJob(job); - } - }); - JobID id = rJob.getID(); - LOG.info("Running job " + id); - - // create user2 - UserGroupInformation user2 = - TestMiniMRWithDFSWithDistinctUsers.createUGI("user2", false); - final JobConf conf_other = mr.createJobConf(); - - FileSystem fs2 = user2.doAs(new PrivilegedExceptionAction() { - @Override - public FileSystem run() throws Exception { - return FileSystem.get(conf_other); - } - }); - - // try accessing mapred.system.dir/jobid/* - try { - Path path = new Path(jt.getSystemDir()); - LOG.info("Try listing the mapred-system-dir as the user (" - + user2.getUserName() + ")"); - fs2.listStatus(path); - fail("JobTracker system dir is accessible to others"); - } catch (IOException ioe) { - assertTrue(ioe.toString(), - ioe.toString().contains("Permission denied")); - } - // try accessing ~/.staging/jobid/* - JobInProgress jip = jt.getJob(id); - Path jobSubmitDirpath = - new Path(jip.getJobConf().get("mapreduce.job.dir")); - try { - LOG.info("Try accessing the job folder for job " + id + " as the user (" - + user2.getUserName() + ")"); - fs2.listStatus(jobSubmitDirpath); - fail("User's staging folder is accessible to others"); - } catch (IOException ioe) { - assertTrue(ioe.toString(), - ioe.toString().contains("Permission denied")); - } - UtilsForTests.signalTasks(dfs, fs, true, mapSignalFile.toString(), - reduceSignalFile.toString()); - // wait for job to be done - UtilsForTests.waitTillDone(jClient); - - // check if the staging area is cleaned up - LOG.info("Check if job submit dir is cleanup or not"); - assertFalse(fs.exists(jobSubmitDirpath)); - } finally { - if (mr != null) { - mr.shutdown(); - } - if (dfs != null) { - dfs.shutdown(); - } - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskChildOptsParsing.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskChildOptsParsing.java deleted file mode 100644 index d952ab83464..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskChildOptsParsing.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; - -import org.junit.Test; - -public class TestTaskChildOptsParsing { - - @SuppressWarnings("deprecation") - private static final TaskAttemptID TASK_ID = new TaskAttemptID(); - private static final String[] EXPECTED_RESULTS = new String[]{"-Dfoo=bar", "-Dbaz=biz"}; - - private void performTest(String input) { - String[] result = TaskRunner.parseChildJavaOpts(input, TASK_ID); - assertArrayEquals(EXPECTED_RESULTS, result); - } - - @Test - public void testParseChildJavaOptsLeadingSpace() { - performTest(" -Dfoo=bar -Dbaz=biz"); - } - - @Test - public void testParseChildJavaOptsTrailingSpace() { - performTest("-Dfoo=bar -Dbaz=biz "); - } - - @Test - public void testParseChildJavaOptsOneSpace() { - performTest("-Dfoo=bar -Dbaz=biz"); - } - - @Test - public void testParseChildJavaOptsMulitpleSpaces() { - performTest("-Dfoo=bar -Dbaz=biz"); - } - - @Test - public void testParseChildJavaOptsOneTab() { - performTest("-Dfoo=bar\t-Dbaz=biz"); - } - - @Test - public void testParseChildJavaOptsMultipleTabs() { - performTest("-Dfoo=bar\t\t-Dbaz=biz"); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java deleted file mode 100644 index 5357e2ee422..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java +++ /dev/null @@ -1,288 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.OutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.URL; -import java.net.HttpURLConnection; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskType; - -public class TestTaskFail extends TestCase { - - private static final Log LOG = LogFactory.getLog( - TestTaskFail.class); - - private static String taskLog = "Task attempt log"; - static String cleanupLog = "cleanup attempt log"; - - public static class MapperClass extends MapReduceBase - implements Mapper { - String taskid; - public void configure(JobConf job) { - taskid = job.get(JobContext.TASK_ATTEMPT_ID); - } - public void map (LongWritable key, Text value, - OutputCollector output, - Reporter reporter) throws IOException { - System.err.println(taskLog); - assertFalse(Boolean.getBoolean(System - .getProperty("hadoop.tasklog.iscleanup"))); - if (taskid.endsWith("_0")) { - throw new IOException(); - } else if (taskid.endsWith("_1")) { - System.exit(-1); - } else if (taskid.endsWith("_2")) { - throw new Error(); - } - } - } - - static class CommitterWithLogs extends FileOutputCommitter { - public void abortTask(TaskAttemptContext context) throws IOException { - System.err.println(cleanupLog); - String attemptId = System.getProperty("hadoop.tasklog.taskid"); - assertNotNull(attemptId); - if (attemptId.endsWith("_0")) { - assertFalse(Boolean.getBoolean(System - .getProperty("hadoop.tasklog.iscleanup"))); - } else { - assertTrue(Boolean.getBoolean(System - .getProperty("hadoop.tasklog.iscleanup"))); - } - super.abortTask(context); - } - } - - static class CommitterWithFailTaskCleanup extends FileOutputCommitter { - public void abortTask(TaskAttemptContext context) throws IOException { - System.err.println(cleanupLog); - System.exit(-1); - } - } - - static class CommitterWithFailTaskCleanup2 extends FileOutputCommitter { - public void abortTask(TaskAttemptContext context) throws IOException { - System.err.println(cleanupLog); - throw new IOException(); - } - } - - /** access a url, ignoring some IOException such as the page does not exist */ - static int getHttpStatusCode(String urlstring, String userName, - String method) throws IOException { - LOG.info("Accessing " + urlstring + " as user " + userName); - URL url = new URL(urlstring + "&user.name=" + userName); - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod(method); - if (method.equals("POST")) { - String encodedData = "action=kill&user.name=" + userName; - connection.setRequestProperty("Content-Type", - "application/x-www-form-urlencoded"); - connection.setRequestProperty("Content-Length", - Integer.toString(encodedData.length())); - connection.setDoOutput(true); - - OutputStream os = connection.getOutputStream(); - os.write(encodedData.getBytes()); - } - connection.connect(); - - return connection.getResponseCode(); - } - - public RunningJob launchJob(JobConf conf, - Path inDir, - Path outDir, - String input) - throws IOException { - // set up the input file system and write input text. - FileSystem inFs = inDir.getFileSystem(conf); - FileSystem outFs = outDir.getFileSystem(conf); - outFs.delete(outDir, true); - if (!inFs.mkdirs(inDir)) { - throw new IOException("Mkdirs failed to create " + inDir.toString()); - } - { - // write input into input file - DataOutputStream file = inFs.create(new Path(inDir, "part-0")); - file.writeBytes(input); - file.close(); - } - - // configure the mapred Job - conf.setMapperClass(MapperClass.class); - conf.setReducerClass(IdentityReducer.class); - conf.setNumReduceTasks(0); - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - conf.setSpeculativeExecution(false); - String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", - "/tmp")).toString().replace(' ', '+'); - conf.set("test.build.data", TEST_ROOT_DIR); - // return the RunningJob handle. - return new JobClient(conf).submitJob(conf); - } - - private void validateAttempt(TaskInProgress tip, TaskAttemptID attemptId, - TaskStatus ts, boolean isCleanup, - boolean containsCleanupLog, JobTracker jt) - throws IOException { - assertEquals(isCleanup, tip.isCleanupAttempt(attemptId)); - assertTrue(ts != null); - assertEquals(TaskStatus.State.FAILED, ts.getRunState()); - // validate tasklogs for task attempt - String log = MapReduceTestUtil.readTaskLog( - TaskLog.LogName.STDERR, attemptId, false); - assertTrue(log.contains(taskLog)); - // access the logs from web url - TaskTrackerStatus ttStatus = jt.getTaskTracker( - tip.machineWhereTaskRan(attemptId)).getStatus(); - String tasklogUrl = TaskLogServlet.getTaskLogUrl("localhost", - String.valueOf(ttStatus.getHttpPort()), attemptId.toString()) + - "&filter=STDERR"; - assertEquals(HttpURLConnection.HTTP_OK, - getHttpStatusCode(tasklogUrl, tip.getUser(), "GET")); - if (containsCleanupLog) { - // validate task logs: tasklog should contain both task logs - // and cleanup logs - assertTrue(log.contains(cleanupLog)); - } - if (isCleanup) { - // validate tasklogs for cleanup attempt - log = MapReduceTestUtil.readTaskLog( - TaskLog.LogName.STDERR, attemptId, true); - assertTrue(log.contains(cleanupLog)); - // access the cleanup attempt's logs from web url - ttStatus = jt.getTaskTracker(tip.machineWhereCleanupRan(attemptId)) - .getStatus(); - String cleanupTasklogUrl = TaskLogServlet.getTaskLogUrl("localhost", - String.valueOf(ttStatus.getHttpPort()), attemptId.toString()) - + "&filter=STDERR&cleanup=true"; - assertEquals(HttpURLConnection.HTTP_OK, - getHttpStatusCode(cleanupTasklogUrl, tip.getUser(), "GET")); - - // Task-cleanup task should not be scheduled on the node that the task just failed - if (jt.taskTrackers().size() >= 2) { - String trackerRanTask = tip.machineWhereTaskRan(attemptId); - String trackerRanCleanupTask = tip.machineWhereCleanupRan(attemptId); - assertFalse(trackerRanTask.equals(trackerRanCleanupTask)); - } - } - } - - private void validateJob(RunningJob job, JobTracker jt, boolean cleanupNeeded) - throws IOException { - assertEquals(JobStatus.SUCCEEDED, job.getJobState()); - - JobID jobId = job.getID(); - // construct the task id of first map task - // this should not be cleanup attempt since the first attempt - // fails with an exception - TaskAttemptID attemptId = - new TaskAttemptID(new TaskID(jobId, TaskType.MAP, 0), 0); - TaskInProgress tip = jt.getTip(attemptId.getTaskID()); - TaskStatus ts = jt.getTaskStatus(attemptId); - // task logs will contain cleanup message because the task is failed by - // throwing IOException - validateAttempt(tip, attemptId, ts, false, true, jt); - - attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, 0), 1); - // this should be cleanup attempt since the second attempt fails - // with System.exit - ts = jt.getTaskStatus(attemptId); - validateAttempt(tip, attemptId, ts, cleanupNeeded, false, jt); - - attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, 0), 2); - // this should be cleanup attempt since the third attempt fails - // with Error - ts = jt.getTaskStatus(attemptId); - validateAttempt(tip, attemptId, ts, cleanupNeeded, false, jt); - } - - public void testWithDFS() throws IOException { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int taskTrackers = 4; - - Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 4, true, null); - fileSys = dfs.getFileSystem(); - mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - final Path inDir = new Path("./input"); - final Path outDir = new Path("./output"); - String input = "The quick brown fox\nhas many silly\nred fox sox\n"; - // launch job with fail tasks - JobConf jobConf = mr.createJobConf(); - // turn down the completion poll interval from the 5 second default - // for better test performance. - jobConf.set(Job.COMPLETION_POLL_INTERVAL_KEY, "50"); - jobConf.setOutputCommitter(CommitterWithLogs.class); - RunningJob rJob = launchJob(jobConf, inDir, outDir, input); - rJob.waitForCompletion(); - validateJob(rJob, jt, true); - // launch job with fail tasks and fail-cleanups - fileSys.delete(outDir, true); - jobConf.setOutputCommitter(CommitterWithFailTaskCleanup.class); - rJob = launchJob(jobConf, inDir, outDir, input); - rJob.waitForCompletion(); - validateJob(rJob, jt, true); - fileSys.delete(outDir, true); - jobConf.setOutputCommitter(CommitterWithFailTaskCleanup2.class); - rJob = launchJob(jobConf, inDir, outDir, input); - rJob.waitForCompletion(); - validateJob(rJob, jt, true); - // launch job with task-cleanup switched off - fileSys.delete(outDir, true); - jobConf.setOutputCommitter(CommitterWithFailTaskCleanup.class); - jobConf.setBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, false); - rJob = launchJob(jobConf, inDir, outDir, input); - rJob.waitForCompletion(); - validateJob(rJob, jt, false); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); } - } - } - - public static void main(String[] argv) throws Exception { - TestTaskFail td = new TestTaskFail(); - td.testWithDFS(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncher.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncher.java deleted file mode 100644 index f455bcb944e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncher.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.LinkedHashMap; - -import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; -import org.apache.hadoop.mapred.TaskTracker.TaskLauncher; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.junit.Test; - -/** - * Tests {@link TaskLauncher} - * - */ -public class TestTaskLauncher { - private static int expectedLaunchAttemptId = 1; - - private static class MyTaskTracker extends TaskTracker { - // override startNewTask just to set the runState, - // not to launch the task really - @Override - void startNewTask(TaskInProgress tip) { - assertEquals(expectedLaunchAttemptId, tip.getTask().getTaskID().getId()); - tip.getStatus().setRunState(TaskStatus.State.RUNNING); - } - } - - /** - * Tests the case "task waiting to be launched is killed externally". - * - * Launches a task which will wait for ever to get slots. Kill the - * task and see if launcher is able to come out of the wait and pickup a - * another task. - * - * @throws IOException - */ - @Test - public void testExternalKillForLaunchTask() throws IOException { - // setup a TaskTracker - JobConf ttConf = new JobConf(); - ttConf.setInt(TTConfig.TT_MAP_SLOTS, 4); - TaskTracker tt = new MyTaskTracker(); - tt.runningTasks = new LinkedHashMap(); - tt.setConf(ttConf); - tt.setIndexCache(new IndexCache(ttConf)); - tt.setTaskMemoryManagerEnabledFlag(); - - // Set up TaskTracker instrumentation - tt.setTaskTrackerInstrumentation( - TaskTracker.createInstrumentation(tt, tt.getJobConf())); - - // start map-task launcher with four slots - TaskLauncher mapLauncher = tt.new TaskLauncher(TaskType.MAP, 4); - mapLauncher.start(); - - // launch a task which requires five slots - String jtId = "test"; - TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, TaskType.MAP, 0, 0); - Task task = new MapTask(null, attemptID, 0, null, 5); - mapLauncher.addToTaskQueue(new LaunchTaskAction(task)); - // verify that task is added to runningTasks - TaskInProgress killTip = tt.runningTasks.get(attemptID); - assertNotNull(killTip); - - // wait for a while for launcher to pick up the task - // this loop waits atmost for 30 seconds - for (int i = 0; i < 300; i++) { - if (mapLauncher.getNumWaitingTasksToLaunch() == 0) { - break; - } - UtilsForTests.waitFor(100); - } - assertEquals("Launcher didnt pick up the task " + attemptID + "to launch", - 0, mapLauncher.getNumWaitingTasksToLaunch()); - - // Now, that launcher has picked up the task, it waits until all five slots - // are available. i.e. it waits for-ever - // lets kill the task so that map launcher comes out - tt.processKillTaskAction(new KillTaskAction(attemptID)); - assertEquals(TaskStatus.State.KILLED, killTip.getRunState()); - - // launch another attempt which requires only one slot - TaskAttemptID runningAttemptID = new TaskAttemptID(jtId, 1, TaskType.MAP, - 0, expectedLaunchAttemptId); - mapLauncher.addToTaskQueue(new LaunchTaskAction(new MapTask(null, - runningAttemptID, 0, null, 1))); - TaskInProgress runningTip = tt.runningTasks.get(runningAttemptID); - assertNotNull(runningTip); - - // wait for a while for the task to be launched - // this loop waits at most for 30 seconds - for (int i = 0; i < 300; i++) { - if (runningTip.getRunState().equals(TaskStatus.State.RUNNING)) { - break; - } - UtilsForTests.waitFor(100); - } - - // verify that the task went to running - assertEquals(TaskStatus.State.RUNNING, runningTip.getRunState()); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java deleted file mode 100644 index 21378f16ddb..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.TreeMap; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; -import org.apache.hadoop.mapred.TaskTracker.TaskLauncher; -import org.apache.hadoop.mapred.TaskTracker.RunningJob; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.Test; - -/** - * Tests {@link TaskLauncherThreaded} - */ -public class TestTaskLauncherThreaded { - private static int jobLocalizedCount = 0; - private static int jobLaunchCount = 0; - private static boolean quitWaiting = false; - private static boolean firstJobStarted = false; - private static boolean firstJobFinished = false; - - private static class MyTaskTracker extends TaskTracker { - - // stub out functions called from startNewTask - @Override - RunningJob localizeJob(TaskInProgress tip) - throws IOException, InterruptedException { - if (firstJobStarted == false) { - firstJobStarted = true; - while (quitWaiting == false) { - Thread.sleep(100); - } - firstJobFinished = true; - } - // mock out a RunningJob - RunningJob rjob = mock(RunningJob.class); - when(rjob.getJobConf()).thenReturn(new JobConf()); - jobLocalizedCount++; - - return rjob; - } - - @Override - protected void launchTaskForJob(TaskInProgress tip, JobConf jobConf, - UserGroupInformation ugi) throws IOException { - jobLaunchCount++; - } - } - - /** - * Tests the case "task localizing doesn't block other tasks". - * - * Launches one task that simulates a task doing large localization, - * then starts a second task and verifies that second task is not - * blocked waiting behind the first task. - * - * @throws IOException - */ - @Test - public void testLocalizationNotBlockingOtherTasks() throws IOException { - // setup a TaskTracker - JobConf ttConf = new JobConf(); - ttConf.setInt("mapred.tasktracker.map.tasks.maximum", 4); - TaskTracker tt = new MyTaskTracker(); - - tt.runningJobs = new TreeMap(); - tt.runningTasks = new LinkedHashMap(); - tt.setIndexCache(new IndexCache(ttConf)); - tt.setTaskMemoryManagerEnabledFlag(); - - // start map-task launcher with four slots - TaskLauncher mapLauncher = tt.new TaskLauncher(TaskType.MAP, 4); - mapLauncher.start(); - - // launch a task which simulates large localization - String jtId = "test"; - TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, true, 0, 0); - Task task = new MapTask(null, attemptID, 0, null, 2); - mapLauncher.addToTaskQueue(new LaunchTaskAction(task)); - // verify that task is added to runningTasks - TaskInProgress runningTip = tt.runningTasks.get(attemptID); - assertNotNull(runningTip); - - // wait for a while for the first task to start initializing - // this loop waits at most for 30 seconds - for (int i = 0; i < 300; i++) { - if (firstJobStarted == true) { - break; - } - UtilsForTests.waitFor(100); - } - - // Now start a second task and make sure it doesn't wait while first one initializes - String secondjtId = "test2"; - TaskAttemptID secondAttemptID = new TaskAttemptID(secondjtId, 1, true, 0, 0); - Task secondTask = new MapTask(null, secondAttemptID, 0, null, 2); - mapLauncher.addToTaskQueue(new LaunchTaskAction(secondTask)); - // verify that task is added to runningTasks - TaskInProgress secondRunningTip = tt.runningTasks.get(secondAttemptID); - assertNotNull(secondRunningTip); - - // wait for a while for the second task to be launched - // this loop waits at most for 30 seconds - for (int i = 0; i < 300; i++) { - if (jobLaunchCount > 0) { - break; - } - UtilsForTests.waitFor(100); - } - - assertEquals("Second task didn't run or both ran", 1, jobLocalizedCount); - assertEquals("second task didn't try to launch", 1, jobLaunchCount); - assertFalse("Second task didn't finish first task initializing", firstJobFinished); - - // tell first task to stop waiting - quitWaiting = true; - - // wait for a while for the first task finishes initializing - // this loop waits at most for 30 seconds - for (int i = 0; i < 300; i++) { - if (firstJobFinished == true) { - break; - } - UtilsForTests.waitFor(100); - } - assertTrue("First task didn't finish initializing", firstJobFinished); - - // wait for a while for the first task finishes - // this loop waits at most for 30 seconds - for (int i = 0; i < 300; i++) { - if (jobLaunchCount > 1) { - break; - } - UtilsForTests.waitFor(100); - } - assertEquals("Both tasks didn't run", 2, jobLocalizedCount); - assertEquals("First task didn't try to launch", 2, jobLaunchCount); - - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLimits.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLimits.java deleted file mode 100644 index c8ecac75374..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLimits.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import junit.framework.TestCase; -import java.io.IOException; - -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; - -/** - * A JUnit test to test configured task limits. - */ -public class TestTaskLimits extends TestCase { - - static void runTest(int maxTasks, int numMaps, int numReds, - boolean shouldFail) throws Exception { - JobConf conf = new JobConf(); - conf.setInt(JTConfig.JT_TASKS_PER_JOB, maxTasks); - conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); - MiniMRCluster mr = new MiniMRCluster(0, "file:///", 1, null, null, conf); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobConf jc = mr.createJobConf(); - jc.setNumMapTasks(numMaps); - jc.setNumReduceTasks(numReds); - JobInProgress jip = new JobInProgress(new JobID(), jc, jt); - boolean failed = false; - try { - jip.checkTaskLimits(); - } catch (IOException e) { - failed = true; - } - assertEquals(shouldFail, failed); - mr.shutdown(); - } - - public void testBeyondLimits() throws Exception { - // Max tasks is 4, Requested is 8, shouldFail = true - runTest(4, 8, 0, true); - } - - public void testTaskWithinLimits() throws Exception { - // Max tasks is 4, requested is 4, shouldFail = false - runTest(4, 4, 0, false); - } - - - public void testTaskWithoutLimits() throws Exception { - // No task limit, requested is 16, shouldFail = false - runTest(-1, 8, 8, false); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java deleted file mode 100644 index a61276cff22..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; - -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.junit.Before; -import org.junit.Test; -import org.mortbay.jetty.testing.HttpTester; -import org.mortbay.jetty.testing.ServletTester; - -public class TestTaskLogServlet { - private static final Log LOG = LogFactory.getLog(TestTaskLogServlet.class); - private ServletTester tester; - - @Before - public void setup() throws Exception { - tester = new ServletTester(); - tester.setContextPath("/"); - tester.addServlet(TaskLogServlet.class, "/tasklog"); - tester.start(); - } - - @Test - public void testMissingParameters() throws Exception { - HttpTester request = new HttpTester(); - request.setMethod("GET"); - request.setURI("/tasklog"); - request.setVersion("HTTP/1.0"); - - HttpTester response = new HttpTester(); - response.parse(tester.getResponses(request.generate())); - - assertEquals(400,response.getStatus()); - } - - private void setupValidLogs(String attemptIdStr) throws IOException { - TaskAttemptID attemptId = TaskAttemptID.forName(attemptIdStr); - File logDir = TaskLog.getAttemptDir(attemptId, false); - FileUtil.fullyDelete(logDir); - logDir.mkdirs(); - assertTrue(logDir.exists()); - - // Now make the logs with some HTML in the output - TaskLog.syncLogs(logDir.getAbsolutePath(), attemptId, false); - makeLog(new File(logDir, "stderr"), "this is stderr"); - makeLog(new File(logDir, "stdout"), "this is stdout"); - makeLog(new File(logDir, "syslog"), "this is syslog"); - TaskLog.syncLogs(logDir.getAbsolutePath(), attemptId, false); - } - - @Test - public void testHtmlLogs() throws Exception { - String attemptIdStr = "attempt_123_0001_m_000001_0"; - setupValidLogs(attemptIdStr); - - HttpTester request = new HttpTester(); - request.setMethod("GET"); - request.setURI("/tasklog?attemptid=" + attemptIdStr); - request.setVersion("HTTP/1.0"); - - // Make sure all the contents show up and properly escaped - HttpTester response = doRequest(request); - assertEquals(HttpServletResponse.SC_OK, response.getStatus()); - assertEquals("text/html; charset=utf-8", response.getHeader("content-type")); - assertTrue(response.getContent().contains("<b>this is stderr")); - assertTrue(response.getContent().contains("<b>this is stdout")); - assertTrue(response.getContent().contains("<b>this is syslog")); - - // Only read a small chunk of each file <***b>thi***s - // (should still be escaped) - request.setURI("/tasklog?attemptid=" + attemptIdStr - + "&start=1&end=6"); - response = doRequest(request); - assertEquals(HttpServletResponse.SC_OK, response.getStatus()); - assertEquals("text/html; charset=utf-8", response.getHeader("content-type")); - assertFalse(response.getContent().contains("<b")); - assertFalse(response.getContent().contains("this is")); - assertTrue(response.getContent().contains("b>thi")); - } - - @Test - public void testPlaintextLogs() throws Exception { - String attemptIdStr = "attempt_123_0001_m_000001_0"; - setupValidLogs(attemptIdStr); - - HttpTester request = new HttpTester(); - request.setMethod("GET"); - request.setURI("/tasklog?plaintext=true&attemptid=" + attemptIdStr); - request.setVersion("HTTP/1.0"); - - // Make sure all the contents show up and properly escaped - HttpTester response = doRequest(request); - // Bad request because we require a 'filter' - assertEquals(HttpServletResponse.SC_BAD_REQUEST, response.getStatus()); - - // Try again with filter - request.setURI("/tasklog?plaintext=true&filter=stdout&attemptid=" + attemptIdStr); - response = doRequest(request); - - // Response should be text/plain, not be escaped - assertEquals("text/plain; charset=utf-8", response.getHeader("content-type")); - assertEquals("this is stdout", response.getContent()); - - // Test range request - request.setURI("/tasklog?plaintext=true&filter=stdout" + - "&attemptid=" + attemptIdStr + - "&start=1&end=6"); - response = doRequest(request); - - // Response should be text/plain, not be escaped - assertEquals("text/plain; charset=utf-8", response.getHeader("content-type")); - assertEquals("b>thi", response.getContent()); - } - - private HttpTester doRequest(HttpTester request) throws Exception { - String reqStr = request.generate(); - LOG.info("Testing request: " + reqStr); - String respStr = tester.getResponses(reqStr); - LOG.info("Response: " + respStr); - HttpTester response = new HttpTester(); - response.parse(respStr); - return response; - } - - private void makeLog(File f, String contents) throws IOException { - LOG.info("Creating log at " + f); - FileWriter fw = new FileWriter(f); - try { - fw.write(contents); - } finally { - fw.close(); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskOutputSize.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskOutputSize.java deleted file mode 100644 index 65ae794cee0..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskOutputSize.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; - -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskCompletionEvent; -import org.junit.After; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestTaskOutputSize { - private static Path rootDir = new Path(System.getProperty("test.build.data", - "/tmp"), "test"); - - @After - public void tearDown() { - FileUtil.fullyDelete(new File(rootDir.toString())); - } - - @Test - public void testTaskOutputSize() throws Exception { - MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1); - Path inDir = new Path(rootDir, "input"); - Path outDir = new Path(rootDir, "output"); - Job job = MapReduceTestUtil.createJob(mr.createJobConf(), inDir, outDir, 1, 1); - job.waitForCompletion(true); - assertTrue("Job failed", job.isSuccessful()); - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - for (TaskCompletionEvent tce : job.getTaskCompletionEvents(0, 100)) { - TaskStatus ts = jt.getTaskStatus(TaskAttemptID.downgrade(tce - .getTaskAttemptId())); - if (tce.isMapTask()) { - assertTrue( - "map output size is not found for " + tce.getTaskAttemptId(), ts - .getOutputSize() > 0); - } else { - assertEquals("task output size not expected for " - + tce.getTaskAttemptId(), -1, ts.getOutputSize()); - } - } - - // test output sizes for job with no reduces - job = MapReduceTestUtil.createJob(mr.createJobConf(), inDir, outDir, 1, 0); - job.waitForCompletion(true); - assertTrue("Job failed", job.isSuccessful()); - for (TaskCompletionEvent tce : job.getTaskCompletionEvents(0, 100)) { - TaskStatus ts = jt.getTaskStatus(TaskAttemptID.downgrade(tce - .getTaskAttemptId())); - assertEquals("task output size not expected for " - + tce.getTaskAttemptId(), -1, ts.getOutputSize()); - } - - // test output sizes for failed job - job = MapReduceTestUtil.createFailJob(mr.createJobConf(), outDir, inDir); - job.waitForCompletion(true); - assertFalse("Job not failed", job.isSuccessful()); - for (TaskCompletionEvent tce : job.getTaskCompletionEvents(0, 100)) { - TaskStatus ts = jt.getTaskStatus(TaskAttemptID.downgrade(tce - .getTaskAttemptId())); - assertEquals("task output size not expected for " - + tce.getTaskAttemptId(), -1, ts.getOutputSize()); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java deleted file mode 100644 index 36483e44b41..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.Map.Entry; - -import javax.security.auth.login.LoginException; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo; -import org.apache.hadoop.mapred.JobTracker.ReasonForBlackListing; -import org.apache.hadoop.mapred.TaskTrackerStatus.TaskTrackerHealthStatus; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; - -public class TestTaskTrackerBlacklisting extends TestCase { - - static String trackers[] = new String[] { "tracker_tracker1:1000", - "tracker_tracker2:1000", "tracker_tracker3:1000" }; - - static String hosts[] = new String[] { "tracker1", "tracker2", "tracker3" }; - - private static FakeJobTracker jobTracker; - - private static FakeJobTrackerClock clock; - - private static short responseId; - - private static final Set nodeUnHealthyReasonSet = - EnumSet.of(ReasonForBlackListing.NODE_UNHEALTHY); - - private static final Set exceedsFailuresReasonSet = - EnumSet.of(ReasonForBlackListing.EXCEEDING_FAILURES); - - private static final Set - unhealthyAndExceedsFailure = EnumSet.of( - ReasonForBlackListing.NODE_UNHEALTHY, - ReasonForBlackListing.EXCEEDING_FAILURES); - - // Add extra millisecond where timer granularity is too coarse - private static final long aDay = 24 * 60 * 60 * 1000 + 1; - - private static class FakeJobTrackerClock extends Clock { - boolean jumpADay = false; - - @Override - long getTime() { - if (!jumpADay) { - return super.getTime(); - } else { - long now = super.getTime(); - return now + aDay; - } - } - } - - static class FakeJobTracker extends - org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker { - - FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException, - InterruptedException, LoginException { - super(conf, clock, tts); - } - - @Override - synchronized void finalizeJob(JobInProgress job) { - List blackListedTrackers = job.getBlackListedTrackers(); - for (String tracker : blackListedTrackers) { - incrementFaults(tracker); - } - } - } - - static class FakeJobInProgress extends - org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress { - HashMap trackerToFailureMap; - - FakeJobInProgress(JobConf jobConf, JobTracker tracker) throws IOException { - super(jobConf, tracker); - // initObjects(tracker, numMaps, numReduces); - trackerToFailureMap = new HashMap(); - } - - public void failTask(TaskAttemptID taskId) { - super.failTask(taskId); - TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId); - addFailuresToTrackers(tip.machineWhereTaskRan(taskId)); - } - - public void addFailuresToTrackers(String trackerName) { - Integer numOfFailures = trackerToFailureMap.get(trackerName); - if (numOfFailures == null) { - numOfFailures = 0; - } - trackerToFailureMap.put(trackerName, numOfFailures + 1); - } - - public List getBlackListedTrackers() { - ArrayList blackListedTrackers = new ArrayList(); - for (Entry entry : trackerToFailureMap.entrySet()) { - Integer failures = entry.getValue(); - String tracker = entry.getKey(); - if (failures.intValue() >= this.getJobConf() - .getMaxTaskFailuresPerTracker()) { - blackListedTrackers.add(JobInProgress - .convertTrackerNameToHostName(tracker)); - } - } - return blackListedTrackers; - } - } - - public static Test suite() { - TestSetup setup = - new TestSetup(new TestSuite(TestTaskTrackerBlacklisting.class)) { - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1); - - jobTracker = - new FakeJobTracker(conf, (clock = new FakeJobTrackerClock()), - trackers); - sendHeartBeat(null, true); - } - protected void tearDown() throws Exception { - //delete the build/test/logs/ dir - } - }; - return setup; - } - - private static void sendHeartBeat(TaskTrackerHealthStatus status, - boolean initialContact) - throws IOException { - for (String tracker : trackers) { - TaskTrackerStatus tts = new TaskTrackerStatus(tracker, JobInProgress - .convertTrackerNameToHostName(tracker)); - if (status != null) { - TaskTrackerHealthStatus healthStatus = tts.getHealthStatus(); - healthStatus.setNodeHealthy(status.isNodeHealthy()); - healthStatus.setHealthReport(status.getHealthReport()); - healthStatus.setLastReported(status.getLastReported()); - } - jobTracker.heartbeat(tts, false, initialContact, - false, responseId); - } - responseId++; - } - - public void testTrackerBlacklistingForJobFailures() throws Exception { - runBlackListingJob(jobTracker, trackers); - assertEquals("Tracker 1 not blacklisted", jobTracker - .getBlacklistedTrackerCount(), 1); - checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet); - clock.jumpADay = true; - sendHeartBeat(null, false); - assertEquals("Tracker 1 still blacklisted after a day", 0, jobTracker - .getBlacklistedTrackerCount()); - //Cleanup the blacklisted trackers. - //Tracker is black listed due to failure count, so clock has to be - //forwarded by a day. - clock.jumpADay = false; - } - - public void testNodeHealthBlackListing() throws Exception { - TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR"); - //Blacklist tracker due to node health failures. - sendHeartBeat(status, false); - for (String host : hosts) { - checkReasonForBlackListing(host, nodeUnHealthyReasonSet); - } - status.setNodeHealthy(true); - status.setLastReported(System.currentTimeMillis()); - status.setHealthReport(""); - //white list tracker so the further test cases can be - //using trackers. - sendHeartBeat(status, false); - assertEquals("Trackers still blacklisted after healthy report", 0, - jobTracker.getBlacklistedTrackerCount()); - } - - - /** - * Test case to check if the task tracker node health failure statistics - * is populated correctly. - * - * We check the since start property and assume that other properties would - * be populated in a correct manner. - */ - public void testTaskTrackerNodeHealthFailureStatistics() throws Exception { - //populate previous failure count, as the job tracker is bought up only - //once in setup of test cases to run all node health blacklist stuff. - int failureCount = getFailureCountSinceStart(jobTracker, trackers[0]); - sendHeartBeat(null, false); - for(String tracker: trackers) { - assertEquals("Failure count updated wrongly for tracker : " + tracker, - failureCount, getFailureCountSinceStart(jobTracker, tracker)); - } - - TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR"); - sendHeartBeat(status, false); - //When the node fails due to health check, the statistics is - //incremented. - failureCount++; - for(String tracker: trackers) { - assertEquals("Failure count updated wrongly for tracker : " + tracker, - failureCount, getFailureCountSinceStart(jobTracker, tracker)); - } - //even if the node reports unhealthy in next status update we dont - //increment it. We increment the statistics if the node goes back to - //healthy and then becomes unhealthy. - sendHeartBeat(status, false); - for(String tracker: trackers) { - assertEquals("Failure count updated wrongly for tracker : " + tracker, - failureCount, getFailureCountSinceStart(jobTracker, tracker)); - } - //make nodes all healthy, but the failure statistics should be - //carried forward. - sendHeartBeat(null, false); - for(String tracker: trackers) { - assertEquals("Failure count updated wrongly for tracker : " + tracker, - failureCount, getFailureCountSinceStart(jobTracker, tracker)); - } - } - - private int getFailureCountSinceStart(JobTracker jt, String tracker) { - JobTrackerStatistics jtStats = jt.getStatistics(); - StatisticsCollector collector = jtStats.collector; - collector.update(); - return jtStats.getTaskTrackerStat(tracker).healthCheckFailedStat - .getValues().get(StatisticsCollector.SINCE_START).getValue(); - } - - public void testBlackListingWithFailuresAndHealthStatus() throws Exception { - runBlackListingJob(jobTracker, trackers); - assertEquals("Tracker 1 not blacklisted", 1, - jobTracker.getBlacklistedTrackerCount()); - checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet); - TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR"); - - sendHeartBeat(status, false); - - assertEquals("All trackers not blacklisted", 3, - jobTracker.getBlacklistedTrackerCount()); - checkReasonForBlackListing(hosts[0], unhealthyAndExceedsFailure); - checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet); - - clock.jumpADay = true; - sendHeartBeat(status, false); - - assertEquals("All trackers not blacklisted", 3, - jobTracker.getBlacklistedTrackerCount()); - - for (String host : hosts) { - checkReasonForBlackListing(host, nodeUnHealthyReasonSet); - } - //clear blacklisted trackers due to node health reasons. - sendHeartBeat(null, false); - - assertEquals("All trackers not white listed", 0, - jobTracker.getBlacklistedTrackerCount()); - //Clear the blacklisted trackers due to failures. - clock.jumpADay = false; - } - - public void testBlacklistingReasonString() throws Exception { - String error = "ERROR"; - String error1 = "ERROR1"; - TaskTrackerHealthStatus status = getUnhealthyNodeStatus(error); - sendHeartBeat(status, false); - - assertEquals("All trackers not blacklisted", 3, - jobTracker.getBlacklistedTrackerCount()); - - checkReasonForBlackListing(hosts[0], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet); - for (int i = 0; i < hosts.length; i++) { - //Replace new line as we are adding new line - //in getFaultReport - assertEquals("Blacklisting reason string not correct for host " + i, - error, - jobTracker.getFaultReport(hosts[i]).replace("\n", "")); - } - status.setNodeHealthy(false); - status.setLastReported(System.currentTimeMillis()); - status.setHealthReport(error1); - sendHeartBeat(status, false); - checkReasonForBlackListing(hosts[0], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet); - for (int i = 0; i < hosts.length; i++) { - //Replace new line as we are adding new line - //in getFaultReport - assertEquals("Blacklisting reason string not correct for host " + i, - error1, - jobTracker.getFaultReport(hosts[i]).replace("\n", "")); - } - //clear the blacklisted trackers with node health reasons. - sendHeartBeat(null, false); - } - - private TaskTrackerHealthStatus getUnhealthyNodeStatus(String error) { - TaskTrackerHealthStatus status = new TaskTrackerHealthStatus(); - status.setNodeHealthy(false); - status.setLastReported(System.currentTimeMillis()); - status.setHealthReport(error); - return status; - } - - public void testBlackListingWithTrackerReservation() throws Exception { - JobConf conf = new JobConf(); - conf.setNumMapTasks(1); - conf.setNumReduceTasks(1); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]); - TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); - tt1.reserveSlots(TaskType.MAP, job, 1); - tt1.reserveSlots(TaskType.REDUCE, job, 1); - tt2.reserveSlots(TaskType.MAP, job, 1); - tt2.reserveSlots(TaskType.REDUCE, job, 1); - assertEquals("Tracker 1 not reserved for the job 1", 2, job - .getNumReservedTaskTrackersForMaps()); - assertEquals("Tracker 1 not reserved for the job 1", 2, job - .getNumReservedTaskTrackersForReduces()); - runBlackListingJob(jobTracker, trackers); - assertEquals("Tracker 1 not unreserved for the job 1", 1, job - .getNumReservedTaskTrackersForMaps()); - assertEquals("Tracker 1 not unreserved for the job 1", 1, job - .getNumReservedTaskTrackersForReduces()); - assertEquals("Tracker 1 not blacklisted", 1, jobTracker - .getBlacklistedTrackerCount()); - checkReasonForBlackListing(hosts[0], exceedsFailuresReasonSet); - - TaskTrackerHealthStatus status = getUnhealthyNodeStatus("ERROR"); - sendHeartBeat(status, false); - assertEquals("All trackers not blacklisted", 3, - jobTracker.getBlacklistedTrackerCount()); - - checkReasonForBlackListing(hosts[0], unhealthyAndExceedsFailure); - checkReasonForBlackListing(hosts[1], nodeUnHealthyReasonSet); - checkReasonForBlackListing(hosts[2], nodeUnHealthyReasonSet); - - assertEquals("Tracker 1 not unreserved for the job 1", 0, job - .getNumReservedTaskTrackersForMaps()); - assertEquals("Tracker 1 not unreserved for the job 1", 0, job - .getNumReservedTaskTrackersForReduces()); - //white list all trackers for health reasons and failure counts - clock.jumpADay = true; - sendHeartBeat(null, false); - } - - /** - * Test case to test if the cluster status is populated with the right - * blacklist information, which would be used by the {@link JobClient} to - * display information on the Command Line interface. - * - */ - public void testClusterStatusBlacklistedReason() throws Exception { - String error = "ERROR"; - String errorWithNewLines = "ERROR\nERROR"; - String expectedErrorReport = "ERROR:ERROR"; - // Create an unhealthy tracker health status. - Collection blackListedTrackerInfo = jobTracker - .getBlackListedTrackers(); - - assertTrue("The blacklisted tracker nodes is not empty.", - blackListedTrackerInfo.isEmpty()); - - TaskTrackerHealthStatus status = getUnhealthyNodeStatus(errorWithNewLines); - // make all tracker unhealthy - sendHeartBeat(status, false); - assertEquals("All trackers not blacklisted", 3, jobTracker - .getBlacklistedTrackerCount()); - // Verify the new method .getBlackListedTracker() which is - // used by the ClusterStatus to set the list of blacklisted - // tracker. - blackListedTrackerInfo = jobTracker.getBlackListedTrackers(); - - // Check if all the black listed tracker information is obtained - // in new method. - assertEquals("Blacklist tracker info does not contain all trackers", 3, - blackListedTrackerInfo.size()); - // verify all the trackers are blacklisted for health reasons. - // Also check the health report. - for (BlackListInfo bi : blackListedTrackerInfo) { - assertEquals("Tracker not blacklisted for health reason", - ReasonForBlackListing.NODE_UNHEALTHY.toString().trim(), bi - .getReasonForBlackListing().trim()); - assertTrue("Tracker blacklist report does not match", - bi.toString().endsWith(expectedErrorReport)); - } - // reset the tracker health status back to normal. - sendHeartBeat(null, false); - runBlackListingJob(jobTracker, trackers); - sendHeartBeat(status, false); - blackListedTrackerInfo = jobTracker.getBlackListedTrackers(); - for (BlackListInfo bi : blackListedTrackerInfo) { - if (bi.getTrackerName().equals(trackers[0])) { - assertTrue( - "Reason for blacklisting of tracker 1 does not contain Unhealthy reasons", - bi.getReasonForBlackListing().contains( - ReasonForBlackListing.NODE_UNHEALTHY.toString().trim())); - assertTrue( - "Reason for blacklisting of tracker 1 does not contain Unhealthy reasons", - bi.getReasonForBlackListing().contains( - ReasonForBlackListing.EXCEEDING_FAILURES.toString().trim())); - assertTrue("Blacklist failure does not contain failure report string", - bi.getBlackListReport().contains("failures on the tracker")); - } else { - assertEquals("Tracker not blacklisted for health reason", - ReasonForBlackListing.NODE_UNHEALTHY.toString().trim(), bi - .getReasonForBlackListing().trim()); - } - assertTrue("Tracker blacklist report does not match", bi - .getBlackListReport().trim().contains(error)); - } - clock.jumpADay = true; - sendHeartBeat(null, false); - } - - /** - * Runs a job which blacklists the first of the tracker - * which is passed to the method. - * - * @param jobTracker JobTracker instance - * @param trackers array of trackers, the method would blacklist - * first element of the array - * @return A job in progress object. - * @throws Exception - */ - static FakeJobInProgress runBlackListingJob(JobTracker jobTracker, - String[] trackers) throws Exception { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[3]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setNumMapTasks(0); - conf.setNumReduceTasks(5); - conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70"); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - conf.setMaxTaskFailuresPerTracker(1); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.setClusterSize(trackers.length); - job.initTasks(); - - taskAttemptID[0] = job.findReduceTask(trackers[0]); - taskAttemptID[1] = job.findReduceTask(trackers[1]); - taskAttemptID[2] = job.findReduceTask(trackers[2]); - job.finishTask(taskAttemptID[1]); - job.finishTask(taskAttemptID[2]); - job.failTask(taskAttemptID[0]); - - taskAttemptID[0] = job.findReduceTask(trackers[0]); - job.failTask(taskAttemptID[0]); - - taskAttemptID[0] = job.findReduceTask(trackers[1]); - job.finishTask(taskAttemptID[0]); - taskAttemptID[0] = job.findReduceTask(trackers[1]); - taskAttemptID[1] = job.findReduceTask(trackers[2]); - job.finishTask(taskAttemptID[0]); - job.finishTask(taskAttemptID[1]); - - jobTracker.finalizeJob(job); - return job; - } - - private void checkReasonForBlackListing(String host, - Set reasonsForBlackListing) { - Set rfbs = jobTracker.getReasonForBlackList(host); - assertEquals("Reasons for blacklisting of " + host + " does not match", - reasonsForBlackListing, rfbs); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java deleted file mode 100644 index 4aea9055314..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Test; -import static org.junit.Assert.*; - -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; - -public class TestTaskTrackerInstrumentation { - private static final Log LOG = LogFactory.getLog( - TestTaskTrackerInstrumentation.class); - - @Test - public void testCreateInstrumentationWithSingleClass() { - // Check that if only a single instrumentation class is given, - // that class is used directly - JobConf conf = new JobConf(); - conf.set(TTConfig.TT_INSTRUMENTATION, - DummyTaskTrackerInstrumentation.class.getName()); - TaskTracker tracker = new TaskTracker(); - TaskTrackerInstrumentation inst = - TaskTracker.createInstrumentation(tracker, conf); - assertEquals(DummyTaskTrackerInstrumentation.class.getName(), - inst.getClass().getName()); - } - - @Test - public void testCreateInstrumentationWithMultipleClasses() { - // Set up configuration to create two dummy instrumentation objects - JobConf conf = new JobConf(); - String dummyClass = DummyTaskTrackerInstrumentation.class.getName(); - String classList = dummyClass + "," + dummyClass; - conf.set(TTConfig.TT_INSTRUMENTATION, classList); - TaskTracker tracker = new TaskTracker(); - - // Check that a composite instrumentation object is created - TaskTrackerInstrumentation inst = - TaskTracker.createInstrumentation(tracker, conf); - assertEquals(CompositeTaskTrackerInstrumentation.class.getName(), - inst.getClass().getName()); - - // Check that each member of the composite is a dummy instrumentation - CompositeTaskTrackerInstrumentation comp = - (CompositeTaskTrackerInstrumentation) inst; - List insts = comp.getInstrumentations(); - assertEquals(2, insts.size()); - assertEquals(DummyTaskTrackerInstrumentation.class.getName(), - insts.get(0).getClass().getName()); - assertEquals(DummyTaskTrackerInstrumentation.class.getName(), - insts.get(1).getClass().getName()); - } - - @Test - public void testCreateInstrumentationWithDefaultClass() { - // Check that if no instrumentation class is given, the default - // class (TaskTrackerMetricsInst) is used. - JobConf conf = new JobConf(); - TaskTracker tracker = new TaskTracker(); - tracker.setConf(conf); // Needed to avoid NullPointerExcepton in - // TaskTrackerMetricsInst constructor - TaskTrackerInstrumentation inst = - TaskTracker.createInstrumentation(tracker, conf); - assertEquals(TaskTrackerMetricsInst.class.getName(), - inst.getClass().getName()); - } - - @Test - public void testCreateInstrumentationWithEmptyParam() { - // Check that if an empty string is given, the default instrumentation - // class (TaskTrackerMetricsInst) is used. An error message should also - // be written to the log, but we do not capture that. - JobConf conf = new JobConf(); - conf.set(TTConfig.TT_INSTRUMENTATION, ""); - TaskTracker tracker = new TaskTracker(); - tracker.setConf(conf); // Needed to avoid NullPointerExcepton in - // TaskTrackerMetricsInst constructor - TaskTrackerInstrumentation inst = - TaskTracker.createInstrumentation(tracker, conf); - assertEquals(TaskTrackerMetricsInst.class.getName(), - inst.getClass().getName()); - } - - @Test - public void testCreateInstrumentationWithInvalidParam() { - // Check that if an invalid class list is given, the default - // instrumentation class (TaskTrackerMetricsInst) is used. An error - // should also be written to the log, but we do not capture that. - JobConf conf = new JobConf(); - conf.set(TTConfig.TT_INSTRUMENTATION, "XYZ,ZZY"); - TaskTracker tracker = new TaskTracker(); - tracker.setConf(conf); // Needed to avoid NullPointerExcepton in - // TaskTrackerMetricsInst constructor - TaskTrackerInstrumentation inst = - TaskTracker.createInstrumentation(tracker, conf); - assertEquals(TaskTrackerMetricsInst.class.getName(), - inst.getClass().getName()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java deleted file mode 100644 index d1a2d054f64..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java +++ /dev/null @@ -1,1075 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.LinkedHashMap; -import java.util.TreeMap; -import java.util.jar.JarOutputStream; -import java.util.zip.ZipEntry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.LocalDirAllocator; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.security.TokenCache; -import org.apache.hadoop.mapreduce.server.tasktracker.Localizer; -import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; - -import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.mapred.JvmManager.JvmEnv; -import org.apache.hadoop.mapred.TaskController.JobInitializationContext; -import org.apache.hadoop.mapred.TaskController.TaskControllerContext; -import org.apache.hadoop.mapred.TaskTracker.RunningJob; -import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; -import org.apache.hadoop.mapred.UtilsForTests.InlineCleanupQueue; - -import junit.framework.TestCase; - -/** - * Test to verify localization of a job and localization of a task on a - * TaskTracker. - * - */ -public class TestTaskTrackerLocalization extends TestCase { - - private static File TEST_ROOT_DIR = - new File(System.getProperty("test.build.data", "/tmp")); - private File ROOT_MAPRED_LOCAL_DIR; - private File HADOOP_LOG_DIR; - private static File PERMISSION_SCRIPT_DIR; - private static File PERMISSION_SCRIPT_FILE; - private static final String PERMISSION_SCRIPT_CONTENT = "ls -l -d $1 | " + - "awk '{print $1\":\"$3\":\"$4}'"; - - private int numLocalDirs = 6; - private static final Log LOG = - LogFactory.getLog(TestTaskTrackerLocalization.class); - - protected TaskTracker tracker; - protected UserGroupInformation taskTrackerUGI; - protected TaskController taskController; - protected JobConf trackerFConf; - private JobConf localizedJobConf; - protected JobID jobId; - protected TaskAttemptID taskId; - protected Task task; - protected String[] localDirs; - protected static LocalDirAllocator lDirAlloc = - new LocalDirAllocator(MRConfig.LOCAL_DIR); - protected Path attemptWorkDir; - protected File[] attemptLogFiles; - protected JobConf localizedTaskConf; - private TaskInProgress tip; - private JobConf jobConf; - private File jobConfFile; - - /** - * Dummy method in this base class. Only derived classes will define this - * method for checking if a test can be run. - */ - protected boolean canRun() { - return true; - } - - @Override - protected void setUp() - throws Exception { - if (!canRun()) { - return; - } - TEST_ROOT_DIR = - new File(System.getProperty("test.build.data", "/tmp"), getClass() - .getSimpleName()); - if (!TEST_ROOT_DIR.exists()) { - TEST_ROOT_DIR.mkdirs(); - } - - ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local"); - ROOT_MAPRED_LOCAL_DIR.mkdirs(); - - HADOOP_LOG_DIR = new File(TEST_ROOT_DIR, "logs"); - HADOOP_LOG_DIR.mkdir(); - System.setProperty("hadoop.log.dir", HADOOP_LOG_DIR.getAbsolutePath()); - - trackerFConf = new JobConf(); - - trackerFConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); - localDirs = new String[numLocalDirs]; - for (int i = 0; i < numLocalDirs; i++) { - localDirs[i] = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i).getPath(); - } - trackerFConf.setStrings(MRConfig.LOCAL_DIR, localDirs); - trackerFConf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - - // Create the job configuration file. Same as trackerConf in this test. - jobConf = new JobConf(trackerFConf); - // Set job view ACLs in conf sothat validation of contents of jobACLsFile - // can be done against this value. Have both users and groups - String jobViewACLs = "user1,user2, group1,group2"; - jobConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, jobViewACLs); - - jobConf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 0); - jobConf.setUser(getJobOwner().getShortUserName()); - - String queue = "default"; - // set job queue name in job conf - jobConf.setQueueName(queue); - // Set queue admins acl in job conf similar to what JobClient does so that - // it goes into job conf also. - jobConf.set(toFullPropertyName(queue, - QueueACL.ADMINISTER_JOBS.getAclName()), - "qAdmin1,qAdmin2 qAdminsGroup1,qAdminsGroup2"); - - Job job = Job.getInstance(jobConf); - String jtIdentifier = "200907202331"; - jobId = new JobID(jtIdentifier, 1); - - // JobClient uploads the job jar to the file system and sets it in the - // jobConf. - uploadJobJar(job); - - // JobClient uploads the jobConf to the file system. - jobConfFile = uploadJobConf(job.getConfiguration()); - - // create jobTokens file - uploadJobTokensFile(); - - taskTrackerUGI = UserGroupInformation.getCurrentUser(); - startTracker(); - - // Set up the task to be localized - taskId = - new TaskAttemptID(jtIdentifier, jobId.getId(), TaskType.MAP, 1, 0); - createTask(); - - // mimic register task - // create the tip - tip = tracker.new TaskInProgress(task, trackerFConf); - } - - private void startTracker() throws IOException { - // Set up the TaskTracker - tracker = new TaskTracker(); - tracker.setConf(trackerFConf); - tracker.setTaskLogCleanupThread(new UserLogCleaner(trackerFConf)); - initializeTracker(); - } - - private void initializeTracker() throws IOException { - tracker.setIndexCache(new IndexCache(trackerFConf)); - tracker.setTaskMemoryManagerEnabledFlag(); - - // for test case system FS is the local FS - tracker.systemFS = FileSystem.getLocal(trackerFConf); - tracker.setLocalFileSystem(tracker.systemFS); - tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath()); - - tracker.runningTasks = new LinkedHashMap(); - tracker.runningJobs = new TreeMap(); - tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf)); - tracker.getAsyncDiskService().cleanupAllVolumes(); - - // Set up TaskTracker instrumentation - tracker.setTaskTrackerInstrumentation( - TaskTracker.createInstrumentation(tracker, trackerFConf)); - - // setup task controller - taskController = createTaskController(); - taskController.setConf(trackerFConf); - taskController.setup(); - tracker.setTaskController(taskController); - tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs, - taskController)); - } - - protected TaskController createTaskController() { - return new DefaultTaskController(); - } - - private void createTask() - throws IOException { - task = new MapTask(jobConfFile.toURI().toString(), taskId, 1, null, 1); - task.setConf(jobConf); // Set conf. Set user name in particular. - task.setUser(jobConf.getUser()); - } - - protected UserGroupInformation getJobOwner() throws IOException { - return UserGroupInformation.getCurrentUser(); - } - - /** - * static block setting up the permission script which would be used by the - * checkFilePermissions - */ - static { - PERMISSION_SCRIPT_DIR = new File(TEST_ROOT_DIR, "permission_script_dir"); - PERMISSION_SCRIPT_FILE = new File(PERMISSION_SCRIPT_DIR, "getperms.sh"); - - if(PERMISSION_SCRIPT_FILE.exists()) { - PERMISSION_SCRIPT_FILE.delete(); - } - - if(PERMISSION_SCRIPT_DIR.exists()) { - PERMISSION_SCRIPT_DIR.delete(); - } - - PERMISSION_SCRIPT_DIR.mkdir(); - - try { - PrintWriter writer = new PrintWriter(PERMISSION_SCRIPT_FILE); - writer.write(PERMISSION_SCRIPT_CONTENT); - writer.close(); - } catch (FileNotFoundException fe) { - fail(); - } - PERMISSION_SCRIPT_FILE.setExecutable(true, true); - } - - /** - * @param job - * @throws IOException - * @throws FileNotFoundException - */ - private void uploadJobJar(Job job) - throws IOException, - FileNotFoundException { - File jobJarFile = new File(TEST_ROOT_DIR, "jobjar-on-dfs.jar"); - JarOutputStream jstream = - new JarOutputStream(new FileOutputStream(jobJarFile)); - ZipEntry ze = new ZipEntry("lib/lib1.jar"); - jstream.putNextEntry(ze); - jstream.closeEntry(); - ze = new ZipEntry("lib/lib2.jar"); - jstream.putNextEntry(ze); - jstream.closeEntry(); - jstream.finish(); - jstream.close(); - job.setJar(jobJarFile.toURI().toString()); - } - - /** - * @param conf - * @return - * @throws FileNotFoundException - * @throws IOException - */ - protected File uploadJobConf(Configuration conf) - throws FileNotFoundException, - IOException { - File jobConfFile = new File(TEST_ROOT_DIR, "jobconf-on-dfs.xml"); - FileOutputStream out = new FileOutputStream(jobConfFile); - conf.writeXml(out); - out.close(); - return jobConfFile; - } - - /** - * create fake JobTokens file - * @return - * @throws IOException - */ - protected void uploadJobTokensFile() throws IOException { - - File dir = new File(TEST_ROOT_DIR, jobId.toString()); - if(!dir.exists()) - assertTrue("faild to create dir="+dir.getAbsolutePath(), dir.mkdirs()); - // writing empty file, we don't need the keys for this test - new Credentials().writeTokenStorageFile(new Path("file:///" + dir, - TokenCache.JOB_TOKEN_HDFS_FILE), new Configuration()); - } - - @Override - protected void tearDown() - throws Exception { - if (!canRun()) { - return; - } - FileUtil.fullyDelete(TEST_ROOT_DIR); - } - - protected static String[] getFilePermissionAttrs(String path) - throws IOException { - String[] command = {"bash",PERMISSION_SCRIPT_FILE.getAbsolutePath(), path}; - String output=Shell.execCommand(command); - return output.split(":|\n"); - } - - - /** - * Utility method to check permission of a given path. Requires the permission - * script directory to be setup in order to call. - * - * - * @param path - * @param expectedPermissions - * @param expectedOwnerUser - * @param expectedOwnerGroup - * @throws IOException - */ - static void checkFilePermissions(String path, String expectedPermissions, - String expectedOwnerUser, String expectedOwnerGroup) - throws IOException { - String[] attrs = getFilePermissionAttrs(path); - assertTrue("File attrs length is not 3 but " + attrs.length, - attrs.length == 3); - assertTrue("Path " + path + " has the permissions " + attrs[0] - + " instead of the expected " + expectedPermissions, attrs[0] - .equals(expectedPermissions)); - assertTrue("Path " + path + " is user owned not by " + expectedOwnerUser - + " but by " + attrs[1], attrs[1].equals(expectedOwnerUser)); - assertTrue("Path " + path + " is group owned not by " + expectedOwnerGroup - + " but by " + attrs[2], attrs[2].equals(expectedOwnerGroup)); - } - - /** - * Verify the task-controller's setup functionality - * - * @throws IOException - */ - public void testTaskControllerSetup() - throws IOException { - if (!canRun()) { - return; - } - // Task-controller is already set up in the test's setup method. Now verify. - for (String localDir : localDirs) { - - // Verify the local-dir itself. - File lDir = new File(localDir); - assertTrue("localDir " + lDir + " doesn't exists!", lDir.exists()); - checkFilePermissions(lDir.getAbsolutePath(), "drwxr-xr-x", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - } - - // Verify the pemissions on the userlogs dir - File taskLog = TaskLog.getUserLogDir(); - checkFilePermissions(taskLog.getAbsolutePath(), "drwxr-xr-x", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - } - - /** - * Test the localization of a user on the TT. - * - * @throws IOException - */ - public void testUserLocalization() - throws IOException { - if (!canRun()) { - return; - } - // /////////// The main method being tested - tracker.getLocalizer().initializeUserDirs(task.getUser()); - // /////////// - - // Check the directory structure and permissions - checkUserLocalization(); - - // For the sake of testing re-entrancy of initializeUserDirs(), we remove - // the user directories now and make sure that further calls of the method - // don't create directories any more. - for (String dir : localDirs) { - File userDir = new File(dir, TaskTracker.getUserDir(task.getUser())); - if (!FileUtil.fullyDelete(userDir)) { - throw new IOException("Uanble to delete " + userDir); - } - } - - // Now call the method again. - tracker.getLocalizer().initializeUserDirs(task.getUser()); - - // Files should not be created now and so shouldn't be there anymore. - for (String dir : localDirs) { - File userDir = new File(dir, TaskTracker.getUserDir(task.getUser())); - assertFalse("Unexpectedly, user-dir " + userDir.getAbsolutePath() - + " exists!", userDir.exists()); - } - } - - protected void checkUserLocalization() - throws IOException { - for (String dir : localDirs) { - - File localDir = new File(dir); - assertTrue(MRConfig.LOCAL_DIR + localDir + " isn'task created!", - localDir.exists()); - - File taskTrackerSubDir = new File(localDir, TaskTracker.SUBDIR); - assertTrue("taskTracker sub-dir in the local-dir " + localDir - + "is not created!", taskTrackerSubDir.exists()); - - File userDir = new File(taskTrackerSubDir, task.getUser()); - assertTrue("user-dir in taskTrackerSubdir " + taskTrackerSubDir - + "is not created!", userDir.exists()); - checkFilePermissions(userDir.getAbsolutePath(), "drwx------", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - - File jobCache = new File(userDir, TaskTracker.JOBCACHE); - assertTrue("jobcache in the userDir " + userDir + " isn't created!", - jobCache.exists()); - checkFilePermissions(jobCache.getAbsolutePath(), "drwx------", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - - // Verify the distributed cache dir. - File distributedCacheDir = - new File(localDir, TaskTracker - .getPrivateDistributedCacheDir(task.getUser())); - assertTrue("distributed cache dir " + distributedCacheDir - + " doesn't exists!", distributedCacheDir.exists()); - checkFilePermissions(distributedCacheDir.getAbsolutePath(), - "drwx------", task.getUser(), taskTrackerUGI.getGroupNames()[0]); - } - } - - /** - * Test job localization on a TT. Tests localization of job.xml, job.jar and - * corresponding setting of configuration. Also test - * {@link TaskController#initializeJob(JobInitializationContext)} - * - * @throws IOException - */ - public void testJobLocalization() - throws Exception { - if (!canRun()) { - return; - } - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - - checkJobLocalization(); - } - - /** - * Test that, if the job log dir can't be created, the job will fail - * during localization rather than at the time when the task itself - * tries to write into it. - */ - public void testJobLocalizationFailsIfLogDirUnwritable() - throws Exception { - if (!canRun()) { - return; - } - - File logDir = TaskLog.getJobDir(jobId); - File logDirParent = logDir.getParentFile(); - - try { - assertTrue(logDirParent.mkdirs() || logDirParent.isDirectory()); - FileUtil.fullyDelete(logDir); - FileUtil.chmod(logDirParent.getAbsolutePath(), "000"); - - tracker.localizeJob(tip); - fail("No exception"); - } catch (IOException ioe) { - LOG.info("Got exception", ioe); - assertTrue(ioe.getMessage().contains("Could not create job user log")); - } finally { - // Put it back just to be safe - FileUtil.chmod(logDirParent.getAbsolutePath(), "755"); - } - } - - protected void checkJobLocalization() - throws IOException { - // Check the directory structure - for (String dir : localDirs) { - - File localDir = new File(dir); - File taskTrackerSubDir = new File(localDir, TaskTracker.SUBDIR); - File userDir = new File(taskTrackerSubDir, task.getUser()); - File jobCache = new File(userDir, TaskTracker.JOBCACHE); - - File jobDir = new File(jobCache, jobId.toString()); - assertTrue("job-dir in " + jobCache + " isn't created!", jobDir.exists()); - - // check the private permissions on the job directory - checkFilePermissions(jobDir.getAbsolutePath(), "drwx------", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - } - - // check the localization of job.xml - assertTrue("job.xml is not localized on this TaskTracker!!", lDirAlloc - .getLocalPathToRead(TaskTracker.getLocalJobConfFile(task.getUser(), - jobId.toString()), trackerFConf) != null); - - // check the localization of job.jar - Path jarFileLocalized = - lDirAlloc.getLocalPathToRead(TaskTracker.getJobJarFile(task.getUser(), - jobId.toString()), trackerFConf); - assertTrue("job.jar is not localized on this TaskTracker!!", - jarFileLocalized != null); - assertTrue("lib/lib1.jar is not unjarred on this TaskTracker!!", new File( - jarFileLocalized.getParent() + Path.SEPARATOR + "lib/lib1.jar") - .exists()); - assertTrue("lib/lib2.jar is not unjarred on this TaskTracker!!", new File( - jarFileLocalized.getParent() + Path.SEPARATOR + "lib/lib2.jar") - .exists()); - - // check the creation of job work directory - assertTrue("job-work dir is not created on this TaskTracker!!", lDirAlloc - .getLocalPathToRead(TaskTracker.getJobWorkDir(task.getUser(), jobId - .toString()), trackerFConf) != null); - - // Check the setting of mapreduce.job.local.dir and job.jar which will eventually be - // used by the user's task - boolean jobLocalDirFlag = false, mapredJarFlag = false; - String localizedJobLocalDir = - localizedJobConf.get(TaskTracker.JOB_LOCAL_DIR); - String localizedJobJar = localizedJobConf.getJar(); - for (String localDir : localizedJobConf.getStrings(MRConfig.LOCAL_DIR)) { - if (localizedJobLocalDir.equals(localDir + Path.SEPARATOR - + TaskTracker.getJobWorkDir(task.getUser(), jobId.toString()))) { - jobLocalDirFlag = true; - } - if (localizedJobJar.equals(localDir + Path.SEPARATOR - + TaskTracker.getJobJarFile(task.getUser(), jobId.toString()))) { - mapredJarFlag = true; - } - } - assertTrue(TaskTracker.JOB_LOCAL_DIR - + " is not set properly to the target users directory : " - + localizedJobLocalDir, jobLocalDirFlag); - assertTrue( - "mapreduce.job.jar is not set properly to the target users directory : " - + localizedJobJar, mapredJarFlag); - - // check job user-log directory permissions - File jobLogDir = TaskLog.getJobDir(jobId); - assertTrue("job log directory " + jobLogDir + " does not exist!", jobLogDir - .exists()); - checkFilePermissions(jobLogDir.toString(), "drwx------", task.getUser(), - taskTrackerUGI.getGroupNames()[0]); - - // Make sure that the job ACLs file job-acls.xml exists in job userlog dir - File jobACLsFile = new File(jobLogDir, TaskTracker.jobACLsFile); - assertTrue("JobACLsFile is missing in the job userlog dir " + jobLogDir, - jobACLsFile.exists()); - - // With default task controller, the job-acls.xml file is owned by TT and - // permissions are 700 - checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rw-------", - taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]); - - validateJobACLsFileContent(); - } - - // Validate the contents of jobACLsFile ( i.e. user name, job-view-acl, queue - // name and queue-admins-acl ). - protected void validateJobACLsFileContent() { - JobConf jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(jobId); - - assertTrue(jobACLsConf.get("user.name").equals( - localizedJobConf.getUser())); - assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB). - equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB))); - - String queue = localizedJobConf.getQueueName(); - assertTrue(queue.equalsIgnoreCase(jobACLsConf.getQueueName())); - - String qACLName = toFullPropertyName(queue, - QueueACL.ADMINISTER_JOBS.getAclName()); - assertTrue(jobACLsConf.get(qACLName).equals( - localizedJobConf.get(qACLName))); - } - - /** - * Test task localization on a TT. - * - * @throws IOException - */ - public void testTaskLocalization() - throws Exception { - if (!canRun()) { - return; - } - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - initializeTask(); - - checkTaskLocalization(); - } - - private void initializeTask() throws IOException { - tip.setJobConf(localizedJobConf); - - // ////////// The central method being tested - tip.localizeTask(task); - // ////////// - - // check the functionality of localizeTask - for (String dir : trackerFConf.getStrings(MRConfig.LOCAL_DIR)) { - File attemptDir = - new File(dir, TaskTracker.getLocalTaskDir(task.getUser(), jobId - .toString(), taskId.toString(), task.isTaskCleanupTask())); - assertTrue("attempt-dir " + attemptDir + " in localDir " + dir - + " is not created!!", attemptDir.exists()); - } - - attemptWorkDir = - lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( - task.getUser(), task.getJobID().toString(), task.getTaskID() - .toString(), task.isTaskCleanupTask()), trackerFConf); - assertTrue("atttempt work dir for " + taskId.toString() - + " is not created in any of the configured dirs!!", - attemptWorkDir != null); - - TaskRunner runner = new MapTaskRunner(tip, tracker, tip.getJobConf()); - tip.setTaskRunner(runner); - - // /////// Few more methods being tested - runner.setupChildTaskConfiguration(lDirAlloc); - TaskRunner.createChildTmpDir(new File(attemptWorkDir.toUri().getPath()), - localizedJobConf); - attemptLogFiles = runner.prepareLogFiles(task.getTaskID(), - task.isTaskCleanupTask()); - - // Make sure the task-conf file is created - Path localTaskFile = - lDirAlloc.getLocalPathToRead(TaskTracker.getTaskConfFile(task - .getUser(), task.getJobID().toString(), task.getTaskID() - .toString(), task.isTaskCleanupTask()), trackerFConf); - assertTrue("Task conf file " + localTaskFile.toString() - + " is not created!!", new File(localTaskFile.toUri().getPath()) - .exists()); - - // /////// One more method being tested. This happens in child space. - localizedTaskConf = new JobConf(localTaskFile); - TaskRunner.setupChildMapredLocalDirs(task, localizedTaskConf); - // /////// - - // Initialize task via TaskController - TaskControllerContext taskContext = - new TaskController.TaskControllerContext(); - taskContext.env = - new JvmEnv(null, null, null, null, -1, new File(localizedJobConf - .get(TaskTracker.JOB_LOCAL_DIR)), null, localizedJobConf); - taskContext.task = task; - // /////////// The method being tested - taskController.initializeTask(taskContext); - // /////////// - } - - protected void checkTaskLocalization() - throws IOException { - // Make sure that the mapreduce.cluster.local.dir is sandboxed - for (String childMapredLocalDir : localizedTaskConf - .getStrings(MRConfig.LOCAL_DIR)) { - assertTrue("Local dir " + childMapredLocalDir + " is not sandboxed !!", - childMapredLocalDir.endsWith(TaskTracker.getLocalTaskDir(task - .getUser(), jobId.toString(), taskId.toString(), - task.isTaskCleanupTask()))); - } - - // Make sure task task.getJobFile is changed and pointed correctly. - assertTrue(task.getJobFile().endsWith( - TaskTracker.getTaskConfFile(task.getUser(), jobId.toString(), taskId - .toString(), task.isTaskCleanupTask()))); - - // Make sure that the tmp directories are created - assertTrue("tmp dir is not created in workDir " - + attemptWorkDir.toUri().getPath(), new File(attemptWorkDir.toUri() - .getPath(), "tmp").exists()); - - // Make sure that the logs are setup properly - File logDir = TaskLog.getAttemptDir(taskId, task.isTaskCleanupTask()); - assertTrue("task's log dir " + logDir.toString() + " doesn't exist!", - logDir.exists()); - checkFilePermissions(logDir.getAbsolutePath(), "drwx------", task - .getUser(), taskTrackerUGI.getGroupNames()[0]); - - File expectedStdout = new File(logDir, TaskLog.LogName.STDOUT.toString()); - assertTrue("stdout log file is improper. Expected : " - + expectedStdout.toString() + " Observed : " - + attemptLogFiles[0].toString(), expectedStdout.toString().equals( - attemptLogFiles[0].toString())); - File expectedStderr = - new File(logDir, Path.SEPARATOR + TaskLog.LogName.STDERR.toString()); - assertTrue("stderr log file is improper. Expected : " - + expectedStderr.toString() + " Observed : " - + attemptLogFiles[1].toString(), expectedStderr.toString().equals( - attemptLogFiles[1].toString())); - } - - /** - * Create a file in the given dir and set permissions r_xr_xr_x sothat no one - * can delete it directly(without doing chmod). - * Creates dir/subDir and dir/subDir/file - */ - static void createFileAndSetPermissions(JobConf jobConf, Path dir) - throws IOException { - Path subDir = new Path(dir, "subDir"); - FileSystem fs = FileSystem.getLocal(jobConf); - fs.mkdirs(subDir); - Path p = new Path(subDir, "file"); - java.io.DataOutputStream out = fs.create(p); - out.writeBytes("dummy input"); - out.close(); - // no write permission for subDir and subDir/file - try { - int ret = 0; - if((ret = FileUtil.chmod(subDir.toUri().getPath(), "a=rx", true)) != 0) { - LOG.warn("chmod failed for " + subDir + ";retVal=" + ret); - } - } catch(InterruptedException e) { - LOG.warn("Interrupted while doing chmod for " + subDir); - } - } - - /** - * Validates the removal of $taskid and $tasid/work under mapred-local-dir - * in cases where those directories cannot be deleted without adding - * write permission to the newly created directories under $taskid and - * $taskid/work - * Also see createFileAndSetPermissions for details - */ - void validateRemoveTaskFiles(boolean needCleanup, boolean jvmReuse, - TaskInProgress tip) throws IOException { - // create files and set permissions 555. Verify if task controller sets - // the permissions for TT to delete the taskDir or workDir - String dir = (!needCleanup || jvmReuse) ? - TaskTracker.getTaskWorkDir(task.getUser(), task.getJobID().toString(), - taskId.toString(), task.isTaskCleanupTask()) - : TaskTracker.getLocalTaskDir(task.getUser(), task.getJobID().toString(), - taskId.toString(), task.isTaskCleanupTask()); - - Path[] paths = tracker.getLocalFiles(localizedJobConf, dir); - assertTrue("No paths found", paths.length > 0); - for (Path p : paths) { - if (tracker.getLocalFileSystem().exists(p)) { - createFileAndSetPermissions(localizedJobConf, p); - } - } - - InlineCleanupQueue cleanupQueue = new InlineCleanupQueue(); - tracker.setCleanupThread(cleanupQueue); - - tip.removeTaskFiles(needCleanup, taskId); - - if (jvmReuse) { - // work dir should still exist and cleanup queue should be empty - assertTrue("cleanup queue is not empty after removeTaskFiles() in case " - + "of jvm reuse.", cleanupQueue.isQueueEmpty()); - boolean workDirExists = false; - for (Path p : paths) { - if (tracker.getLocalFileSystem().exists(p)) { - workDirExists = true; - } - } - assertTrue("work dir does not exist in case of jvm reuse", workDirExists); - - // now try to delete the work dir and verify that there are no stale paths - JvmManager.deleteWorkDir(tracker, task); - } - - assertTrue("Some task files are not deleted!! Number of stale paths is " - + cleanupQueue.stalePaths.size(), cleanupQueue.stalePaths.size() == 0); - } - - /** - * Validates if task cleanup is done properly for a succeeded task - * @throws IOException - */ - public void testTaskFilesRemoval() - throws Exception { - if (!canRun()) { - return; - } - testTaskFilesRemoval(false, false);// no needCleanup; no jvmReuse - } - - /** - * Validates if task cleanup is done properly for a task that is not succeeded - * @throws IOException - */ - public void testFailedTaskFilesRemoval() - throws Exception { - if (!canRun()) { - return; - } - testTaskFilesRemoval(true, false);// needCleanup; no jvmReuse - - // initialize a cleanupAttempt for the task. - task.setTaskCleanupTask(); - // localize task cleanup attempt - initializeTask(); - checkTaskLocalization(); - - // verify the cleanup of cleanup attempt. - testTaskFilesRemoval(true, false);// needCleanup; no jvmReuse - } - - /** - * Validates if task cleanup is done properly for a succeeded task - * @throws IOException - */ - public void testTaskFilesRemovalWithJvmUse() - throws Exception { - if (!canRun()) { - return; - } - testTaskFilesRemoval(false, true);// no needCleanup; jvmReuse - } - - /** - * Validates if task cleanup is done properly - */ - private void testTaskFilesRemoval(boolean needCleanup, boolean jvmReuse) - throws Exception { - // Localize job and localize task. - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - if (jvmReuse) { - localizedJobConf.setNumTasksToExecutePerJvm(2); - } - initializeTask(); - - // TODO: Let the task run and create files. - - // create files and set permissions 555. Verify if task controller sets - // the permissions for TT to delete the task dir or work dir properly - validateRemoveTaskFiles(needCleanup, jvmReuse, tip); - } - - /** - * Test userlogs cleanup. - * - * @throws IOException - */ - private void verifyUserLogsRemoval() - throws IOException { - // verify user logs cleanup - File jobUserLogDir = TaskLog.getJobDir(jobId); - // Logs should be there before cleanup. - assertTrue("Userlogs dir " + jobUserLogDir + " is not present as expected!!", - jobUserLogDir.exists()); - tracker.purgeJob(new KillJobAction(jobId)); - tracker.getTaskLogCleanupThread().processCompletedJobs(); - - // Logs should be gone after cleanup. - assertFalse("Userlogs dir " + jobUserLogDir + " is not deleted as expected!!", - jobUserLogDir.exists()); - } - - /** - * Test job cleanup by doing the following - * - create files with no write permissions to TT under job-work-dir - * - create files with no write permissions to TT under task-work-dir - */ - public void testJobFilesRemoval() throws IOException, InterruptedException { - if (!canRun()) { - return; - } - - LOG.info("Running testJobCleanup()"); - // Localize job and localize task. - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - - // Set an inline cleanup queue - InlineCleanupQueue cleanupQueue = new InlineCleanupQueue(); - tracker.setCleanupThread(cleanupQueue); - - // Create a file in job's work-dir with 555 - String jobWorkDir = - TaskTracker.getJobWorkDir(task.getUser(), task.getJobID().toString()); - Path[] jPaths = tracker.getLocalFiles(localizedJobConf, jobWorkDir); - assertTrue("No paths found for job", jPaths.length > 0); - for (Path p : jPaths) { - if (tracker.getLocalFileSystem().exists(p)) { - createFileAndSetPermissions(localizedJobConf, p); - } - } - - // Initialize task dirs - tip.setJobConf(localizedJobConf); - tip.localizeTask(task); - - // Create a file in task local dir with 555 - // this is to simply test the case where the jvm reuse is enabled and some - // files in task-attempt-local-dir are left behind to be cleaned up when the - // job finishes. - String taskLocalDir = - TaskTracker.getLocalTaskDir(task.getUser(), task.getJobID().toString(), - task.getTaskID().toString(), false); - Path[] tPaths = tracker.getLocalFiles(localizedJobConf, taskLocalDir); - assertTrue("No paths found for task", tPaths.length > 0); - for (Path p : tPaths) { - if (tracker.getLocalFileSystem().exists(p)) { - createFileAndSetPermissions(localizedJobConf, p); - } - } - - // remove the job work dir - tracker.removeJobFiles(task.getUser(), task.getJobID()); - - // check the task-local-dir - boolean tLocalDirExists = false; - for (Path p : tPaths) { - if (tracker.getLocalFileSystem().exists(p)) { - tLocalDirExists = true; - } - } - assertFalse("Task " + task.getTaskID() + " local dir exists after cleanup", - tLocalDirExists); - - // Verify that the TaskTracker (via the task-controller) cleans up the dirs. - // check the job-work-dir - boolean jWorkDirExists = false; - for (Path p : jPaths) { - if (tracker.getLocalFileSystem().exists(p)) { - jWorkDirExists = true; - } - } - assertFalse("Job " + task.getJobID() + " work dir exists after cleanup", - jWorkDirExists); - // Test userlogs cleanup. - verifyUserLogsRemoval(); - - // Check that the empty $mapred.local.dir/taskTracker/$user dirs are still - // there. - for (String localDir : localDirs) { - Path userDir = - new Path(localDir, TaskTracker.getUserDir(task.getUser())); - assertTrue("User directory " + userDir + " is not present!!", - tracker.getLocalFileSystem().exists(userDir)); - } - } - - /** - * Tests TaskTracker restart after the localization. - * - * This tests the following steps: - * - * Localize Job, initialize a task. - * Then restart the Tracker. - * launch a cleanup attempt for the task. - * - * @throws IOException - * @throws InterruptedException - */ - public void testTrackerRestart() throws IOException, InterruptedException { - if (!canRun()) { - return; - } - - // Localize job and localize task. - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - initializeTask(); - - // imitate tracker restart - startTracker(); - - // create a task cleanup attempt - createTask(); - task.setTaskCleanupTask(); - // register task - tip = tracker.new TaskInProgress(task, trackerFConf); - - // localize the job again. - rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - checkJobLocalization(); - - // localize task cleanup attempt - initializeTask(); - checkTaskLocalization(); - } - - /** - * Tests TaskTracker re-init after the localization. - * - * This tests the following steps: - * - * Localize Job, initialize a task. - * Then reinit the Tracker. - * launch a cleanup attempt for the task. - * - * @throws IOException - * @throws InterruptedException - */ - public void testTrackerReinit() throws IOException, InterruptedException { - if (!canRun()) { - return; - } - - // Localize job and localize task. - TaskTracker.RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - initializeTask(); - - // imitate tracker reinit - initializeTracker(); - - // create a task cleanup attempt - createTask(); - task.setTaskCleanupTask(); - // register task - tip = tracker.new TaskInProgress(task, trackerFConf); - - // localize the job again. - rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - checkJobLocalization(); - - // localize task cleanup attempt - initializeTask(); - checkTaskLocalization(); - } - - /** - * Localizes a cleanup task and validates permissions. - * - * @throws InterruptedException - * @throws IOException - */ - public void testCleanupTaskLocalization() throws IOException, - InterruptedException { - if (!canRun()) { - return; - } - - task.setTaskCleanupTask(); - // register task - tip = tracker.new TaskInProgress(task, trackerFConf); - - // localize the job. - RunningJob rjob = tracker.localizeJob(tip); - localizedJobConf = rjob.getJobConf(); - checkJobLocalization(); - - // localize task cleanup attempt - initializeTask(); - checkTaskLocalization(); - - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java deleted file mode 100644 index 39ba02fd5e7..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java +++ /dev/null @@ -1,606 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin; -import org.apache.hadoop.mapreduce.util.ProcfsBasedProcessTree; -import org.apache.hadoop.mapreduce.SleepJob; -import org.apache.hadoop.mapreduce.util.TestProcfsBasedProcessTree; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.ToolRunner; - -import org.junit.After; -import org.junit.Ignore; -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * Test class to verify memory management of tasks. - */ -public class TestTaskTrackerMemoryManager { - - private static final Log LOG = - LogFactory.getLog(TestTaskTrackerMemoryManager.class); - private static String TEST_ROOT_DIR = new Path(System.getProperty( - "test.build.data", "/tmp")).toString().replace(' ', '+'); - - private MiniMRCluster miniMRCluster; - - private String taskOverLimitPatternString = - "TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond.*memory-limits. " - + "Current usage : [0-9]*bytes. Limit : %sbytes. Killing task."; - - private void startCluster(JobConf conf) - throws Exception { - conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); - conf.set(TTConfig.TT_MAP_SLOTS, "1"); - conf.set(TTConfig.TT_REDUCE_SLOTS, "1"); - conf.set(TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL, "0"); - miniMRCluster = new MiniMRCluster(1, "file:///", 1, null, null, conf); - } - - @After - public void tearDown() { - if (miniMRCluster != null) { - miniMRCluster.shutdown(); - } - } - - private int runSleepJob(JobConf conf) throws Exception { - String[] args = { "-m", "3", "-r", "1", "-mt", "3000", "-rt", "1000" }; - return ToolRunner.run(conf, new SleepJob(), args); - } - - private void runAndCheckSuccessfulJob(JobConf conf) - throws IOException { - Pattern taskOverLimitPattern = - Pattern.compile(String.format(taskOverLimitPatternString, "[0-9]*")); - Matcher mat = null; - - // Start the job. - int ret; - try { - ret = runSleepJob(conf); - } catch (Exception e) { - ret = 1; - } - - // Job has to succeed - assertTrue(ret == 0); - - JobClient jClient = new JobClient(conf); - JobStatus[] jStatus = jClient.getAllJobs(); - JobStatus js = jStatus[0]; // Our only job - RunningJob rj = jClient.getJob(js.getJobID()); - - // All events - TaskCompletionEvent[] taskComplEvents = rj.getTaskCompletionEvents(0); - - for (TaskCompletionEvent tce : taskComplEvents) { - String[] diagnostics = - rj.getTaskDiagnostics(tce.getTaskAttemptId()); - - if (diagnostics != null) { - for (String str : diagnostics) { - mat = taskOverLimitPattern.matcher(str); - // The error pattern shouldn't be there in any TIP's diagnostics - assertFalse(mat.find()); - } - } - } - } - - private boolean isProcfsBasedTreeAvailable() { - try { - if (!ProcfsBasedProcessTree.isAvailable()) { - LOG.info("Currently ProcessTree has only one implementation " - + "ProcfsBasedProcessTree, which is not available on this " - + "system. Not testing"); - return false; - } - } catch (Exception e) { - LOG.info(StringUtils.stringifyException(e)); - return false; - } - return true; - } - - /** - * Test for verifying that nothing is killed when memory management is - * disabled on the TT, even when the tasks run over their limits. - * - * @throws Exception - */ - @Test - public void testTTLimitsDisabled() - throws Exception { - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Task-memory management disabled by default. - startCluster(new JobConf()); - long PER_TASK_LIMIT = 1L; // Doesn't matter how low. - JobConf conf = miniMRCluster.createJobConf(); - conf.setMemoryForMapTask(PER_TASK_LIMIT); - conf.setMemoryForReduceTask(PER_TASK_LIMIT); - runAndCheckSuccessfulJob(conf); - } - - /** - * Test for verifying that tasks within limits, with the cumulative usage also - * under TT's limits succeed. - * - * @throws Exception - */ - @Test - public void testTasksWithinLimits() - throws Exception { - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Large so that sleepjob goes through and fits total TT usage - long PER_TASK_LIMIT = 2 * 1024L; - - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - fConf.setLong(MRConfig.MAPMEMORY_MB, 2 * 1024L); - fConf.setLong(MRConfig.REDUCEMEMORY_MB, 2 * 1024L); - // Reserve only 1 mb of the memory on TaskTrackers - fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L); - startCluster(new JobConf()); - - JobConf conf = new JobConf(miniMRCluster.createJobConf()); - conf.setMemoryForMapTask(PER_TASK_LIMIT); - conf.setMemoryForReduceTask(PER_TASK_LIMIT); - // Set task physical memory limits - conf.setLong(MRJobConfig.MAP_MEMORY_PHYSICAL_MB, PER_TASK_LIMIT); - conf.setLong(MRJobConfig.REDUCE_MEMORY_PHYSICAL_MB, PER_TASK_LIMIT); - runAndCheckSuccessfulJob(conf); - } - - /** - * Test for verifying that tasks that go beyond limits get killed. - * - * @throws Exception - */ - @Ignore("Intermittent, unexpected task success causes test to fail.") - @Test - public void testTasksBeyondLimits() - throws Exception { - - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - // very small value, so that no task escapes to successful completion. - fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100); - fConf.setLong(MRConfig.MAPMEMORY_MB, 2 * 1024); - fConf.setLong(MRConfig.REDUCEMEMORY_MB, 2 * 1024); - startCluster(fConf); - runJobExceedingMemoryLimit(false); - } - - /** - * Test for verifying that tasks that go beyond physical limits get killed. - * - * @throws Exception - */ - @Ignore("Intermittent, unexpected task success causes test to fail.") - @Test - public void testTasksBeyondPhysicalLimits() - throws Exception { - - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - // very small value, so that no task escapes to successful completion. - fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100); - // Reserve only 1 mb of the memory on TaskTrackers - fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L); - startCluster(fConf); - runJobExceedingMemoryLimit(true); - } - - /** - * Runs tests with tasks beyond limit and using old configuration values for - * the TaskTracker. - * - * @throws Exception - */ - @Ignore("Intermittent, unexpected task success causes test to fail.") - @Test - public void testTaskMemoryMonitoringWithDeprecatedConfiguration () - throws Exception { - - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - // very small value, so that no task escapes to successful completion. - fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100); - //set old values, max vm property per task and upper limit on the tasks - //vm - //setting the default maximum vmem property to 2 GB - fConf.setLong(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY, - (2L * 1024L * 1024L * 1024L)); - fConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, - (3L * 1024L * 1024L * 1024L)); - startCluster(fConf); - runJobExceedingMemoryLimit(false); - } - - /** - * Runs a job which should fail the when run by the memory monitor. - * - * @param doPhysicalMemory If it is true, use physical memory limit. - * Otherwise use virtual memory limit. - * @throws IOException - */ - private void runJobExceedingMemoryLimit(boolean doPhysicalMemory) - throws IOException { - long PER_TASK_LIMIT = 1L; // Low enough to kill off sleepJob tasks. - - Pattern taskOverLimitPattern = - Pattern.compile(String.format(taskOverLimitPatternString, String - .valueOf(PER_TASK_LIMIT*1024*1024L))); - Matcher mat = null; - - // Set up job. - JobConf conf = new JobConf(miniMRCluster.createJobConf()); - if (doPhysicalMemory) { - conf.setLong(MRJobConfig.MAP_MEMORY_PHYSICAL_MB, PER_TASK_LIMIT); - conf.setLong(MRJobConfig.REDUCE_MEMORY_PHYSICAL_MB, PER_TASK_LIMIT); - } else { - conf.setMemoryForMapTask(PER_TASK_LIMIT); - conf.setMemoryForReduceTask(PER_TASK_LIMIT); - } - conf.setMaxMapAttempts(1); - conf.setMaxReduceAttempts(1); - - // Start the job. - int ret = 0; - try { - ret = runSleepJob(conf); - } catch (Exception e) { - ret = 1; - } - - // Job has to fail - assertTrue(ret != 0); - - JobClient jClient = new JobClient(conf); - JobStatus[] jStatus = jClient.getAllJobs(); - JobStatus js = jStatus[0]; // Our only job - RunningJob rj = jClient.getJob(js.getJobID()); - - // All events - TaskCompletionEvent[] taskComplEvents = rj.getTaskCompletionEvents(0); - - for (TaskCompletionEvent tce : taskComplEvents) { - // Every task HAS to fail - assertTrue("Failure expected, task: " + tce.getTaskStatus(), - tce.getTaskStatus() == TaskCompletionEvent.Status.TIPFAILED || - tce.getTaskStatus() == TaskCompletionEvent.Status.FAILED); - - String[] diagnostics = - rj.getTaskDiagnostics(tce.getTaskAttemptId()); - - // Every task HAS to spit out the out-of-memory errors - assertNotNull(diagnostics); - - for (String str : diagnostics) { - mat = taskOverLimitPattern.matcher(str); - // Every task HAS to spit out the out-of-memory errors in the same - // format. And these are the only diagnostic messages. - assertTrue(mat.find()); - } - } - } - - /** - * Test for verifying that tasks causing cumulative usage to go beyond TT's - * limit get killed even though they all are under individual limits. Memory - * management for tasks with disabled task-limits also traverses the same - * code-path, so we don't need a separate testTaskLimitsDisabled. - * - * @throws Exception - */ - @Test - public void testTasksCumulativelyExceedingTTLimits() - throws Exception { - - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Large enough for SleepJob Tasks. - long PER_TASK_LIMIT = 100 * 1024L; - - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - fConf.setLong(MRConfig.MAPMEMORY_MB, - 1L); - fConf.setLong( - MRConfig.REDUCEMEMORY_MB, 1L); - - // Because of the above, the total tt limit is 2mb - long TASK_TRACKER_LIMIT = 2 * 1024 * 1024L; - - // very small value, so that no task escapes to successful completion. - fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100); - - startCluster(fConf); - - Pattern taskOverLimitPattern = - Pattern.compile(String.format(taskOverLimitPatternString, String - .valueOf(PER_TASK_LIMIT))); - - Pattern trackerOverLimitPattern = - Pattern - .compile("Killing one of the least progress tasks - .*, as " - + "the cumulative memory usage of all the tasks on the TaskTracker" - + " exceeds virtual memory limit " + TASK_TRACKER_LIMIT + "."); - Matcher mat = null; - - // Set up job. - JobConf conf = new JobConf(miniMRCluster.createJobConf()); - conf.setMemoryForMapTask(PER_TASK_LIMIT); - conf.setMemoryForReduceTask(PER_TASK_LIMIT); - - JobClient jClient = new JobClient(conf); - SleepJob sleepJob = new SleepJob(); - sleepJob.setConf(conf); - // Start the job - Job job = sleepJob.createJob(1, 1, 5000, 1, 1000, 1); - job.submit(); - boolean TTOverFlowMsgPresent = false; - while (true) { - List allTaskReports = new ArrayList(); - allTaskReports.addAll(Arrays.asList(jClient - .getSetupTaskReports(JobID.downgrade(job.getJobID())))); - allTaskReports.addAll(Arrays.asList(jClient - .getMapTaskReports(JobID.downgrade(job.getJobID())))); - for (TaskReport tr : allTaskReports) { - String[] diag = tr.getDiagnostics(); - for (String str : diag) { - mat = taskOverLimitPattern.matcher(str); - assertFalse(mat.find()); - mat = trackerOverLimitPattern.matcher(str); - if (mat.find()) { - TTOverFlowMsgPresent = true; - } - } - } - if (TTOverFlowMsgPresent) { - break; - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // nothing - } - } - // If it comes here without a test-timeout, it means there was a task that - // was killed because of crossing cumulative TT limit. - - // Test succeeded, kill the job. - job.killJob(); - } - - /** - * Test to verify the check for whether a process tree is over limit or not. - * @throws IOException if there was a problem setting up the - * fake procfs directories or files. - */ - @Test - public void testProcessTreeLimits() throws IOException { - - // set up a dummy proc file system - File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); - String[] pids = { "100", "200", "300", "400", "500", "600", "700" }; - try { - TestProcfsBasedProcessTree.setupProcfsRootDir(procfsRootDir); - - // create pid dirs. - TestProcfsBasedProcessTree.setupPidDirs(procfsRootDir, pids); - - // create process infos. - TestProcfsBasedProcessTree.ProcessStatInfo[] procs = - new TestProcfsBasedProcessTree.ProcessStatInfo[7]; - - // assume pids 100, 500 are in 1 tree - // 200,300,400 are in another - // 600,700 are in a third - procs[0] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"100", "proc1", "1", "100", "100", "100000"}); - procs[1] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"200", "proc2", "1", "200", "200", "200000"}); - procs[2] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"300", "proc3", "200", "200", "200", "300000"}); - procs[3] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"400", "proc4", "200", "200", "200", "400000"}); - procs[4] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"500", "proc5", "100", "100", "100", "1500000"}); - procs[5] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"600", "proc6", "1", "600", "600", "100000"}); - procs[6] = new TestProcfsBasedProcessTree.ProcessStatInfo( - new String[] {"700", "proc7", "600", "600", "600", "100000"}); - // write stat files. - TestProcfsBasedProcessTree.writeStatFiles(procfsRootDir, pids, procs); - - // vmem limit - long limit = 700000; - - // Create TaskMemoryMonitorThread - TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L, - 5000L); - // create process trees - // tree rooted at 100 is over limit immediately, as it is - // twice over the mem limit. - ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree( - "100", true, 100L, - procfsRootDir.getAbsolutePath()); - pTree.getProcessTree(); - assertTrue("tree rooted at 100 should be over limit " + - "after first iteration.", - test.isProcessTreeOverLimit(pTree, "dummyId", limit)); - - // the tree rooted at 200 is initially below limit. - pTree = new ProcfsBasedProcessTree("200", true, 100L, - procfsRootDir.getAbsolutePath()); - pTree.getProcessTree(); - assertFalse("tree rooted at 200 shouldn't be over limit " + - "after one iteration.", - test.isProcessTreeOverLimit(pTree, "dummyId", limit)); - // second iteration - now the tree has been over limit twice, - // hence it should be declared over limit. - pTree.getProcessTree(); - assertTrue("tree rooted at 200 should be over limit after 2 iterations", - test.isProcessTreeOverLimit(pTree, "dummyId", limit)); - - // the tree rooted at 600 is never over limit. - pTree = new ProcfsBasedProcessTree("600", true, 100L, - procfsRootDir.getAbsolutePath()); - pTree.getProcessTree(); - assertFalse("tree rooted at 600 should never be over limit.", - test.isProcessTreeOverLimit(pTree, "dummyId", limit)); - - // another iteration does not make any difference. - pTree.getProcessTree(); - assertFalse("tree rooted at 600 should never be over limit.", - test.isProcessTreeOverLimit(pTree, "dummyId", limit)); - } finally { - FileUtil.fullyDelete(procfsRootDir); - } - } - - /** - * Test for verifying that tasks causing cumulative usage of physical memory - * to go beyond TT's limit get killed. - * - * @throws Exception - */ - @Test - public void testTasksCumulativelyExceedingTTPhysicalLimits() - throws Exception { - - // Run the test only if memory management is enabled - if (!isProcfsBasedTreeAvailable()) { - return; - } - - // Start cluster with proper configuration. - JobConf fConf = new JobConf(); - - // very small value, so that no task escapes to successful completion. - fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100); - - // reserve all memory on TT so that the job will exceed memory limits - LinuxResourceCalculatorPlugin memoryCalculatorPlugin = - new LinuxResourceCalculatorPlugin(); - long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize(); - long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1; - fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, - reservedPhysicalMemory); - long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory - - reservedPhysicalMemory * 1024 * 1024L; - Pattern physicalMemoryOverLimitPattern = Pattern.compile( - "Killing one of the memory-consuming tasks - .*" - + ", as the cumulative RSS memory usage of all the tasks on " - + "the TaskTracker exceeds physical memory limit " - + maxRssMemoryAllowedForAllTasks + "."); - - startCluster(fConf); - Matcher mat = null; - - // Set up job. - JobConf conf = new JobConf(miniMRCluster.createJobConf()); - // Set per task physical memory limits to be a higher value - conf.setLong(MRJobConfig.MAP_MEMORY_PHYSICAL_MB, 2 * 1024L); - conf.setLong(MRJobConfig.REDUCE_MEMORY_PHYSICAL_MB, 2 * 1024L); - JobClient jClient = new JobClient(conf); - SleepJob sleepJob = new SleepJob(); - sleepJob.setConf(conf); - // Start the job - Job job = sleepJob.createJob(1, 1, 100000, 1, 100000, 1); - job.submit(); - boolean TTOverFlowMsgPresent = false; - while (true) { - List allTaskReports = new ArrayList(); - allTaskReports.addAll(Arrays.asList(jClient - .getSetupTaskReports(JobID.downgrade(job.getJobID())))); - allTaskReports.addAll(Arrays.asList(jClient - .getMapTaskReports(JobID.downgrade(job.getJobID())))); - for (TaskReport tr : allTaskReports) { - String[] diag = tr.getDiagnostics(); - for (String str : diag) { - mat = physicalMemoryOverLimitPattern.matcher(str); - if (mat.find()) { - TTOverFlowMsgPresent = true; - } - } - } - if (TTOverFlowMsgPresent) { - break; - } - assertFalse("Job should not finish successfully", job.isSuccessful()); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // nothing - } - } - // If it comes here without a test-timeout, it means there was a task that - // was killed because of crossing cumulative TT limit. - - // Test succeeded, kill the job. - job.killJob(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerSlotManagement.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerSlotManagement.java deleted file mode 100644 index 3bfbc681844..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerSlotManagement.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.net.URI; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.ClusterMetrics; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Regression test for MAPREDUCE-913 - */ -public class TestTaskTrackerSlotManagement { - - private static final Path TEST_DIR = new Path(System.getProperty( - "test.build.data", "/tmp"), "tt_slots"); - private static final String CACHE_FILE_PATH = new Path(TEST_DIR, "test.txt") - .toString(); - - /** - * Test-setup. Create the cache-file. - * - * @throws Exception - */ - @Before - public void setUp() throws Exception { - new File(TEST_DIR.toString()).mkdirs(); - File myFile = new File(CACHE_FILE_PATH); - myFile.createNewFile(); - } - - /** - * Test-cleanup. Remove the cache-file. - * - * @throws Exception - */ - @After - public void tearDown() throws Exception { - File myFile = new File(CACHE_FILE_PATH); - myFile.delete(); - new File(TEST_DIR.toString()).delete(); - } - - /** - * Test case to test addition of free slot when the job fails localization due - * to cache file being modified after the job has started running. - * - * @throws Exception - */ - @Test - public void testFreeingOfTaskSlots() throws Exception { - // Start a cluster with no task tracker. - MiniMRCluster mrCluster = new MiniMRCluster(0, "file:///", 1); - Configuration conf = mrCluster.createJobConf(); - Cluster cluster = new Cluster(conf); - // set the debug script so that TT tries to launch the debug - // script for failed tasks. - conf.set(JobContext.MAP_DEBUG_SCRIPT, "/bin/echo"); - conf.set(JobContext.REDUCE_DEBUG_SCRIPT, "/bin/echo"); - Job j = MapReduceTestUtil.createJob(conf, new Path(TEST_DIR, "in"), - new Path(TEST_DIR, "out"), 0, 0); - // Add the local filed created to the cache files of the job - j.addCacheFile(new URI(CACHE_FILE_PATH)); - j.setMaxMapAttempts(1); - j.setMaxReduceAttempts(1); - // Submit the job and return immediately. - // Job submit now takes care setting the last - // modified time of the cache file. - j.submit(); - // Look up the file and modify the modification time. - File myFile = new File(CACHE_FILE_PATH); - myFile.setLastModified(0L); - // Start up the task tracker after the time has been changed. - mrCluster.startTaskTracker(null, null, 0, 1); - // Now wait for the job to fail. - j.waitForCompletion(false); - Assert.assertFalse("Job successfully completed.", j.isSuccessful()); - - ClusterMetrics metrics = cluster.getClusterStatus(); - // validate number of slots in JobTracker - Assert.assertEquals(0, metrics.getOccupiedMapSlots()); - Assert.assertEquals(0, metrics.getOccupiedReduceSlots()); - - // validate number of slots in TaskTracker - TaskTracker tt = mrCluster.getTaskTrackerRunner(0).getTaskTracker(); - Assert.assertEquals(metrics.getMapSlotCapacity(), tt.getFreeSlots(true)); - Assert.assertEquals(metrics.getReduceSlotCapacity(), tt.getFreeSlots(false)); - - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerBlacklistAcrossJobs.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerBlacklistAcrossJobs.java deleted file mode 100644 index c4b93ee9d83..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerBlacklistAcrossJobs.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.lib.NullOutputFormat; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; - -public class TestTrackerBlacklistAcrossJobs extends TestCase { - private static final String hosts[] = new String[] { - "host1.rack.com", "host2.rack.com", "host3.rack.com" - }; - - public static class FailOnHostMapper extends MapReduceBase - implements Mapper { - String hostname = ""; - - public void configure(JobConf job) { - this.hostname = job.get(TTConfig.TT_HOST_NAME); - } - - public void map(NullWritable key, NullWritable value, - OutputCollector output, - Reporter reporter) - throws IOException { - if (this.hostname.equals(hosts[0])) { - // fail here - throw new IOException("failing on host: " + hosts[0]); - } - } - } - - public void testBlacklistAcrossJobs() throws IOException { - MiniMRCluster mr = null; - FileSystem fileSys = null; - Configuration conf = new Configuration(); - fileSys = FileSystem.get(conf); - // start mr cluster - JobConf jtConf = new JobConf(); - jtConf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1); - - mr = new MiniMRCluster(3, fileSys.getUri().toString(), - 1, null, hosts, jtConf); - - // setup job configuration - JobConf mrConf = mr.createJobConf(); - JobConf job = new JobConf(mrConf); - job.setInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 1); - job.setNumMapTasks(6); - job.setNumReduceTasks(0); - job.setMapperClass(FailOnHostMapper.class); - job.setMapOutputKeyClass(NullWritable.class); - job.setMapOutputValueClass(NullWritable.class); - job.setOutputFormat(NullOutputFormat.class); - job.setInputFormat(TestReduceFetchFromPartialMem.FakeIF.class); - - // run the job - JobClient jc = new JobClient(mrConf); - RunningJob running = JobClient.runJob(job); - assertEquals("Job failed", JobStatus.SUCCEEDED, running.getJobState()); - assertEquals("Did not blacklist the host", 1, - jc.getClusterStatus().getBlacklistedTrackers()); - assertEquals("Fault count should be 1", 1, mr.getFaultCount(hosts[0])); - - // run the same job once again - // there should be no change in blacklist count - running = JobClient.runJob(job); - assertEquals("Job failed", JobStatus.SUCCEEDED, running.getJobState()); - assertEquals("Didn't blacklist the host", 1, - jc.getClusterStatus().getBlacklistedTrackers()); - assertEquals("Fault count should be 1", 1, mr.getFaultCount(hosts[0])); - - if (fileSys != null) { fileSys.close(); } - if (mr!= null) { mr.shutdown(); } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java deleted file mode 100644 index 254a88a6513..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; - -import javax.security.auth.login.LoginException; - -import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress; -import org.apache.hadoop.mapreduce.ClusterMetrics; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; - -import junit.extensions.TestSetup; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -public class TestTrackerReservation extends TestCase { - - static String[] trackers = new String[] { "tracker_tracker1:1000", - "tracker_tracker2:1000", "tracker_tracker3:1000" }; - private static FakeJobTracker jobTracker; - - private static class FakeJobTracker extends - org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker { - - FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException, - InterruptedException, LoginException { - super(conf, clock, tts); - } - - @Override - synchronized void finalizeJob(JobInProgress job) { - // Do nothing - } - } - - - public static Test suite() { - TestSetup setup = new TestSetup(new TestSuite(TestTrackerReservation.class)) { - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - jobTracker = new FakeJobTracker(conf, new Clock(), trackers); - for (String tracker : trackers) { - FakeObjectUtilities.establishFirstContact(jobTracker, tracker); - } - } - - protected void tearDown() throws Exception { - } - }; - return setup; - } - - /** - * Test case to test if task tracker reservation. - *

    - *
  1. Run a cluster with 3 trackers.
    2. - *
  2. Submit a job which reserves all the slots in two - * trackers.
    3. - *
  3. Run the job on another tracker which has - * no reservations
    4. - *
  4. Finish the job and observe the reservations are - * successfully canceled
    5. - *
- * - * @throws Exception - */ - public void testTaskTrackerReservation() throws Exception { - JobConf conf = new JobConf(); - - conf.setNumMapTasks(1); - conf.setNumReduceTasks(1); - conf.setSpeculativeExecution(false); - - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - - //Set task tracker objects for reservation. - TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]); - TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); - TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]); - TaskTrackerStatus status1 = new TaskTrackerStatus( - trackers[0],JobInProgress.convertTrackerNameToHostName( - trackers[0]),0,new ArrayList(), 0, 2, 2); - TaskTrackerStatus status2 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - TaskTrackerStatus status3 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - tt1.setStatus(status1); - tt2.setStatus(status2); - tt3.setStatus(status3); - - FakeJobInProgress fjob = new FakeJobInProgress(conf, jobTracker); - fjob.setClusterSize(3); - fjob.initTasks(); - - tt1.reserveSlots(TaskType.MAP, fjob, 2); - tt1.reserveSlots(TaskType.REDUCE, fjob, 2); - tt3.reserveSlots(TaskType.MAP, fjob, 2); - tt3.reserveSlots(TaskType.REDUCE, fjob, 2); - - assertEquals("Trackers not reserved for the job : maps", - 2, fjob.getNumReservedTaskTrackersForMaps()); - assertEquals("Trackers not reserved for the job : reduces", - 2, fjob.getNumReservedTaskTrackersForReduces()); - ClusterMetrics metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 4, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 4, metrics.getReservedReduceSlots()); - - TaskAttemptID mTid = fjob.findMapTask(trackers[1]); - TaskAttemptID rTid = fjob.findReduceTask(trackers[1]); - - fjob.finishTask(mTid); - fjob.finishTask(rTid); - - assertEquals("Job didnt complete successfully complete", fjob.getStatus() - .getRunState(), JobStatus.SUCCEEDED); - - assertEquals("Reservation for the job not released: Maps", - 0, fjob.getNumReservedTaskTrackersForMaps()); - assertEquals("Reservation for the job not released : Reduces", - 0, fjob.getNumReservedTaskTrackersForReduces()); - metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 0, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 0, metrics.getReservedReduceSlots()); - } - - /** - * Test case to check task tracker reservation for a job which - * has a job blacklisted tracker. - *
    - *
  1. Run a job which fails on one of the tracker.
    2. - *
  2. Check if the job succeeds and has no reservation.
    3. - *
- * - * @throws Exception - */ - - public void testTrackerReservationWithJobBlackListedTracker() throws Exception { - FakeJobInProgress job = TestTaskTrackerBlacklisting.runBlackListingJob( - jobTracker, trackers); - assertEquals("Job has no blacklisted trackers", 1, job - .getBlackListedTrackers().size()); - assertTrue("Tracker 1 not blacklisted for the job", job - .getBlackListedTrackers().contains( - JobInProgress.convertTrackerNameToHostName(trackers[0]))); - assertEquals("Job didnt complete successfully complete", job.getStatus() - .getRunState(), JobStatus.SUCCEEDED); - assertEquals("Reservation for the job not released: Maps", - 0, job.getNumReservedTaskTrackersForMaps()); - assertEquals("Reservation for the job not released : Reduces", - 0, job.getNumReservedTaskTrackersForReduces()); - ClusterMetrics metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 0, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 0, metrics.getReservedReduceSlots()); - } - - /** - * Test case to check if the job reservation is handled properly if the - * job has a reservation on a black listed tracker. - * - * @throws Exception - */ - public void testReservationOnBlacklistedTracker() throws Exception { - TaskAttemptID[] taskAttemptID = new TaskAttemptID[3]; - JobConf conf = new JobConf(); - conf.setSpeculativeExecution(false); - conf.setNumMapTasks(2); - conf.setNumReduceTasks(2); - conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70"); - conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70"); - conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false); - conf.setMaxTaskFailuresPerTracker(1); - FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); - job.setClusterSize(trackers.length); - job.initTasks(); - - TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]); - TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); - TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]); - TaskTrackerStatus status1 = new TaskTrackerStatus( - trackers[0],JobInProgress.convertTrackerNameToHostName( - trackers[0]),0,new ArrayList(), 0, 2, 2); - TaskTrackerStatus status2 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - TaskTrackerStatus status3 = new TaskTrackerStatus( - trackers[1],JobInProgress.convertTrackerNameToHostName( - trackers[1]),0,new ArrayList(), 0, 2, 2); - tt1.setStatus(status1); - tt2.setStatus(status2); - tt3.setStatus(status3); - - tt1.reserveSlots(TaskType.MAP, job, 2); - tt1.reserveSlots(TaskType.REDUCE, job, 2); - tt3.reserveSlots(TaskType.MAP, job, 2); - tt3.reserveSlots(TaskType.REDUCE, job, 2); - - assertEquals("Trackers not reserved for the job : maps", - 2, job.getNumReservedTaskTrackersForMaps()); - assertEquals("Trackers not reserved for the job : reduces", - 2, job.getNumReservedTaskTrackersForReduces()); - ClusterMetrics metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 4, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 4, metrics.getReservedReduceSlots()); - - /* - * FakeJobInProgress.findMapTask does not handle - * task failures. So working around it by failing - * reduce and blacklisting tracker. - * Then finish the map task later. - */ - TaskAttemptID mTid = job.findMapTask(trackers[0]); - TaskAttemptID rTid = job.findReduceTask(trackers[0]); - //Task should blacklist the tasktracker. - job.failTask(rTid); - - assertEquals("Tracker 0 not blacklisted for the job", 1, - job.getBlackListedTrackers().size()); - assertEquals("Extra Trackers reserved for the job : maps", - 1, job.getNumReservedTaskTrackersForMaps()); - assertEquals("Extra Trackers reserved for the job : reduces", - 1, job.getNumReservedTaskTrackersForReduces()); - metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 2, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 2, metrics.getReservedReduceSlots()); - - //Finish the map task on the tracker 1. Finishing it here to work - //around bug in the FakeJobInProgress object - job.finishTask(mTid); - mTid = job.findMapTask(trackers[1]); - rTid = job.findReduceTask(trackers[1]); - job.finishTask(mTid); - job.finishTask(rTid); - rTid = job.findReduceTask(trackers[1]); - job.finishTask(rTid); - assertEquals("Job didnt complete successfully complete", job.getStatus() - .getRunState(), JobStatus.SUCCEEDED); - assertEquals("Trackers not unreserved for the job : maps", - 0, job.getNumReservedTaskTrackersForMaps()); - assertEquals("Trackers not unreserved for the job : reduces", - 0, job.getNumReservedTaskTrackersForReduces()); - metrics = jobTracker.getClusterMetrics(); - assertEquals("reserved map slots do not match", - 0, metrics.getReservedMapSlots()); - assertEquals("reserved reduce slots do not match", - 0, metrics.getReservedReduceSlots()); - } -} - \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java deleted file mode 100644 index 1592025d062..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java +++ /dev/null @@ -1,288 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.server.tasktracker.Localizer; -import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Test; - -public class TestUserLogCleanup { - private static String jtid = "test"; - private static long ONE_HOUR = 1000 * 60 * 60; - private Localizer localizer; - private UserLogCleaner taskLogCleanupThread; - private TaskTracker tt; - private FakeClock myClock = new FakeClock(); - private JobID jobid1 = new JobID(jtid, 1); - private JobID jobid2 = new JobID(jtid, 2); - private JobID jobid3 = new JobID(jtid, 3); - private JobID jobid4 = new JobID(jtid, 4); - private File foo = new File(TaskLog.getUserLogDir(), "foo"); - private File bar = new File(TaskLog.getUserLogDir(), "bar"); - - public TestUserLogCleanup() throws IOException { - Configuration conf = new Configuration(); - localizer = new Localizer(FileSystem.get(conf), conf - .getStrings(MRConfig.LOCAL_DIR), new DefaultTaskController()); - taskLogCleanupThread = new UserLogCleaner(conf); - taskLogCleanupThread.setClock(myClock); - tt = new TaskTracker(); - tt.setConf(new JobConf(conf)); - tt.setLocalizer(localizer); - tt.setTaskLogCleanupThread(taskLogCleanupThread); - } - - @After - public void tearDown() { - FileUtil.fullyDelete(TaskLog.getUserLogDir()); - } - - private File localizeJob(JobID jobid) throws IOException { - File jobUserlog = TaskLog.getJobDir(jobid); - - JobConf conf = new JobConf(); - // localize job log directory - tt.initializeJobLogDir(jobid, conf); - assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists()); - return jobUserlog; - } - - private void jobFinished(JobID jobid, int logRetainHours) { - Configuration jobconf = new Configuration(); - jobconf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, logRetainHours); - taskLogCleanupThread.markJobLogsForDeletion(myClock.getTime(), jobconf, - jobid); - } - - /** - * Tests job user-log directory deletion. - * - * Adds two jobs for log deletion. One with one hour retain hours, other with - * two retain hours. After an hour, - * TaskLogCleanupThread.processCompletedJobs() call, - * makes sure job with 1hr retain hours is removed and other is retained. - * After one more hour, job with 2hr retain hours is also removed. - * - * @throws IOException - */ - @Test - public void testJobLogCleanup() throws IOException { - File jobUserlog1 = localizeJob(jobid1); - File jobUserlog2 = localizeJob(jobid2); - - // add job user log directory for deletion, with 2 hours for deletion - jobFinished(jobid1, 2); - - // add the job for deletion with one hour as retain hours - jobFinished(jobid2, 1); - - // remove old logs and see jobid1 is not removed and jobid2 is removed - myClock.advance(ONE_HOUR); - taskLogCleanupThread.processCompletedJobs(); - assertTrue(jobUserlog1 + " got deleted", jobUserlog1.exists()); - assertFalse(jobUserlog2 + " still exists.", jobUserlog2.exists()); - - myClock.advance(ONE_HOUR); - // remove old logs and see jobid1 is removed now - taskLogCleanupThread.processCompletedJobs(); - assertFalse(jobUserlog1 + " still exists.", jobUserlog1.exists()); - } - - /** - * Tests user-log directory cleanup on a TT re-init with 3 hours as log - * retain hours for tracker. - * - * Adds job1 deletion before the re-init with 2 hour retain hours. - * Adds job2 for which there are no tasks/killJobAction after the re-init. - * Adds job3 for which there is localizeJob followed by killJobAction - * with 3 hours as retain hours. - * Adds job4 for which there are some tasks after the re-init. - * - * @throws IOException - */ - @Test - public void testUserLogCleanup() throws IOException { - File jobUserlog1 = localizeJob(jobid1); - File jobUserlog2 = localizeJob(jobid2); - File jobUserlog3 = localizeJob(jobid3); - File jobUserlog4 = localizeJob(jobid4); - // create a some files/dirs in userlog - foo.mkdirs(); - bar.createNewFile(); - - // add the jobid1 for deletion with retainhours = 2 - jobFinished(jobid1, 2); - - // time is now 1. - myClock.advance(ONE_HOUR); - - // mimic TaskTracker reinit - // clear userlog directory - // job directories will be added with 3 hours as retain hours. They will be - // deleted at time 4. - Configuration conf = new Configuration(); - conf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 3); - taskLogCleanupThread.clearOldUserLogs(conf); - assertFalse(foo.exists()); - assertFalse(bar.exists()); - assertTrue(jobUserlog1.exists()); - assertTrue(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - assertTrue(new File(TaskLog.getUserLogDir(), MRAsyncDiskService.TOBEDELETED) - .exists()); - - myClock.advance(ONE_HOUR); - // time is now 2. - taskLogCleanupThread.processCompletedJobs(); - assertFalse(jobUserlog1.exists()); - assertTrue(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - - // mimic localizeJob followed KillJobAction for jobid3 - // add the job for deletion with retainhours = 3. - // jobid3 should be deleted at time 5. - jobUserlog3 = localizeJob(jobid3); - jobFinished(jobid3, 3); - - // mimic localizeJob for jobid4 - jobUserlog4 = localizeJob(jobid4); - - // do cleanup - myClock.advance(2 * ONE_HOUR); - // time is now 4. - taskLogCleanupThread.processCompletedJobs(); - - // jobid2 will be deleted - assertFalse(jobUserlog1.exists()); - assertFalse(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - - myClock.advance(ONE_HOUR); - // time is now 5. - // do cleanup again - taskLogCleanupThread.processCompletedJobs(); - - // jobid3 will be deleted - assertFalse(jobUserlog1.exists()); - assertFalse(jobUserlog2.exists()); - assertFalse(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - } - - /** - * Tests user-log directory cleanup on a TT restart. - * - * Adds job1 deletion before the restart with 2 hour retain hours. - * Adds job2 for which there are no tasks/killJobAction after the restart. - * Adds job3 for which there is localizeJob followed by killJobAction after - * the restart with 3 hours retain hours. - * Adds job4 for which there are some tasks after the restart. - * - * @throws IOException - */ - @Test - public void testUserLogCleanupAfterRestart() throws IOException { - File jobUserlog1 = localizeJob(jobid1); - File jobUserlog2 = localizeJob(jobid2); - File jobUserlog3 = localizeJob(jobid3); - File jobUserlog4 = localizeJob(jobid4); - // create a some files/dirs in userlog - foo.mkdirs(); - bar.createNewFile(); - - // add the jobid1 for deletion with retainhours = 2 - jobFinished(jobid1, 2); - - // time is now 1. - myClock.advance(ONE_HOUR); - - // mimic TaskTracker restart - // clear userlog directory - // job directories will be added with 3 hours as retain hours. - Configuration conf = new Configuration(); - conf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 3); - taskLogCleanupThread = new UserLogCleaner(conf); - myClock = new FakeClock(); // clock is reset. - taskLogCleanupThread.setClock(myClock); - taskLogCleanupThread.clearOldUserLogs(conf); - tt.setTaskLogCleanupThread(taskLogCleanupThread); - assertFalse(foo.exists()); - assertFalse(bar.exists()); - assertTrue(jobUserlog1.exists()); - assertTrue(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - assertTrue(new File(TaskLog.getUserLogDir(), MRAsyncDiskService.TOBEDELETED) - .exists()); - - myClock.advance(ONE_HOUR); - // time is now 1. - taskLogCleanupThread.processCompletedJobs(); - assertTrue(jobUserlog1.exists()); - assertTrue(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - - // mimic localizeJob followed KillJobAction for jobid3 - // add the job for deletion with retainhours = 3. - // jobid3 should be deleted at time 4. - jobUserlog3 = localizeJob(jobid3); - jobFinished(jobid3, 3); - - // mimic localizeJob for jobid4 - jobUserlog4 = localizeJob(jobid4); - - // do cleanup - myClock.advance(2 * ONE_HOUR); - // time is now 3. - taskLogCleanupThread.processCompletedJobs(); - - // jobid1 and jobid2 will be deleted - assertFalse(jobUserlog1.exists()); - assertFalse(jobUserlog2.exists()); - assertTrue(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - - myClock.advance(ONE_HOUR); - // time is now 4. - // do cleanup again - taskLogCleanupThread.processCompletedJobs(); - - // jobid3 will be deleted - assertFalse(jobUserlog1.exists()); - assertFalse(jobUserlog2.exists()); - assertFalse(jobUserlog3.exists()); - assertTrue(jobUserlog4.exists()); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java deleted file mode 100644 index 1c7e70c4a4d..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java +++ /dev/null @@ -1,792 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.File; -import java.io.FileInputStream; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.mapred.SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat; -import org.apache.hadoop.mapred.lib.IdentityMapper; -import org.apache.hadoop.mapred.lib.IdentityReducer; -import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.util.StringUtils; - -import org.apache.commons.logging.Log; - -/** - * Utilities used in unit test. - * - */ -public class UtilsForTests { - - static final Log LOG = LogFactory.getLog(UtilsForTests.class); - - final static long KB = 1024L * 1; - final static long MB = 1024L * KB; - final static long GB = 1024L * MB; - final static long TB = 1024L * GB; - final static long PB = 1024L * TB; - final static Object waitLock = new Object(); - - static DecimalFormat dfm = new DecimalFormat("####.000"); - static DecimalFormat ifm = new DecimalFormat("###,###,###,###,###"); - - public static String dfmt(double d) { - return dfm.format(d); - } - - public static String ifmt(double d) { - return ifm.format(d); - } - - public static String formatBytes(long numBytes) { - StringBuffer buf = new StringBuffer(); - boolean bDetails = true; - double num = numBytes; - - if (numBytes < KB) { - buf.append(numBytes + " B"); - bDetails = false; - } else if (numBytes < MB) { - buf.append(dfmt(num / KB) + " KB"); - } else if (numBytes < GB) { - buf.append(dfmt(num / MB) + " MB"); - } else if (numBytes < TB) { - buf.append(dfmt(num / GB) + " GB"); - } else if (numBytes < PB) { - buf.append(dfmt(num / TB) + " TB"); - } else { - buf.append(dfmt(num / PB) + " PB"); - } - if (bDetails) { - buf.append(" (" + ifmt(numBytes) + " bytes)"); - } - return buf.toString(); - } - - public static String formatBytes2(long numBytes) { - StringBuffer buf = new StringBuffer(); - long u = 0; - if (numBytes >= TB) { - u = numBytes / TB; - numBytes -= u * TB; - buf.append(u + " TB "); - } - if (numBytes >= GB) { - u = numBytes / GB; - numBytes -= u * GB; - buf.append(u + " GB "); - } - if (numBytes >= MB) { - u = numBytes / MB; - numBytes -= u * MB; - buf.append(u + " MB "); - } - if (numBytes >= KB) { - u = numBytes / KB; - numBytes -= u * KB; - buf.append(u + " KB "); - } - buf.append(u + " B"); //even if zero - return buf.toString(); - } - - static final String regexpSpecials = "[]()?*+|.!^-\\~@"; - - public static String regexpEscape(String plain) { - StringBuffer buf = new StringBuffer(); - char[] ch = plain.toCharArray(); - int csup = ch.length; - for (int c = 0; c < csup; c++) { - if (regexpSpecials.indexOf(ch[c]) != -1) { - buf.append("\\"); - } - buf.append(ch[c]); - } - return buf.toString(); - } - - public static String safeGetCanonicalPath(File f) { - try { - String s = f.getCanonicalPath(); - return (s == null) ? f.toString() : s; - } catch (IOException io) { - return f.toString(); - } - } - - public static String slurp(File f) throws IOException { - int len = (int) f.length(); - byte[] buf = new byte[len]; - FileInputStream in = new FileInputStream(f); - String contents = null; - try { - in.read(buf, 0, len); - contents = new String(buf, "UTF-8"); - } finally { - in.close(); - } - return contents; - } - - public static String slurpHadoop(Path p, FileSystem fs) throws IOException { - int len = (int) fs.getFileStatus(p).getLen(); - byte[] buf = new byte[len]; - InputStream in = fs.open(p); - String contents = null; - try { - in.read(buf, 0, len); - contents = new String(buf, "UTF-8"); - } finally { - in.close(); - } - return contents; - } - - public static String rjustify(String s, int width) { - if (s == null) s = "null"; - if (width > s.length()) { - s = getSpace(width - s.length()) + s; - } - return s; - } - - public static String ljustify(String s, int width) { - if (s == null) s = "null"; - if (width > s.length()) { - s = s + getSpace(width - s.length()); - } - return s; - } - - static char[] space; - static { - space = new char[300]; - Arrays.fill(space, '\u0020'); - } - - public static String getSpace(int len) { - if (len > space.length) { - space = new char[Math.max(len, 2 * space.length)]; - Arrays.fill(space, '\u0020'); - } - return new String(space, 0, len); - } - - /** - * Gets job status from the jobtracker given the jobclient and the job id - */ - static JobStatus getJobStatus(JobClient jc, JobID id) throws IOException { - JobStatus[] statuses = jc.getAllJobs(); - for (JobStatus jobStatus : statuses) { - if (jobStatus.getJobID().equals(id)) { - return jobStatus; - } - } - return null; - } - - /** - * A utility that waits for specified amount of time - */ - public static void waitFor(long duration) { - try { - synchronized (waitLock) { - waitLock.wait(duration); - } - } catch (InterruptedException ie) {} - } - - /** - * Wait for the jobtracker to be RUNNING. - */ - static void waitForJobTracker(JobClient jobClient) { - while (true) { - try { - ClusterStatus status = jobClient.getClusterStatus(); - while (status.getJobTrackerStatus() != JobTrackerStatus.RUNNING) { - waitFor(100); - status = jobClient.getClusterStatus(); - } - break; // means that the jt is ready - } catch (IOException ioe) {} - } - } - - /** - * Waits until all the jobs at the jobtracker complete. - */ - static void waitTillDone(JobClient jobClient) throws IOException { - // Wait for the last job to complete - while (true) { - boolean shouldWait = false; - for (JobStatus jobStatuses : jobClient.getAllJobs()) { - if (jobStatuses.getRunState() != JobStatus.SUCCEEDED - && jobStatuses.getRunState() != JobStatus.FAILED - && jobStatuses.getRunState() != JobStatus.KILLED) { - shouldWait = true; - break; - } - } - if (shouldWait) { - waitFor(100); - } else { - break; - } - } - } - - /** - * Configure a waiting job - */ - static void configureWaitingJobConf(JobConf jobConf, Path inDir, - Path outputPath, int numMaps, int numRed, - String jobName, String mapSignalFilename, - String redSignalFilename) - throws IOException { - jobConf.setJobName(jobName); - jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); - jobConf.setOutputFormat(SequenceFileOutputFormat.class); - FileInputFormat.setInputPaths(jobConf, inDir); - FileOutputFormat.setOutputPath(jobConf, outputPath); - jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class); - jobConf.setReducerClass(IdentityReducer.class); - jobConf.setOutputKeyClass(BytesWritable.class); - jobConf.setOutputValueClass(BytesWritable.class); - jobConf.setInputFormat(RandomInputFormat.class); - jobConf.setNumMapTasks(numMaps); - jobConf.setNumReduceTasks(numRed); - jobConf.setJar("build/test/mapred/testjar/testjob.jar"); - jobConf.set(getTaskSignalParameter(true), mapSignalFilename); - jobConf.set(getTaskSignalParameter(false), redSignalFilename); - } - - /** - * Commonly used map and reduce classes - */ - - /** - * Map is a Mapper that just waits for a file to be created on the dfs. The - * file creation is a signal to the mappers and hence acts as a waiting job. - */ - - static class WaitingMapper - extends MapReduceBase - implements Mapper { - - FileSystem fs = null; - Path signal; - int id = 0; - int totalMaps = 0; - - /** - * Checks if the map task needs to wait. By default all the maps will wait. - * This method needs to be overridden to make a custom waiting mapper. - */ - public boolean shouldWait(int id) { - return true; - } - - /** - * Returns a signal file on which the map task should wait. By default all - * the maps wait on a single file passed as test.mapred.map.waiting.target. - * This method needs to be overridden to make a custom waiting mapper - */ - public Path getSignalFile(int id) { - return signal; - } - - /** The waiting function. The map exits once it gets a signal. Here the - * signal is the file existence. - */ - public void map(WritableComparable key, Writable val, - OutputCollector output, - Reporter reporter) - throws IOException { - if (shouldWait(id)) { - if (fs != null) { - while (!fs.exists(getSignalFile(id))) { - try { - reporter.progress(); - synchronized (this) { - this.wait(1000); // wait for 1 sec - } - } catch (InterruptedException ie) { - System.out.println("Interrupted while the map was waiting for " - + " the signal."); - break; - } - } - } else { - throw new IOException("Could not get the DFS!!"); - } - } - } - - public void configure(JobConf conf) { - try { - String taskId = conf.get(JobContext.TASK_ATTEMPT_ID); - id = Integer.parseInt(taskId.split("_")[4]); - totalMaps = Integer.parseInt(conf.get(JobContext.NUM_MAPS)); - fs = FileSystem.get(conf); - signal = new Path(conf.get(getTaskSignalParameter(true))); - } catch (IOException ioe) { - System.out.println("Got an exception while obtaining the filesystem"); - } - } - } - - /** Only the later half of the maps wait for the signal while the rest - * complete immediately. - */ - static class HalfWaitingMapper extends WaitingMapper { - @Override - public boolean shouldWait(int id) { - return id >= (totalMaps / 2); - } - } - - /** - * Reduce that just waits for a file to be created on the dfs. The - * file creation is a signal to the reduce. - */ - - static class WaitingReducer extends MapReduceBase - implements Reducer { - - FileSystem fs = null; - Path signal; - - /** The waiting function. The reduce exits once it gets a signal. Here the - * signal is the file existence. - */ - public void reduce(WritableComparable key, Iterator val, - OutputCollector output, - Reporter reporter) - throws IOException { - if (fs != null) { - while (!fs.exists(signal)) { - try { - reporter.progress(); - synchronized (this) { - this.wait(1000); // wait for 1 sec - } - } catch (InterruptedException ie) { - System.out.println("Interrupted while the map was waiting for the" - + " signal."); - break; - } - } - } else { - throw new IOException("Could not get the DFS!!"); - } - } - - public void configure(JobConf conf) { - try { - fs = FileSystem.get(conf); - signal = new Path(conf.get(getTaskSignalParameter(false))); - } catch (IOException ioe) { - System.out.println("Got an exception while obtaining the filesystem"); - } - } - } - - static String getTaskSignalParameter(boolean isMap) { - return isMap - ? "test.mapred.map.waiting.target" - : "test.mapred.reduce.waiting.target"; - } - - /** - * Signal the maps/reduces to start. - */ - static void signalTasks(MiniDFSCluster dfs, FileSystem fileSys, - String mapSignalFile, - String reduceSignalFile, int replication) - throws Exception { - writeFile(dfs.getNameNode(), fileSys.getConf(), new Path(mapSignalFile), - (short)replication); - writeFile(dfs.getNameNode(), fileSys.getConf(), new Path(reduceSignalFile), - (short)replication); - } - - /** - * Signal the maps/reduces to start. - */ - static void signalTasks(MiniDFSCluster dfs, FileSystem fileSys, - boolean isMap, String mapSignalFile, - String reduceSignalFile) - throws Exception { - // signal the maps to complete - writeFile(dfs.getNameNode(), fileSys.getConf(), - isMap - ? new Path(mapSignalFile) - : new Path(reduceSignalFile), (short)1); - } - - static String getSignalFile(Path dir) { - return (new Path(dir, "signal")).toString(); - } - - static String getMapSignalFile(Path dir) { - return (new Path(dir, "map-signal")).toString(); - } - - static String getReduceSignalFile(Path dir) { - return (new Path(dir, "reduce-signal")).toString(); - } - - static void writeFile(NameNode namenode, Configuration conf, Path name, - short replication) throws Exception { - FileSystem fileSys = FileSystem.get(conf); - SequenceFile.Writer writer = - SequenceFile.createWriter(fileSys, conf, name, - BytesWritable.class, BytesWritable.class, - CompressionType.NONE); - writer.append(new BytesWritable(), new BytesWritable()); - writer.close(); - fileSys.setReplication(name, replication); - DFSTestUtil.waitReplication(fileSys, name, replication); - } - - // Input formats - /** - * A custom input format that creates virtual inputs of a single string - * for each map. - */ - public static class RandomInputFormat implements InputFormat { - - public InputSplit[] getSplits(JobConf job, - int numSplits) throws IOException { - InputSplit[] result = new InputSplit[numSplits]; - Path outDir = FileOutputFormat.getOutputPath(job); - for(int i=0; i < result.length; ++i) { - result[i] = new FileSplit(new Path(outDir, "dummy-split-" + i), - 0, 1, (String[])null); - } - return result; - } - - static class RandomRecordReader implements RecordReader { - Path name; - public RandomRecordReader(Path p) { - name = p; - } - public boolean next(Text key, Text value) { - if (name != null) { - key.set(name.getName()); - name = null; - return true; - } - return false; - } - public Text createKey() { - return new Text(); - } - public Text createValue() { - return new Text(); - } - public long getPos() { - return 0; - } - public void close() {} - public float getProgress() { - return 0.0f; - } - } - - public RecordReader getRecordReader(InputSplit split, - JobConf job, - Reporter reporter) - throws IOException { - return new RandomRecordReader(((FileSplit) split).getPath()); - } - } - - // Start a job and return its RunningJob object - static RunningJob runJob(JobConf conf, Path inDir, Path outDir) - throws IOException { - return runJob(conf, inDir, outDir, conf.getNumMapTasks(), conf.getNumReduceTasks()); - } - - // Start a job and return its RunningJob object - static RunningJob runJob(JobConf conf, Path inDir, Path outDir, int numMaps, - int numReds) throws IOException { - - String input = "The quick brown fox\n" + "has many silly\n" - + "red fox sox\n"; - - // submit the job and wait for it to complete - return runJob(conf, inDir, outDir, numMaps, numReds, input); - } - - // Start a job with the specified input and return its RunningJob object - static RunningJob runJob(JobConf conf, Path inDir, Path outDir, int numMaps, - int numReds, String input) throws IOException { - FileSystem fs = FileSystem.get(conf); - if (fs.exists(outDir)) { - fs.delete(outDir, true); - } - if (!fs.exists(inDir)) { - fs.mkdirs(inDir); - } - - for (int i = 0; i < numMaps; ++i) { - DataOutputStream file = fs.create(new Path(inDir, "part-" + i)); - file.writeBytes(input); - file.close(); - } - - conf.setInputFormat(TextInputFormat.class); - conf.setOutputKeyClass(LongWritable.class); - conf.setOutputValueClass(Text.class); - - FileInputFormat.setInputPaths(conf, inDir); - FileOutputFormat.setOutputPath(conf, outDir); - conf.setNumMapTasks(numMaps); - conf.setNumReduceTasks(numReds); - - JobClient jobClient = new JobClient(conf); - RunningJob job = jobClient.submitJob(conf); - - return job; - } - - // Run a job that will be succeeded and wait until it completes - public static RunningJob runJobSucceed(JobConf conf, Path inDir, Path outDir) - throws IOException { - conf.setJobName("test-job-succeed"); - conf.setMapperClass(IdentityMapper.class); - conf.setReducerClass(IdentityReducer.class); - - RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); - while (!job.isComplete()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - - return job; - } - - // Run a job that will be failed and wait until it completes - public static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) - throws IOException { - conf.setJobName("test-job-fail"); - conf.setMapperClass(FailMapper.class); - conf.setReducerClass(IdentityReducer.class); - conf.setMaxMapAttempts(1); - - RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); - while (!job.isComplete()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - - return job; - } - - // Run a job that will be killed and wait until it completes - public static RunningJob runJobKill(JobConf conf, Path inDir, Path outDir) - throws IOException { - - conf.setJobName("test-job-kill"); - conf.setMapperClass(KillMapper.class); - conf.setReducerClass(IdentityReducer.class); - - RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); - while (job.getJobState() != JobStatus.RUNNING) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - job.killJob(); - while (job.cleanupProgress() == 0.0f) { - try { - Thread.sleep(10); - } catch (InterruptedException ie) { - break; - } - } - - return job; - } - - /** - * Cleans up files/dirs inline. CleanupQueue deletes in a separate thread - * asynchronously. - */ - public static class InlineCleanupQueue extends CleanupQueue { - List stalePaths = new ArrayList(); - - public InlineCleanupQueue() { - // do nothing - } - - @Override - public void addToQueue(PathDeletionContext... contexts) { - // delete paths in-line - for (PathDeletionContext context : contexts) { - try { - if (!deletePath(context)) { - LOG.warn("Stale path " + context.fullPath); - stalePaths.add(context.fullPath); - } - } catch (IOException e) { - LOG.warn("Caught exception while deleting path " - + context.fullPath); - LOG.info(StringUtils.stringifyException(e)); - stalePaths.add(context.fullPath); - } - } - } - } - - static class FakeClock extends Clock { - long time = 0; - - public void advance(long millis) { - time += millis; - } - - @Override - long getTime() { - return time; - } - } - // Mapper that fails - static class FailMapper extends MapReduceBase implements - Mapper { - - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - //NOTE- the next line is required for the TestDebugScript test to succeed - System.err.println("failing map"); - throw new RuntimeException("failing map"); - } - } - - // Mapper that sleeps for a long time. - // Used for running a job that will be killed - static class KillMapper extends MapReduceBase implements - Mapper { - - public void map(WritableComparable key, Writable value, - OutputCollector out, Reporter reporter) - throws IOException { - - try { - Thread.sleep(1000000); - } catch (InterruptedException e) { - // Do nothing - } - } - } - - static void setUpConfigFile(Properties confProps, File configFile) - throws IOException { - Configuration config = new Configuration(false); - FileOutputStream fos = new FileOutputStream(configFile); - - for (Enumeration e = confProps.propertyNames(); e.hasMoreElements();) { - String key = (String) e.nextElement(); - config.set(key, confProps.getProperty(key)); - } - - config.writeXml(fos); - fos.close(); - } - - static JobTracker getJobTracker() { - JobTracker jt = new JobTracker(); - return jt; - } - - /** - * This creates a file in the dfs - * @param dfs FileSystem Local File System where file needs to be picked - * @param URIPATH Path dfs path where file needs to be copied - * @param permission FsPermission File permission - * @return returns the DataOutputStream - */ - public static DataOutputStream - createTmpFileDFS(FileSystem dfs, Path URIPATH, - FsPermission permission, String input) throws Exception { - //Creating the path with the file - DataOutputStream file = - FileSystem.create(dfs, URIPATH, permission); - file.writeBytes(input); - file.close(); - return file; - } - - /** - * This formats the long tasktracker name to just the FQDN - * @param taskTrackerLong String The long format of the tasktracker string - * @return String The FQDN of the tasktracker - * @throws Exception - */ - public static String getFQDNofTT (String taskTrackerLong) throws Exception { - //Getting the exact FQDN of the tasktracker from the tasktracker string. - String[] firstSplit = taskTrackerLong.split("_"); - String tmpOutput = firstSplit[1]; - String[] secondSplit = tmpOutput.split(":"); - String tmpTaskTracker = secondSplit[0]; - return tmpTaskTracker; - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/WordCount.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/WordCount.java deleted file mode 100644 index 60d29001924..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/WordCount.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.StringTokenizer; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MapReduceBase; -import org.apache.hadoop.mapred.Mapper; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reducer; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - -/** - * This is an example Hadoop Map/Reduce application. - * It reads the text input files, breaks each line into words - * and counts them. The output is a locally sorted list of words and the - * count of how often they occurred. - * - * To run: bin/hadoop jar build/hadoop-examples.jar wordcount - * [-m maps] [-r reduces] in-dir out-dir - */ -public class WordCount extends Configured implements Tool { - - /** - * Counts the words in each line. - * For each line of input, break the line into words and emit them as - * (word, 1). - */ - public static class MapClass extends MapReduceBase - implements Mapper { - - private final static IntWritable one = new IntWritable(1); - private Text word = new Text(); - - public void map(LongWritable key, Text value, - OutputCollector output, - Reporter reporter) throws IOException { - String line = value.toString(); - StringTokenizer itr = new StringTokenizer(line); - while (itr.hasMoreTokens()) { - word.set(itr.nextToken()); - output.collect(word, one); - } - } - } - - /** - * A reducer class that just emits the sum of the input values. - */ - public static class Reduce extends MapReduceBase - implements Reducer { - - public void reduce(Text key, Iterator values, - OutputCollector output, - Reporter reporter) throws IOException { - int sum = 0; - while (values.hasNext()) { - sum += values.next().get(); - } - output.collect(key, new IntWritable(sum)); - } - } - - static int printUsage() { - System.out.println("wordcount [-m ] [-r ] "); - ToolRunner.printGenericCommandUsage(System.out); - return -1; - } - - /** - * The main driver for word count map/reduce program. - * Invoke this method to submit the map/reduce job. - * @throws IOException When there is communication problems with the - * job tracker. - */ - public int run(String[] args) throws Exception { - JobConf conf = new JobConf(getConf(), WordCount.class); - conf.setJobName("wordcount"); - - // the keys are words (strings) - conf.setOutputKeyClass(Text.class); - // the values are counts (ints) - conf.setOutputValueClass(IntWritable.class); - - conf.setMapperClass(MapClass.class); - conf.setCombinerClass(Reduce.class); - conf.setReducerClass(Reduce.class); - - List other_args = new ArrayList(); - for(int i=0; i < args.length; ++i) { - try { - if ("-m".equals(args[i])) { - conf.setNumMapTasks(Integer.parseInt(args[++i])); - } else if ("-r".equals(args[i])) { - conf.setNumReduceTasks(Integer.parseInt(args[++i])); - } else { - other_args.add(args[i]); - } - } catch (NumberFormatException except) { - System.out.println("ERROR: Integer expected instead of " + args[i]); - return printUsage(); - } catch (ArrayIndexOutOfBoundsException except) { - System.out.println("ERROR: Required parameter missing from " + - args[i-1]); - return printUsage(); - } - } - // Make sure there are exactly 2 parameters left. - if (other_args.size() != 2) { - System.out.println("ERROR: Wrong number of parameters: " + - other_args.size() + " instead of 2."); - return printUsage(); - } - FileInputFormat.setInputPaths(conf, other_args.get(0)); - FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1))); - - JobClient.runJob(conf); - return 0; - } - - - public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new WordCount(), args); - System.exit(res); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.bz2 b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.bz2 deleted file mode 100644 index f31fb0c32bb5e5b7db4e5622d78c21202dcaf30a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 208 zcmZ>Y%CIzaj8qGbyyK_&f`NhaW&?wO1A_pAb^4tN3JiumYO|PqoaKT~Nxtg&GH+?9 z%`I1+JKLxCKRevi#W?A(WBIATcYH_hO!N~`F&FNtVsL_+SEgzt&A`9}GtW->ihu%x z(k!{fR}xnQGq7B$T($6q(b@JO;VFVYc2Ckg)Y80BbHWXTe#-}wZZI&gfb@GXNHADI m^f&u^nA~(~o0Y)WDXD)x&rQ)I%4hzgcOJ{RgxGC_q8R|WP)b<< diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.gz b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/concat.gz deleted file mode 100644 index 53d5a07fcaede992597112d3d05af8a71557fc30..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmb2|=3tPx^wpbz+55DPmR_L8`cpaq9_P+``{_IkI^lcP*Gu=2)_I+i0U9SYynI7X zoIK;L`;3X<%dI~vv>6x}ac_V8Mgz9Btc&Dt?Ee=)c1hEOpj#FSOf&$Ed Nh9=1$IUWE2 diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java deleted file mode 100644 index d160de5db61..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapred.jobcontrol; - -import java.io.IOException; -import java.text.NumberFormat; -import java.util.Iterator; -import java.util.List; -import java.util.Random; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MapReduceBase; -import org.apache.hadoop.mapred.Mapper; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reducer; -import org.apache.hadoop.mapred.Reporter; - -/** - * Utility methods used in various Job Control unit tests. - */ -public class JobControlTestUtils { - - static private Random rand = new Random(); - - private static NumberFormat idFormat = NumberFormat.getInstance(); - - static { - idFormat.setMinimumIntegerDigits(4); - idFormat.setGroupingUsed(false); - } - - /** - * Cleans the data from the passed Path in the passed FileSystem. - * - * @param fs FileSystem to delete data from. - * @param dirPath Path to be deleted. - * @throws IOException If an error occurs cleaning the data. - */ - static void cleanData(FileSystem fs, Path dirPath) throws IOException { - fs.delete(dirPath, true); - } - - /** - * Generates a string of random digits. - * - * @return A random string. - */ - private static String generateRandomWord() { - return idFormat.format(rand.nextLong()); - } - - /** - * Generates a line of random text. - * - * @return A line of random text. - */ - private static String generateRandomLine() { - long r = rand.nextLong() % 7; - long n = r + 20; - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < n; i++) { - sb.append(generateRandomWord()).append(" "); - } - sb.append("\n"); - return sb.toString(); - } - - /** - * Generates data that can be used for Job Control tests. - * - * @param fs FileSystem to create data in. - * @param dirPath Path to create the data in. - * @throws IOException If an error occurs creating the data. - */ - static void generateData(FileSystem fs, Path dirPath) throws IOException { - FSDataOutputStream out = fs.create(new Path(dirPath, "data.txt")); - for (int i = 0; i < 10000; i++) { - String line = generateRandomLine(); - out.write(line.getBytes("UTF-8")); - } - out.close(); - } - - /** - * Creates a simple copy job. - * - * @param indirs List of input directories. - * @param outdir Output directory. - * @return JobConf initialised for a simple copy job. - * @throws Exception If an error occurs creating job configuration. - */ - static JobConf createCopyJob(List indirs, Path outdir) throws Exception { - - Configuration defaults = new Configuration(); - JobConf theJob = new JobConf(defaults, TestJobControl.class); - theJob.setJobName("DataMoveJob"); - - FileInputFormat.setInputPaths(theJob, indirs.toArray(new Path[0])); - theJob.setMapperClass(DataCopy.class); - FileOutputFormat.setOutputPath(theJob, outdir); - theJob.setOutputKeyClass(Text.class); - theJob.setOutputValueClass(Text.class); - theJob.setReducerClass(DataCopy.class); - theJob.setNumMapTasks(12); - theJob.setNumReduceTasks(4); - return theJob; - } - - /** - * Simple Mapper and Reducer implementation which copies data it reads in. - */ - public static class DataCopy extends MapReduceBase implements - Mapper, Reducer { - public void map(LongWritable key, Text value, OutputCollector output, - Reporter reporter) throws IOException { - output.collect(new Text(key.toString()), value); - } - - public void reduce(Text key, Iterator values, - OutputCollector output, Reporter reporter) - throws IOException { - Text dumbKey = new Text(""); - while (values.hasNext()) { - Text data = values.next(); - output.collect(dumbKey, data); - } - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.jar b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.jar deleted file mode 100644 index 07539b138db309800aec87b5568eedcfe2717ff0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 518 zcmWIWW@Zs#-~htEnLkY!kN_tG3xls~h@-BjpPT-_Qw$8u3<2Kk93T};P-Ou)HH!dM zz%~0i`gyv!28ZbRx_$ONbK1vSSMMUPx31Q?Gv_x48C)@b@U%$J%U8$K_hRWP7S0*3 zB#%hX(9B?&{!H^nnwt1i@o=%}OP+~oS3FaFo%yNgV-X|RA-Vf*U2_Lo1HuT0{6O;s z$h{@0#U*+r6(!s!Pw0d(EOKd`bmfsy>y|TfLRv39x+2oL<&V;afV8+>PDM#YY#ryk zbhNGot~jL=;Bm$~@Wg5TQ@&>yvo)_hW&&GVF!|6+N1&meU~3tfL>N$m4;D6{;6nxQ z@Bjs7fH$gEWZ!@S4*_g}Ot@C0phPwS;c3NeYV zsCI1W)N*6g9E>w{E~A0U2gW)%<(Z1o#pMGV*671!FRS@j)dfNkoQt1 y0tg_000IagfB*srAb&z~cqwOA3+fBxWGnxjsKWa3 z%Q9ARy(6K5A8RFkDA*ea3(XeTT+yB7Tp1*_IW2VGGpFpeC3#-Cs}^Z-Pkr6IvS(7w z>1Fet39JsxZZzjb6l1`p2LFubg-lw{CgU;qG=9Z-z` diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.tgz b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/test.tgz deleted file mode 100644 index 699dbc9642ecf8987133e2ce45f4e00d8888e998..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 180 zcmb2|=3uy)BkIAx{PvPB*C7LewuftMkIFcBDQ8v->I>v#EB~FS!us*cGFEZDBcXyH zYbAat*c%87%@){P(VgX786>qiEp*>Ar|h*Qd0x4z7HM%$ecimWXHw1SW%Hg1tQ8L| z+hAAm{o9@8f8|Tx#7qbYOZvrmY1h7qmL^-bOpc#^r()V<+fwhiOTurSzFU`cJnGm< d{%!H=TXMd?b!0#W56;^#ysWL1WYAz>003CoO=Nx|epFCBqAPq2t6LrH3JiC#%XNoWWs1M}DFV=f>ZTEW6l z8j-@lee#4(7{el$)=5_$3AJuHGbg0=(xWRPty}&mZ3sw<+vQZ0RK(VC&Pzw@THuOP zIsqPMyaP|1)<5NYhA~_7+G8e$0B=Snab{e$^8($#z`zK^OBz8egj-l4ZV9bm02;x{ Q29jh1LT4ad1L80M0M;-;n*aa+ diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.bz2 b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.bz2 deleted file mode 100644 index a21c0e2c10b6e7572cb330bf793d4e3aea7e25db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3056 zcmV zlrFZ~gAIvN)I$F0I>X?nA8#etbWU6qJWKA&vp134Le4RDBSjJ1HZRb43HNhm@ygmekMb$y7F-9i5}1m=jes?lJR0t%3a)( zcUM;5k2QzE0e_7(j^@q;nOp<^UViOG}h%f|_i?~clAWpL0PZqFZSdb~msW93B=(lQp zP;ZIne*bm1Jv`ea3WG>$+uJlaD1@T1f1sVa8-_-QT81L6NL-kaq-@u3T21#oUoj6eJPi`Ds8xT4*ydL0KkKS=jNn6952rUw{A* zKnDN_xaM4dKkwi0Kmv((S8hvc!&4AyVI~DVqMnd?nFo|2Z2*YXG-TRApa9TB^*oh4 zQ1pg?WB?igDWMfVB4C=2O;aJD$&(Fv_(OGk?Z>yuKi(Y9UtS2srC~h`Jvg1EB~+Mk zwvPBm)vmc?gHNSsSr_)B>kD%cC;FdnT2Vv@+#s7Kvy^-+k%6;P04fzvAC-byvE!&L z3>pMTJA;I2n!4kK@MbZBJWsXeln;`&!Wa@36ogxmUr<3?OfOi?-Zz4&9JeWwWUhhG z)&9~5*Fz~eTP1)vgf!KMXO8$d29dEkf{Uj1LzQihD?~Poz-@M&oR;55YRhyIN!`)} z*+RDeBw(3~`YH-^sV5jj^9IGH zCJQVN9Zx6|x{jHestfO;RClhEp7G;%Unei&$QEWzPvia)Lli z2O}!JtY@0y)!EdnR9rK&)l#G87$$k72YSw~(1))A<|)&HCEj#+J;Cy+5*2|>tVPDX z;*G3Q0(_tALjx01oE$)1l!=d(Z0~p^iQlclh}ko>mV1*9@Nt7A9sVxlig2MJvEyzg zLRx4xF+o`-Q&~fy6pR1>hhKmI5MT!Y2(TrLAV2@#;6MV%aJHm?XiSYH@{_|&QN1Y8 z(<4GD`3aFUG*HkQ1JnZA=UuK*l!yg1!o8zPsKJ3S$I`DznzGa`EW;o;vEF;{){);A6Dmf>AOwnf z)Ypj>tT-`m)gEC>j8!nE!Mzh8NJ@RMG$3GsN=y(sQGp{Or;0 zN&5=ch}IP_#0B6Xz@f;|F#B!fy1u1W_EORb%7uP(i?XVVE>Do}rjm~tyU^s5bJJ{2 zF;u7(Au?Z#rfne9*rkD~7vO?dDA{(D5n&Rktk}>fI~_JEjB}cs2#P8-cPoiq5w}Nn zc)+SKfR!kCM$yYTd`q1SE|7-ZEFYcmQ%;=XD^qvK%yP21eJr(NR*@Hb*OGag(=YoG z5-p~UgnrvWKx&z` zLtb4Vep7o<`}Ed7dh56^(^sc$_6lEWloe#dtN;g{$e5X1O9suQ$I949)InWRI#L2Y zK3<^beqt8+J44ijID!$EKSDL8wG6|9s$mA#zw&#KapU)=c|C(2cW^W$suGV$LSPXbVV=`wJ>=_ml9NQ~Oy9^9M z&qV@k4-)-dZ#eVX9$XQn_Xh<$6eJP7`aWP5XqZU@O$(6$EG#{D?jAVDcz`ZN%Qn{N zm!X>toSCc2aopom7-NBbbkb^nZYXu^o@d+E(Ka3k3j4dAh#j4_htUwS^4yeB4x6?& z6b$~Z*2%`yP&`v|#JOm()iaTSYP6Ps`ynR-`1Q*Eab_-S7?SMjog{)-wF`l=VWnbI zXex@Jb=+FSJ6oo(&_OM9tU8_hi@744C`c64OYHzVKZp(h diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.gz b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.gz deleted file mode 100644 index 75e5f8c7f7405238ec5f36380b4026aa20411128..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3413 zcmai!Wmppm8-_=Uv`9>Hph%3AkdBRX0#ZY17z0Mb1`0>|Qvxkzr;s-hA`7_Rs9oNDh^7(gEz?d*_9}0&^Vy#hMK+!9qf=+HZ*MUm%PU?=Z zhvaP`Uf2S*OeiMmLi|%s<#tD~cDto z9p~LmKTXUazZG?^mZ?=DRnkO&9v_Ga z9Xb@+Z{PVTniI3bccy;8s~E|8>qsKZib-?PjR_jLRbWFlP#_l%2&zFfk&zdpcB=PY zOqxOjwmWFIw{$nFKSJXa@pJR0ba(mq4HSvzD$ed2x#GS)kBF+J90Gg+hBv8MnDQHe zI#bmLJL%3+u{N89xguJXR3rHy<&h%xa1cfi##Ref%Z*bn^4s0Ey9P7j=fx4B=B%8VRZ zk{+H0JyJr&orhmo6ojwvRojK`9_p)AuIfA&C9ZfAFc_n-4z}Tjdr9R*=2~k+(J-s| z?xP!cd)mz-%E87JoJb9Hlf}#N_50ue`PYo#p)-iW1A(I}X-RE52y8b;sSmQv>L%XH zh!Ugziae6gv(3x-%H*w+y2m=*U0MJ@h`<0CdLtnNprHZ$@BU;AmE|J00W|+&`3->5 zBVE6ba?JWa)aclp$LU@Go9-!6*@8S7-Aq0-i6%?&mhum%M*gKZ_$JQ8xTq*~T3hy> z__}WQ+O(#nQH$Z2Vq8$hm{r$=kZB>zyn#rcsDFK^qbf1(tw7;HC$M1>ol7kyu7AoV zmXfP#!JSjTW9VQrSeEv!T|k);PrTEGi{~wnJ-jLPiXEytWZyyh$jF&wzL;l4zyzdY z0}5phliRS)J(7M?SGhErGs69%5rZCvhII|a)SPO1CDwayp5jK!I8`)(q8OzHKCU`x z70`|ci;z}wVYa&}8J9ju`l7kUnkZQxQB(dTXTUIatnUo|@=_377BNdhhp5ll;rY!; zc9t0KYad7(i2M=^_-RqU)XVdrQiAE3*vtLsBxgjNj)e)&uL?yEhjnu_=h$kY*H9^g zktCH#Xo`zB^3zR-N=l4i;o&Ou(ztv`bOzzC11!z8XrFA870M4;ujWtkfLC}12Pqv5 zW4&?2wO3}4Bm4FHYc_p%0YkekP5!yhPtTa>!5pq~aAU=Iw(592HAaIAcSU=S@s3>5 z`qmQV_)wSRF9H?50WNHA?*ZTirN}CF3iU$x;(7#>_L8^?emhd~dxC|3J7yAwmC z(W&Uhq+w)3&C9z5Xt4=V1`Lum!(nJn!U!y7PTmWNcl~yjUF`AeJ zLeIRqjkoBN&D!RgH-Eq`ZQatQII!EweHt9q6$x+h%<$gSDml5URYP2zX+Pf(J5(3` z7hmZs{*SK;0DY_Wpbv8F-}bmR3#ZxI$SMx*H%wiWB!q*EMf$tDW%Lhr4!AS>ABRfK zzasowf72B4g_1sAc6l{6Cv>9T zC(P$+VSm6^rpN{R!k_qgf((6AJh0(9*h8sUueIJm<@eN3B3&$m@fOmHAOi1-LA=(Y zc}=e+L0%fbi8Q7N_7!%1vT+=`F7>t2KGQ9)QZ9NVqJ1KG5jPbltpGl2WRlG+`DRDz zx!l~|oDjvLdG}jxz zMXh&V5Jfa=zIHju68O9o`#K_f?8y?PQQaf@FjB@ki};@4N*zlgQGW~AhC~T(ZZ{V6>t>5XPx{VUCmQyQTS!nxRE6E1lG4dX!K@tDjMD< zfxTrAZ^ZPN*+<`?nSQsareSMt*BmO;?#;TiaJKg=d5a#$9++t6zS?S5f7QTdMC2IP zN;Ui2J$SUWfBm|+T!ukm%6;lAKHPjZ5jCU9Qi1+jv=6nS`!zMN6@gF z=N8<)UT*rTh{-ENvf16X0P6P*Dk`v&T$R0l3hco+Y91*KkB@W(WaIZ*(t#?IZ>ew;A^!T`ah)PB%^9EPfIPqriY(;^g!u)Tqt%<(& z^YPdk^PzvSTkMbB)c>}|iU0>|UE+!|bJr%65C}%btGqul)_&yQ!*ZHWt7CcGx znRxSZmzI9JI&QSjw^cmpi6qIiyKz?o9T}v*3djb5#^&DL z7o2)L&m#jTohe8;Ue5s4>4CG|K0(wD@$>D9YDqi zM}%Pf_#i38!3K<5LHqDuL))zv4;pg`-PMYKSIlsID(>S%_lp%~9 zcfx92G~(rH=AECl2O>68DX<_$KWtbxP#|q4lp}v)k5pa}@d)^^36MfxH5-@-WM%MS zkMd{y@TiAB)slq;ctjPmxxmu>Bg4Ft3N@1lJP8MWhX*gY0wb79_Q(ho{yY38x5T>* z*GlsbXh(?dbWJV-x5JOw3-c;Y!7Ml9Hr8kKKIaCmS}kI(aN>e2ouqRD#Mhe52GG0b zt))6+rjV2Z8dNU4WNA;wxF%TMjZcAJLoA|p_o6dRHI?Si^vWF_R|8zdnJ?dsnw)9|se=T2M)^dJ$-lkU3?n1BZ z^L(CPta;HErYH2mYz}CZgDW`~ zi(P4}RXI4n9_QVc`a|q!BS$fpQkJk4azy0ugS0nj&9bfPPLT9|Z-pIE6Iq-@z z#}D!4x6Ji|ao22bOj89a%8Zibz}tZ+q%-M=)>B72yh8iDvwjR>8$qq@Es7SBiwcpb ioY`()s2VWYqn(KHpPK%f2X-tt;d%Z<_F2&d0N_8eqBH^k diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.bz2 b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.bz2 deleted file mode 100644 index 5983e52cc034a6a9b482e7c0bdca8515bb99dad0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2567 zcmV+i3i$OxT4*^jL0KkKS+$^s@Pri3F*LSUz&o{69h0MHtz=$M%kK@{~SjG6|3 z8USi~G(e}Sc{LeEjDP?D2AH7KG(aY!)bxR%003r@5T~knBSsL=&}avu07!x$Aroa! zDTL4f05hn8M7d7tg)e^Ct?BYU81O5`etAv4yI1vcCl9>gmRT+Nwh0lPB|`l1VW-3C z`T5mqpTTwdl(Yr!_WGv8WmM#PxjWQ0c?lOpN@+pP7S@z!1tp$$l` zNIgn?TTj7h+CnMHM}$tqKPT-W5R6|AB$-dq;kUe_de^7sV?eddx{dh4=FisEbwHt@JYNbu6glC7tgPu+sw6BA@o&=tvKARfP5j}wx2VSATDuE>799&7G(StRP4@vUVJw7iP2FI#kFbh~*78*@28JQLeftmaS{Jelsq&$sLfQ zyM`teGj!2h#;-XL9KjT%AtABONSV6?fx;q-<{!(5IE(R_ed#j?IJDmkl#p#@qZW43 z3ov5ttL5%X+n9%3c)Mn+5-51`+Mt?y<9#)_)b1NLwRy}{=x||ubrmFceOR@xj-g3H z-Sq+aRmQuri5aOJ*PUZtFN{)d66p({CrZb;*ZYVbr7m{uE;;V;%a#f1Q0f{cda|%L zN>tZOJy;|c*Mt(qm(lSK+CHm{%Tf@O(`KdA+Xm8ds;+Fnm_f`(AzsgujOZ7-O}J$w z;6+FfF3c@8vT~t!oNVNyWV=eFUw4#9#V;_|5Rh4P94;XU0n8t&Zw3sFzoWOYu#~xk^_Rj)Zj+b5P4sUU9giActm;Q{ zfHvVquB-}zr5DQP#v;%&qYjz)Z1I&BI*$cqw&Lz8t1x7C!{pCbu+K&-H#9a!hAqZ6 zh=75$1ffEe8ElPH1TtW|!orvo#kyH{#&g>j##aox2v-L(8Fc0*aN&@}d$Y@k6_78N zUL82$xT!`how@U^ccq005D(Q1o*NQ@LXA&aK-0c^P8!J1D9QNyphISjf_JV zBdowsPS2I(!50!^Wgs7ms+&%20N_*6FTxkKYOdUUc~zRu-oHnVG*E>D0c;Uw+@Ec! zLFn5Tp4$9bc5@EUi?yR}G^T5arIeWx7Dr&~3(ut3FnF1irecwKD%gyJ;}qYl^~ebb z`?RzvRjOv%GlDpBN$OEcQUP1;`OMX$-cjw4EJX-G%x{t$UU;Q8;;xEryhwJ5uG$<) z^J~LH8A1&%sxs-zW&h9m-IWPDZOyihJZ8@(D-M(6?5hZ&% zwg$sV+xYjbQc9P^N(0i;IfM)nwS5}qt_Fgl$0ePa=DkD`8bu=wunLM_YIbY4q!Td| z(BX)QF)aYwU=75{wP?|x)6tYkmbDp^>V}CkDVl_#D;6nU9&RO$@nF>=&}$T85VkLA zZlYcF9|S0IOj-j9-{-%dtA4h3Z-G6XWiDVagaC@)2*u?B3739FfQ59H1A|{)R9r^x zVRWvZ8#FPrHgc^_DwnGed2U4)i`~7kSMl&(G?19PMsIGG#P8)#zcO1`fgEcg~tf&ZAP`0zm+w2uLFOBps{9jx!*5s?>oEFxP&h1}PndDmLE?(X>Unjzw%mTq5(9R8fvsTfzY`a%LW8SH(eOmY4PH185YYv)M1q8 zMx8pAC!F5UZu?_D8pl&WcHy9GT(MYBo}1@`QNkCrO=oU?$?l7AyRO*+2A#k-OjlN> z{LaS4niW(>1u~-u+UdpZ+2eUyontFHZMI&6To4LgnY$YD7mOz+3E5GWtzCuP2}N*2 z1c0?lg;K6iDji=T9b!m8S>UpPMQqHSWLkyS&E^zDf3lDSD?KbKejHcUau!sZ+CGV#!2SW%g*`X z-t<5O$itZ%)3LbtEgjd)%>)Sc&$<$dqMP2fp$P6rz&r*vOfjDSA4e62J2lPM-YNmi zhpqK?I;)4{XQ*1r%LE@dfm}+CwAlhpF%4!nd&qpKqU092X~Q7Ah6{z>weXHN9qHR# z(oDp%nAfztxg?%*=$#!}p^7;kPNR|xJqr9}IuwXq`s0t_9Yt)T}vb&8l| z=apabtQjEN%us0W?5GWbnO4usNc6@oF?v+&Kt_s-mNb^T>sV!GV?guYGZ9WWcXZn@ z$gu-;EuAmXM!7A-Nq6Q1k_|{z<&AyMuqT3+k@Th?+X9yXSRJU1zF3&Y$@UX8O6;0I z4W&i}ln{j#vvDLCqcOQCwU%=ZRdhpZgtEhyY{Q48lO8$U>w^P=+>a1lMl^z=3Xa#A z1`*;4*;ROI7xy?vFqb0?8Bv+kUDVa;T|>1qiAqejR5U9z6IiQYhL0n;w8M>BA^lof^Uxs^2|#^{g-XGB)pOfODg9gXLVs1`T15 zq6SI^IW|XhNa|UR1t0HUJGJ2m$s*>8w_17v;nM`?80E|hpBa7bFtH^#tiwc;o|Wx- zuvldc1$#npJ`S7EBYUZ3+jS7eK8iB&*{qu) dRko)yK|*LTRT&7Sh8zFI+>uTcBrRvOlYseC%kuyL diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.gz b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.gz deleted file mode 100644 index 6e8eaa56f78953871b3f1dd0081f7719c1e4b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2734 zcmV;f3Q_eRiwFpK?;}hC19W9`bVF}$V_|euXk~6gZ*6dLWpi^bba-?CZCF8$+&B`v z^A+@IE>-=3J~TXDVBCXQuQ6@F*dt3yN!+GLElHI|e*H!yCAYm8z-U*MBr+Kp85yY$ zE(AM^eJ<1NBs|!cgm1gJT&x?@^%~_a+0H)pc1oKKDUNo@()l7`zg{PI_i;4A*BZp8 zY-6%>in3cD?Y!G%%|$NO;qjR2%s@`)IeHH!{aYKHgJUV|CdFo^~zV_9g>hvmFkD zH8z(8=oKjq_*=$3ylq|dKpEL31YCZ%!x|;KfPH3>`lwGxS&(5+V}FYSkxl!OiXEkz zvUU^h#EJ_pO?(Cm&d6MyT;(~l{CD+8EKXZ$CTr%zZ59O6d4P@N8xZL(QFA}!G}~T+ z2M|alb}m(xg<*ptspY63mDS-jzOB&n7UWxmLx3wOF>ArFt^t$9*iWE>$(Ks z7~AB>$yPrz1|El@6mixSfS)4&f3}@Ny1z`w7ydCV>#bB@*NRxd^Tn5%Lc}I-LbF3iYxZTbADo^6G(l{B)?BU&_TmNwOzj`BCm@rGqgVGT<~;P|7Y2eV zVI*f2l{Ev(((u8JzJTdU<=x&uzt<=m&U!H}%n9Yy>O8ia#b;nW0?i9fOdJRXPKCuK zUu=iJj9Lj2K^I`Gu{dT*kfk^lpa!P(MWalefDRimnB~Ek7H@=NAm$w)VB?}S-}$(S4JwVPu(09imV(rI?%En4FFM;$5sze14egjA*gnMJBjPh8v+nv zgGiJ{WyKnB)K`szC0h;%QLOHGMrFr^G}NOCZT2ID8AS@x4rF9&N{1{_Iz6Dc2^q(r zh$ci%pa{RV9#hZ;K@}ioNRNWUKPn$YUeRZEaEjQj3-=6~XrQ_TSICzuve|Px)kaX~ zVj>!C`|XCLPULBadhe$8V=v_uFn&a3GwZD=e0j3(b0Zj4C7QuYKvG6nfhWzr?Nbvc zQuE0ehEEXR=?~a{X8q)o(iil$PIazkr~IeresA=xq`)K?lM#Y>Q?Bq9@Y>%6dB$^b zly~T^O)xhi*(IDZDsZ#PBtDp<(7}V?XJ1coBAkzi5q!?tJ!605x=gO$p(JV~_Qc=K zThnbjL8qZT=*Jy``IJ<_cRR1xg{l1yKOpN-ThVze@<2gCpDq4|gLQ|hQVIqnx#|6b zecwu>VZOK8xL#*c_^?0o^$MDgD@p$xc^E2#7N$O;h$}~NjXH}JaaHzmIi=C)Z|Kli z1Wu6Ue6a%NuxqUR!W%Xq_hA2xL%qer0$wAKdoW)_I}hv*WWKo~H%5U4w0EEDDkt7i zq;KIjp-S`ON@?fedH|uwQCpDU?7Ab5$Sm!*0T=#L7L5 z`U;@$?znk!@S?Hl7AkL90;NqIiKAO|%d{6YWlK(jl#()8TU30T6=ptRB{v{4OYf_C z{>E`V9l5kgPv9vveCVMm*_k)Q!Rdwt@lhDRT42PqpwZNm09?>YOtxGuG}d)k(6@&eBvgzz!1GA|+RQ0~S$ z<%hC>#Sd7-!JHiUO(_H_eqQ72#kOwccxvjUsswa7Yved0gIUl+4&suEd%gfc6~P0q04>SDH0CwBs-$oE0WnfLZ@b zG4vxB%<9eDE9`f~e1Y;pN6XQw&fBf*(r#+r==3(GX-$)$bCSe}qBii338UGg&uGJ{ zazNsDjEri3qt-!zAD3dVT>GQ|YA0&0x@T31T&YRc&71|l_z~_S_@L6Xz0uUL20{Eh z<3VLVyy92ayvh$VkowoOe0RQ$Nbl@d{579Ej9qrx;2kODv{}%ZAqkh`je-^|_R9blQ!!6yEN zX>9;1KnnDrl7CM-tJ$$Kwb7$}5f|Sw!#j0K=D2Cw`v$&ksVd=jf6J=QCk?qICg#w$ zmZP0N_H92}hQ@r(qklJ4{gh(ERX)Qk0mVY)Yl-Iz0({wjro(TjUzq+8o|9GpvLuUR zR!uLMv|81(u5Y=)gB^c|mhS9bE16P&ZayZT6i&U z9)mCL>d_esD{+g`pJK>(_@Y1;LmV7rnM(>8YQ-E)3$upwxM>UIu!7i? oAv@043!5TLEBajU!_g^D0D1GPDEqvsBHHu+0Pi;d`Qj7+0K39QG5`Po diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testscript.txt b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testscript.txt deleted file mode 100644 index a9dd99db5d2..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/testscript.txt +++ /dev/null @@ -1,2 +0,0 @@ -echo 'Test Script' -cat $2 diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java deleted file mode 100644 index de2fcc4a730..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java +++ /dev/null @@ -1,463 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobInProgress; -import org.apache.hadoop.mapred.JobTracker; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.Operation; -import static org.apache.hadoop.mapred.QueueManagerTestUtils.*; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.Before; -import org.junit.Test; -import org.junit.After; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -/** - * Verify the job-ACLs - * - */ -public class TestJobACLs { - - static final Log LOG = LogFactory.getLog(TestJobACLs.class); - - private MiniMRCluster mr = null; - - private static final Path TEST_DIR = - new Path(System.getProperty("test.build.data", "/tmp"), - TestJobACLs.class.getCanonicalName() + Path.SEPARATOR - + "completed-job-store"); - - private String jobSubmitter = "jobSubmitter"; - private String viewColleague = "viewColleague"; - private String modifyColleague = "modifyColleague"; - private String qAdmin = "qAdmin"; - - /** - * Start the cluster before running the actual test. - * - * @throws IOException - */ - @Before - public void setup() throws Exception { - // Start the cluster - startCluster(false); - } - - private void startCluster(boolean reStart) throws Exception { - - // Configure job queues - String[] queueNames = {"default"}; - createQueuesConfigFile(queueNames, - new String[] { jobSubmitter }, new String[] { qAdmin }); - - JobConf conf = new JobConf(); - - // Enable queue and job level authorization - conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); - - // Enable CompletedJobStore - FileSystem fs = FileSystem.getLocal(conf); - if (!reStart) { - fs.delete(TEST_DIR, true); - } - conf.set(JTConfig.JT_PERSIST_JOBSTATUS_DIR, - fs.makeQualified(TEST_DIR).toString()); - conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, true); - conf.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); - - UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); - mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, MR_UGI, conf); - } - - /** - * Kill the cluster after the test is done. - */ - @After - public void tearDown() { - deleteQueuesConfigFile(); - if (mr != null) { - mr.shutdown(); - } - } - - /** - * Test view-job-acl, modify-job-acl and acl persistence to the - * completed-jobs-store. - * - * @throws IOException - * @throws InterruptedException - * @throws ClassNotFoundException - */ - @Test - public void testACLS() throws Exception { - verifyACLViewJob(); - verifyACLModifyJob(modifyColleague); - verifyACLModifyJob(qAdmin); - verifyACLPersistence(); - } - - /** - * Verify JobContext.JOB_ACL_VIEW_JOB - * - * @throws IOException - * @throws InterruptedException - */ - private void verifyACLViewJob() throws IOException, InterruptedException { - - // Set the job up. - final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague); - - // Submit the job as user1 - Job job = submitJobAsUser(myConf, jobSubmitter); - - final JobID jobId = job.getJobID(); - - // Try operations as an unauthorized user. - verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague); - - // Try operations as an authorized user, who is part of view-job-acl. - verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague); - - // Try operations as an authorized user, who is a queue administrator. - verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin); - - // Clean up the job - job.killJob(); - } - - /** - * Submits a sleep job with 1 map task that runs for a long time(60 sec) and - * wait for the job to go into RUNNING state. - * @param clusterConf - * @param user the jobOwner - * @return Job that is started - * @throws IOException - * @throws InterruptedException - */ - private Job submitJobAsUser(final Configuration clusterConf, String user) - throws IOException, InterruptedException { - UserGroupInformation ugi = - UserGroupInformation.createUserForTesting(user, new String[] {}); - Job job = (Job) ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - SleepJob sleepJob = new SleepJob(); - sleepJob.setConf(clusterConf); - // Disable setup/cleanup tasks at the job level - sleepJob.getConf().setBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, false); - Job myJob = sleepJob.createJob(1, 0, 60000, 1, 1, 1); - myJob.submit(); - return myJob; - } - }); - - // Make the job go into RUNNING state by forceful initialization. - JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); - JobInProgress jip = - jt.getJob(org.apache.hadoop.mapred.JobID.downgrade(job.getJobID())); - jt.initJob(jip); - - return job; - } - - private void verifyViewJobAsAuthorizedUser(final Configuration myConf, - final JobID jobId, String authorizedUser) throws IOException, - InterruptedException { - UserGroupInformation authorizedUGI = - UserGroupInformation.createUserForTesting(authorizedUser, - new String[] {}); - authorizedUGI.doAs(new PrivilegedExceptionAction() { - @SuppressWarnings("null") - @Override - public Object run() throws Exception { - Job myJob = null; - try { - Cluster cluster = new Cluster(myConf); - myJob = cluster.getJob(jobId); - } catch (Exception e) { - fail("Exception .." + e); - } - - assertNotNull("Job " + jobId + " is not known to the JobTracker!", - myJob); - - // Tests authorization with getCounters - try { - myJob.getCounters(); - } catch (IOException ioe) { - fail("Unexpected.. exception.. " + ioe); - } - - // Tests authorization with getTaskReports - try { - myJob.getTaskReports(TaskType.JOB_CLEANUP); - } catch (IOException ioe) { - fail("Unexpected.. exception.. " + ioe); - } - - return null; - } - }); - } - - private void verifyViewJobAsUnauthorizedUser(final Configuration myConf, - final JobID jobId, String unauthorizedUser) throws IOException, - InterruptedException { - UserGroupInformation unauthorizedUGI = - UserGroupInformation.createUserForTesting(unauthorizedUser, - new String[] {}); - unauthorizedUGI.doAs(new PrivilegedExceptionAction() { - @SuppressWarnings("null") - @Override - public Object run() { - Job myJob = null; - try { - Cluster cluster = new Cluster(myConf); - myJob = cluster.getJob(jobId); - } catch (Exception e) { - fail("Exception .." + e); - } - - assertNotNull("Job " + jobId + " is not known to the JobTracker!", - myJob); - - // Tests authorization failure with getCounters - try { - myJob.getCounters(); - fail("AccessControlException expected.."); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - " cannot perform operation " + JobACL.VIEW_JOB)); - } catch (InterruptedException e) { - fail("Exception .. interrupted.." + e); - } - - // Tests authorization failure with getTaskReports - try { - myJob.getTaskReports(TaskType.JOB_SETUP); - fail("AccessControlException expected.."); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - " cannot perform operation " + JobACL.VIEW_JOB)); - } catch (InterruptedException e) { - fail("Exception .. interrupted.." + e); - } - - return null; - } - }); - } - - /** - * Verify MRConfig.Job_ACL_MODIFY_JOB - * - * @throws IOException - * @throws InterruptedException - * @throws ClassNotFoundException - */ - private void verifyACLModifyJob(String authorizedUser) throws IOException, - InterruptedException, ClassNotFoundException { - - // Set the job up. - final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, modifyColleague); - - // Submit the job as user1 - Job job = submitJobAsUser(myConf, jobSubmitter); - - final JobID jobId = job.getJobID(); - - // Try operations as an unauthorized user. - verifyModifyJobAsUnauthorizedUser(myConf, jobId, viewColleague); - - // Try operations as an authorized user. - verifyModifyJobAsAuthorizedUser(myConf, jobId, authorizedUser); - } - - private void verifyModifyJobAsAuthorizedUser( - final Configuration clusterConf, final JobID jobId, - String authorizedUser) throws IOException, InterruptedException { - UserGroupInformation authorizedUGI = - UserGroupInformation.createUserForTesting(authorizedUser, - new String[] {}); - authorizedUGI.doAs(new PrivilegedExceptionAction() { - @SuppressWarnings("null") - @Override - public Object run() throws Exception { - Job myJob = null; - try { - Cluster cluster = new Cluster(clusterConf); - myJob = cluster.getJob(jobId); - } catch (Exception e) { - fail("Exception .." + e); - } - - assertNotNull("Job " + jobId + " is not known to the JobTracker!", - myJob); - - // Test authorization success with setJobPriority - try { - myJob.setPriority(JobPriority.HIGH); - assertEquals(myJob.getPriority(), JobPriority.HIGH); - } catch (IOException ioe) { - fail("Unexpected.. exception.. " + ioe); - } - - // Test authorization success with killJob - try { - myJob.killJob(); - } catch (IOException ioe) { - fail("Unexpected.. exception.. " + ioe); - } - - return null; - } - }); - } - - private void verifyModifyJobAsUnauthorizedUser( - final Configuration clusterConf, final JobID jobId, - String unauthorizedUser) throws IOException, InterruptedException { - UserGroupInformation unauthorizedUGI = - UserGroupInformation.createUserForTesting(unauthorizedUser, - new String[] {}); - unauthorizedUGI.doAs(new PrivilegedExceptionAction() { - @SuppressWarnings("null") - @Override - public Object run() { - Job myJob = null; - try { - Cluster cluster = new Cluster(clusterConf); - myJob = cluster.getJob(jobId); - } catch (Exception e) { - fail("Exception .." + e); - } - - assertNotNull("Job " + jobId + " is not known to the JobTracker!", - myJob); - - // Tests authorization failure with killJob - try { - myJob.killJob(); - fail("AccessControlException expected.."); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - " cannot perform operation " + Operation.KILL_JOB)); - } catch (InterruptedException e) { - fail("Exception .. interrupted.." + e); - } - - // Tests authorization failure with setJobPriority - try { - myJob.setPriority(JobPriority.HIGH); - fail("AccessControlException expected.."); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - " cannot perform operation " + Operation.SET_JOB_PRIORITY)); - } catch (InterruptedException e) { - fail("Exception .. interrupted.." + e); - } - - return null; - } - }); - } - - private void verifyACLPersistence() throws Exception { - - // Set the job up. - final Configuration myConf = mr.createJobConf(); - myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group2"); - - // Submit the job as user1 - Job job = submitJobAsUser(myConf, jobSubmitter); - - final JobID jobId = job.getJobID(); - - // Kill the job and wait till it is actually killed so that it is written to - // CompletedJobStore - job.killJob(); - while (job.getJobState() != JobStatus.State.KILLED) { - LOG.info("Waiting for the job to be killed successfully.."); - Thread.sleep(200); - } - - // Now kill the cluster, so that the job is 'forgotten' - tearDown(); - - // Re-start the cluster - startCluster(true); - - final Configuration myNewJobConf = mr.createJobConf(); - // Now verify view-job works off CompletedJobStore - verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, viewColleague); - verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, qAdmin); - - // Only JobCounters is persisted on the JobStore. So test counters only. - UserGroupInformation unauthorizedUGI = - UserGroupInformation.createUserForTesting( - modifyColleague, new String[] {}); - - unauthorizedUGI.doAs(new PrivilegedExceptionAction() { - @SuppressWarnings("null") - @Override - public Object run() { - Job myJob = null; - try { - Cluster cluster = new Cluster(myNewJobConf); - myJob = cluster.getJob(jobId); - } catch (Exception e) { - fail("Exception .." + e); - } - - assertNotNull("Job " + jobId + " is not known to the JobTracker!", - myJob); - - // Tests authorization failure with getCounters - try { - myJob.getCounters(); - fail("AccessControlException expected.."); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains( - " cannot perform operation " + Operation.VIEW_JOB_COUNTERS)); - } catch (InterruptedException e) { - fail("Exception .. interrupted.." + e); - } - - return null; - } - }); - - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java deleted file mode 100644 index 638f5f27988..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce; - -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.examples.MultiFileWordCount; -import org.apache.hadoop.examples.WordCount; -import org.apache.hadoop.examples.WordCount.IntSumReducer; -import org.apache.hadoop.examples.WordCount.TokenizerMapper; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; -import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.util.ToolRunner; - -/** - * A JUnit test to test min map-reduce cluster with local file system. - */ -public class TestMapReduceLocal extends TestCase { - private static Path TEST_ROOT_DIR = - new Path(System.getProperty("test.build.data","/tmp")); - private static Configuration conf = new Configuration(); - private static FileSystem localFs; - static { - try { - localFs = FileSystem.getLocal(conf); - } catch (IOException io) { - throw new RuntimeException("problem getting local fs", io); - } - } - - public static Path writeFile(String name, String data) throws IOException { - Path file = new Path(TEST_ROOT_DIR + "/" + name); - localFs.delete(file, false); - DataOutputStream f = localFs.create(file); - f.write(data.getBytes()); - f.close(); - return file; - } - - public static String readFile(String name) throws IOException { - DataInputStream f = localFs.open(new Path(TEST_ROOT_DIR + "/" + name)); - BufferedReader b = new BufferedReader(new InputStreamReader(f)); - StringBuilder result = new StringBuilder(); - String line = b.readLine(); - while (line != null) { - result.append(line); - result.append('\n'); - line = b.readLine(); - } - b.close(); - return result.toString(); - } - - public void testWithLocal() throws Exception { - MiniMRCluster mr = null; - try { - mr = new MiniMRCluster(2, "file:///", 3); - Configuration conf = mr.createJobConf(); - runWordCount(conf); - runMultiFileWordCount(conf); - } finally { - if (mr != null) { mr.shutdown(); } - } - } - - public static class TrackingTextInputFormat extends TextInputFormat { - - public static class MonoProgressRecordReader extends LineRecordReader { - private float last = 0.0f; - private boolean progressCalled = false; - @Override - public float getProgress() throws IOException { - progressCalled = true; - final float ret = super.getProgress(); - assertTrue("getProgress decreased", ret >= last); - last = ret; - return ret; - } - @Override - public synchronized void close() throws IOException { - assertTrue("getProgress never called", progressCalled); - super.close(); - } - } - - @Override - public RecordReader createRecordReader( - InputSplit split, TaskAttemptContext context) { - return new MonoProgressRecordReader(); - } - } - - private void runWordCount(Configuration conf - ) throws IOException, - InterruptedException, - ClassNotFoundException { - final String COUNTER_GROUP = "org.apache.hadoop.mapreduce.TaskCounter"; - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeFile("in/part1", "this is a test\nof word count test\ntest\n"); - writeFile("in/part2", "more test"); - Job job = Job.getInstance(conf, "word count"); - job.setJarByClass(WordCount.class); - job.setMapperClass(TokenizerMapper.class); - job.setCombinerClass(IntSumReducer.class); - job.setReducerClass(IntSumReducer.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(IntWritable.class); - job.setInputFormatClass(TrackingTextInputFormat.class); - FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in")); - FileOutputFormat.setOutputPath(job, new Path(TEST_ROOT_DIR + "/out")); - assertTrue(job.waitForCompletion(false)); - String out = readFile("out/part-r-00000"); - System.out.println(out); - assertEquals("a\t1\ncount\t1\nis\t1\nmore\t1\nof\t1\ntest\t4\nthis\t1\nword\t1\n", - out); - Counters ctrs = job.getCounters(); - System.out.println("Counters: " + ctrs); - long mapIn = ctrs.findCounter(FileInputFormatCounter.BYTES_READ).getValue(); - assertTrue(mapIn != 0); - long combineIn = ctrs.findCounter(COUNTER_GROUP, - "COMBINE_INPUT_RECORDS").getValue(); - long combineOut = ctrs.findCounter(COUNTER_GROUP, - "COMBINE_OUTPUT_RECORDS").getValue(); - long reduceIn = ctrs.findCounter(COUNTER_GROUP, - "REDUCE_INPUT_RECORDS").getValue(); - long mapOut = ctrs.findCounter(COUNTER_GROUP, - "MAP_OUTPUT_RECORDS").getValue(); - long reduceOut = ctrs.findCounter(COUNTER_GROUP, - "REDUCE_OUTPUT_RECORDS").getValue(); - long reduceGrps = ctrs.findCounter(COUNTER_GROUP, - "REDUCE_INPUT_GROUPS").getValue(); - long mergedMapOutputs = ctrs.findCounter(COUNTER_GROUP, - "MERGED_MAP_OUTPUTS").getValue(); - long shuffledMaps = ctrs.findCounter(COUNTER_GROUP, - "SHUFFLED_MAPS").getValue(); - assertEquals("map out = combine in", mapOut, combineIn); - assertEquals("combine out = reduce in", combineOut, reduceIn); - assertTrue("combine in > combine out", combineIn > combineOut); - assertEquals("reduce groups = reduce out", reduceGrps, reduceOut); - assertEquals("Mismatch in mergedMapOutputs", mergedMapOutputs, 2); - assertEquals("Mismatch in shuffledMaps", shuffledMaps, 2); - String group = "Random Group"; - CounterGroup ctrGrp = ctrs.getGroup(group); - assertEquals(0, ctrGrp.size()); - } - - public void runMultiFileWordCount(Configuration conf) throws Exception { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeFile("in/part1", "this is a test\nof " + - "multi file word count test\ntest\n"); - writeFile("in/part2", "more test"); - - int ret = ToolRunner.run(conf, new MultiFileWordCount(), - new String[] {TEST_ROOT_DIR + "/in", TEST_ROOT_DIR + "/out"}); - assertTrue("MultiFileWordCount failed", ret == 0); - String out = readFile("out/part-r-00000"); - System.out.println(out); - assertEquals("a\t1\ncount\t1\nfile\t1\nis\t1\n" + - "more\t1\nmulti\t1\nof\t1\ntest\t4\nthis\t1\nword\t1\n", out); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java deleted file mode 100644 index 8f44754f5ef..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java +++ /dev/null @@ -1,1019 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.filecache; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import javax.security.auth.login.LoginException; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.DefaultTaskController; -import org.apache.hadoop.mapred.TaskController; -import org.apache.hadoop.mapred.TaskTracker; -import org.apache.hadoop.mapreduce.Cluster; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.LocalDirAllocator; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.mapreduce.filecache.TaskDistributedCacheManager; -import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager; -import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager.CacheStatus; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ReflectionUtils; -import org.mortbay.log.Log; - -import org.mockito.Matchers; -import static org.mockito.Mockito.*; - -public class TestTrackerDistributedCacheManager extends TestCase { - - protected String TEST_ROOT_DIR = - new File(System.getProperty("test.build.data", "/tmp"), - TestTrackerDistributedCacheManager.class.getSimpleName()) - .getAbsolutePath(); - - protected File ROOT_MAPRED_LOCAL_DIR; - protected int numLocalDirs = 6; - - private static final int TEST_FILE_SIZE = 4 * 1024; // 4K - private static final int LOCAL_CACHE_LIMIT = 5 * 1024; //5K - private static final int LOCAL_CACHE_SUBDIR = 2; - protected Configuration conf; - protected Path firstCacheFile; - protected Path secondCacheFile; - private FileSystem fs; - - protected LocalDirAllocator localDirAllocator = - new LocalDirAllocator(MRConfig.LOCAL_DIR); - protected TaskController taskController; - - @Override - protected void setUp() throws IOException,InterruptedException { - - // Prepare the tests' root dir - File TEST_ROOT = new File(TEST_ROOT_DIR); - if (!TEST_ROOT.exists()) { - TEST_ROOT.mkdirs(); - } - - conf = new Configuration(); - conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); - fs = FileSystem.get(conf); - - // This test suite will fail if any ancestor directory of the - // test directory is not world-searchable (ie +x). - // We prefer to fail the test in an obvious manner up front - // during setUp() rather than in a subtle way later. - assertTrue("Test root directory " + TEST_ROOT + " and all of its " + - "parent directories must have a+x permissions", - ClientDistributedCacheManager.ancestorsHaveExecutePermissions( - fs, new Path(TEST_ROOT.toString()))); - - // Prepare the tests' mapred-local-dir - ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local"); - ROOT_MAPRED_LOCAL_DIR.mkdirs(); - - String []localDirs = new String[numLocalDirs]; - for (int i = 0; i < numLocalDirs; i++) { - File localDir = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i); - localDirs[i] = localDir.getPath(); - localDir.mkdir(); - } - - conf.setStrings(MRConfig.LOCAL_DIR, localDirs); - Class taskControllerClass = conf.getClass( - TTConfig.TT_TASK_CONTROLLER, DefaultTaskController.class, - TaskController.class); - taskController = (TaskController) ReflectionUtils.newInstance( - taskControllerClass, conf); - - // setup permissions for mapred local dir - taskController.setup(); - - // Create the temporary cache files to be used in the tests. - firstCacheFile = new Path(TEST_ROOT_DIR, "firstcachefile"); - secondCacheFile = new Path(TEST_ROOT_DIR, "secondcachefile"); - createPrivateTempFile(firstCacheFile); - createPrivateTempFile(secondCacheFile); - } - - protected void refreshConf(Configuration conf) throws IOException { - taskController.setConf(conf); - taskController.setup(); - } - - /** - * Whether the test can run on the machine - * - * @return true if test can run on the machine, false otherwise - */ - protected boolean canRun() { - return true; - } - - /** - * This is the typical flow for using the DistributedCache classes. - * - * @throws IOException - * @throws LoginException - */ - public void testManagerFlow() throws IOException, LoginException { - if (!canRun()) { - return; - } - - // ****** Imitate JobClient code - // Configures a task/job with both a regular file and a "classpath" file. - Configuration subConf = new Configuration(conf); - String userName = getJobOwnerName(); - subConf.set(MRJobConfig.USER_NAME, userName); - DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf); - DistributedCache.addFileToClassPath(secondCacheFile, subConf); - ClientDistributedCacheManager.determineTimestamps(subConf); - ClientDistributedCacheManager.determineCacheVisibilities(subConf); - // ****** End of imitating JobClient code - - Path jobFile = new Path(TEST_ROOT_DIR, "job.xml"); - FileOutputStream os = new FileOutputStream(new File(jobFile.toString())); - subConf.writeXml(os); - os.close(); - - // ****** Imitate TaskRunner code. - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf, taskController); - TaskDistributedCacheManager handle = - manager.newTaskDistributedCacheManager(subConf); - assertNull(null, DistributedCache.getLocalCacheFiles(subConf)); - File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString()); - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - // ****** End of imitating TaskRunner code - - Path[] localCacheFiles = DistributedCache.getLocalCacheFiles(subConf); - assertNotNull(null, localCacheFiles); - assertEquals(2, localCacheFiles.length); - Path cachedFirstFile = localCacheFiles[0]; - Path cachedSecondFile = localCacheFiles[1]; - assertFileLengthEquals(firstCacheFile, cachedFirstFile); - assertFalse("Paths should be different.", - firstCacheFile.equals(cachedFirstFile)); - - assertEquals(1, handle.getClassPaths().size()); - assertEquals(cachedSecondFile.toString(), handle.getClassPaths().get(0)); - - checkFilePermissions(localCacheFiles); - - // Cleanup - handle.release(); - manager.purgeCache(); - assertFalse(pathToFile(cachedFirstFile).exists()); - } - - /** - * This DistributedCacheManager fails in localizing firstCacheFile. - */ - public class FakeTrackerDistributedCacheManager extends - TrackerDistributedCacheManager { - public FakeTrackerDistributedCacheManager(Configuration conf) - throws IOException { - super(conf, taskController); - } - - @Override - Path localizeCache(Configuration conf, URI cache, long confFileStamp, - CacheStatus cacheStatus, boolean isArchive, boolean isPublic) - throws IOException { - if (cache.equals(firstCacheFile.toUri())) { - throw new IOException("fake fail"); - } - return super.localizeCache(conf, cache, confFileStamp, cacheStatus, - isArchive, isPublic); - } - } - - public void testReferenceCount() throws IOException, LoginException, - URISyntaxException, InterruptedException { - if (!canRun()) { - return; - } - TrackerDistributedCacheManager manager = - new FakeTrackerDistributedCacheManager(conf); - String userName = getJobOwnerName(); - File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString()); - - // Configures a job with a regular file - Job job1 = Job.getInstance(conf); - job1.setUser(userName); - job1.addCacheFile(secondCacheFile.toUri()); - Configuration conf1 = job1.getConfiguration(); - ClientDistributedCacheManager.determineTimestamps(conf1); - ClientDistributedCacheManager.determineCacheVisibilities(conf1); - - // Task localizing for first job - TaskDistributedCacheManager handle = manager - .newTaskDistributedCacheManager(conf1); - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - handle.release(); - for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) { - assertEquals(0, manager.getReferenceCount(c.uri, conf1, c.timestamp, - c.owner, false)); - } - - Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile"); - createPrivateTempFile(thirdCacheFile); - - // Configures another job with three regular files. - Job job2 = Job.getInstance(conf); - job2.setUser(userName); - // add a file that would get failed to localize - job2.addCacheFile(firstCacheFile.toUri()); - // add a file that is already localized by different job - job2.addCacheFile(secondCacheFile.toUri()); - // add a file that is never localized - job2.addCacheFile(thirdCacheFile.toUri()); - Configuration conf2 = job2.getConfiguration(); - ClientDistributedCacheManager.determineTimestamps(conf2); - ClientDistributedCacheManager.determineCacheVisibilities(conf2); - - // Task localizing for second job - // localization for the "firstCacheFile" will fail. - handle = manager.newTaskDistributedCacheManager(conf2); - Throwable th = null; - try { - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - } catch (IOException e) { - th = e; - Log.info("Exception during setup", e); - } - assertNotNull(th); - assertTrue(th.getMessage().contains("fake fail")); - handle.release(); - th = null; - for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) { - try { - assertEquals(0, manager.getReferenceCount(c.uri, conf2, c.timestamp, - c.owner, false)); - } catch (IOException ie) { - th = ie; - Log.info("Exception getting reference count for " + c.uri, ie); - } - } - assertNotNull(th); - assertTrue(th.getMessage().contains(thirdCacheFile.getName())); - fs.delete(thirdCacheFile, false); - } - - /** - * Tests that localization of distributed cache file happens in the desired - * directory - * @throws IOException - * @throws LoginException - */ - public void testPublicPrivateCache() - throws IOException, LoginException, InterruptedException { - if (!canRun()) { - return; - } - checkLocalizedPath(true); - checkLocalizedPath(false); - } - - public void testPrivateCacheForMultipleUsers() - throws IOException, LoginException, InterruptedException{ - if (!canRun()) { - return; - } - // Try to initialize the distributed cache for the same file on the - // HDFS, for two different users. - // First initialize as the user running the test, then as some other user. - // Although the same cache file is used in both, the localization - // should happen twice. - - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - Path p = ugi.doAs(new PrivilegedExceptionAction() { - public Path run() - throws IOException, LoginException, InterruptedException { - return checkLocalizedPath(false); - } - }); - String distCacheDir = TaskTracker.getPrivateDistributedCacheDir( - ugi.getShortUserName()); - assertTrue("Cache file didn't get localized in the expected directory. " + - "Expected localization to happen within " + - ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir + - ", but was localized at " + - p, p.toString().contains(distCacheDir)); - - ugi = UserGroupInformation.createRemoteUser("fooUserInMachine"); - p = ugi.doAs(new PrivilegedExceptionAction() { - public Path run() - throws IOException, LoginException, InterruptedException { - return checkLocalizedPath(false); - } - }); - distCacheDir = TaskTracker.getPrivateDistributedCacheDir( - ugi.getShortUserName()); - assertTrue("Cache file didn't get localized in the expected directory. " + - "Expected localization to happen within " + - ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir + - ", but was localized at " + - p, p.toString().contains(distCacheDir)); - - } - - private Path checkLocalizedPath(boolean visibility) - throws IOException, LoginException, InterruptedException { - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf, taskController); - String userName = getJobOwnerName(); - File workDir = new File(TEST_ROOT_DIR, "workdir"); - Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile"); - if (visibility) { - createPublicTempFile(cacheFile); - } else { - createPrivateTempFile(cacheFile); - } - - Job job1 = Job.getInstance(conf); - job1.setUser(userName); - job1.addCacheFile(cacheFile.toUri()); - Configuration conf1 = job1.getConfiguration(); - ClientDistributedCacheManager.determineTimestamps(conf1); - ClientDistributedCacheManager.determineCacheVisibilities(conf1); - - // Task localizing for job - TaskDistributedCacheManager handle = manager - .newTaskDistributedCacheManager(conf1); - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0); - String distCacheDir; - if (visibility) { - distCacheDir = TaskTracker.getPublicDistributedCacheDir(); - } else { - distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName); - } - Path localizedPath = - manager.getLocalCache(cacheFile.toUri(), conf1, distCacheDir, - fs.getFileStatus(cacheFile), false, - c.timestamp, new Path(TEST_ROOT_DIR), false, - visibility); - assertTrue("Cache file didn't get localized in the expected directory. " + - "Expected localization to happen within " + - ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir + - ", but was localized at " + - localizedPath, localizedPath.toString().contains(distCacheDir)); - if (visibility) { - checkPublicFilePermissions(new Path[]{localizedPath}); - } else { - checkFilePermissions(new Path[]{localizedPath}); - } - return localizedPath; - } - - /** - * Check proper permissions on the cache files - * - * @param localCacheFiles - * @throws IOException - */ - protected void checkFilePermissions(Path[] localCacheFiles) - throws IOException { - // All the files should have executable permissions on them. - for (Path p : localCacheFiles) { - assertTrue("Cache file is not executable!", new File(p - .toUri().getPath()).canExecute()); - } - } - - /** - * Check permissions on the public cache files - * - * @param localCacheFiles - * @throws IOException - */ - private void checkPublicFilePermissions(Path[] localCacheFiles) - throws IOException { - checkPublicFilePermissions(fs, localCacheFiles); - } - - /** - * Verify the permissions for a file localized as a public distributed - * cache file - * @param fs The Local FileSystem used to get the permissions - * @param localCacheFiles The list of files whose permissions should be - * verified. - * @throws IOException - */ - public static void checkPublicFilePermissions(FileSystem fs, - Path[] localCacheFiles) throws IOException { - // All the files should have read and executable permissions for others - for (Path p : localCacheFiles) { - FsPermission perm = fs.getFileStatus(p).getPermission(); - assertTrue("cache file is not readable / executable by owner: perm=" - + perm.getUserAction(), perm.getUserAction() - .implies(FsAction.READ_EXECUTE)); - assertTrue("cache file is not readable / executable by group: perm=" - + perm.getGroupAction(), perm.getGroupAction() - .implies(FsAction.READ_EXECUTE)); - assertTrue("cache file is not readable / executable by others: perm=" - + perm.getOtherAction(), perm.getOtherAction() - .implies(FsAction.READ_EXECUTE)); - } - } - - /** - * Verify the ownership for files localized as a public distributed cache - * file. - * @param fs The Local FileSystem used to get the ownership - * @param localCacheFiles THe list of files whose ownership should be - * verified - * @param owner The owner of the files - * @param group The group owner of the files. - * @throws IOException - */ - public static void checkPublicFileOwnership(FileSystem fs, - Path[] localCacheFiles, String owner, String group) - throws IOException { - for (Path p: localCacheFiles) { - assertEquals(owner, fs.getFileStatus(p).getOwner()); - assertEquals(group, fs.getFileStatus(p).getGroup()); - } - } - - public static class MyTrackerDistributedCacheManager - extends TrackerDistributedCacheManager { - - public Throwable caught = null; - public CountDownLatch done = new CountDownLatch(1); - - public MyTrackerDistributedCacheManager(Configuration conf, - TaskController controller) throws IOException { - super(conf, controller); - this.baseDirManager = new TrackerDistributedCacheManager.BaseDirManager() { - - @Override - public void checkAndCleanup() throws IOException { - throw new RuntimeException("This is a test!!!!"); - } - }; - - this.cleanupThread = new TestCleanupThread(conf); - } - - class TestCleanupThread extends TrackerDistributedCacheManager.CleanupThread { - - public TestCleanupThread(Configuration conf) { - super(conf); - } - - @Override - protected void exitTaskTracker(Throwable t) { - caught = t; - this.stopRunning(); - done.countDown(); - } - } - } - - public void testRuntimeExceptionInCleanup() throws Exception { - if(!canRun()) { - return; - } - - Configuration conf2 = new Configuration(conf); - conf2.set("mapred.local.dir", ROOT_MAPRED_LOCAL_DIR.toString()); - conf2.setLong("local.cache.size", LOCAL_CACHE_LIMIT); - conf2.setLong("mapreduce.tasktracker.distributedcache.checkperiod", 0); // 0 ms (Don't sleep) - - refreshConf(conf2); - MyTrackerDistributedCacheManager manager = - new MyTrackerDistributedCacheManager(conf2, taskController); - manager.startCleanupThread(); - - assertTrue(manager.done.await(200l, TimeUnit.MILLISECONDS)); - assertNotNull(manager.caught); - assertTrue(manager.caught instanceof RuntimeException); - } - - protected String getJobOwnerName() throws IOException { - return UserGroupInformation.getCurrentUser().getUserName(); - } - - private long getFileStamp(Path file) throws IOException { - FileStatus fileStatus = fs.getFileStatus(file); - return fileStatus.getModificationTime(); - } - - public static final long CACHE_DELETE_PERIOD_MS = 100l; - - /** test delete cache */ - public void testLRUDeleteCache() throws Exception { - if (!canRun()) { - return; - } - // This test needs MRConfig.LOCAL_DIR to be single directory - // instead of four, because it assumes that both - // firstcachefile and secondcachefile will be localized on same directory - // so that second localization triggers deleteCache. - // If MRConfig.LOCAL_DIR is four directories, second localization might not - // trigger deleteCache, if it is localized in different directory. - Configuration conf2 = new Configuration(conf); - conf2.set(MRConfig.LOCAL_DIR, ROOT_MAPRED_LOCAL_DIR.toString()); - //Make it larger then expected - conf2.setLong(TTConfig.TT_LOCAL_CACHE_SIZE, 21 * 1024l); - conf2.setLong(TTConfig.TT_LOCAL_CACHE_SUBDIRS_LIMIT, 3); - //The goal is to get down to 15.75K and 2 dirs - conf2.setFloat(TTConfig.TT_LOCAL_CACHE_KEEP_AROUND_PCT, 0.75f); - conf2.setLong(TTConfig.TT_DISTRIBUTED_CACHE_CHECK_PERIOD, CACHE_DELETE_PERIOD_MS); - refreshConf(conf2); - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf2, taskController); - manager.startCleanupThread(); - FileSystem localfs = FileSystem.getLocal(conf2); - String userName = getJobOwnerName(); - conf2.set(MRJobConfig.USER_NAME, userName); - - //Here we are testing the LRU. In this case we will add in 4 cache entries - // 2 of them are 8k each and 2 of them are very small. We want to verify - // That they are deleted in LRU order. - // So what we will do is add in the two large files first, 1 then 2, and - // then one of the small ones 3. We will then release them in opposite - // order 3, 2, 1. - // - // Finally we will add in the last small file. This last file should push - // us over the 3 entry limit to trigger a cleanup. So LRU order is 3, 2, 1 - // And we will only delete 2 entries so that should leave 1 un touched - // but 3 and 2 deleted - - Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile"); - Path fourthCacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile"); - // Adding two more small files, so it triggers the number of sub directory - // limit but does not trigger the file size limit. - createTempFile(thirdCacheFile, 1); - createTempFile(fourthCacheFile, 1); - - Path firstLocalCache = manager.getLocalCache(firstCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(firstCacheFile), false, - getFileStamp(firstCacheFile), new Path(TEST_ROOT_DIR), false, false); - - Path secondLocalCache = manager.getLocalCache(secondCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(secondCacheFile), false, - getFileStamp(secondCacheFile), new Path(TEST_ROOT_DIR), false, false); - - Path thirdLocalCache = manager.getLocalCache(thirdCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(thirdCacheFile), false, - getFileStamp(thirdCacheFile), new Path(TEST_ROOT_DIR), false, false); - - manager.releaseCache(thirdCacheFile.toUri(), conf2, - getFileStamp(thirdCacheFile), - TrackerDistributedCacheManager.getLocalizedCacheOwner(false), false); - - manager.releaseCache(secondCacheFile.toUri(), conf2, - getFileStamp(secondCacheFile), - TrackerDistributedCacheManager.getLocalizedCacheOwner(false), false); - - manager.releaseCache(firstCacheFile.toUri(), conf2, - getFileStamp(firstCacheFile), - TrackerDistributedCacheManager.getLocalizedCacheOwner(false), false); - - - // Getting the fourth cache will make the number of sub directories becomes - // 4 which is greater than 3. So the released cache will be deleted. - manager.getLocalCache(fourthCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(fourthCacheFile), false, - getFileStamp(fourthCacheFile), new Path(TEST_ROOT_DIR), false, false); - - checkCacheDeletion(localfs, secondLocalCache, "DistributedCache failed " + - "deleting second cache LRU order"); - - checkCacheDeletion(localfs, thirdLocalCache, - "DistributedCache failed deleting third" + - " cache LRU order."); - - checkCacheNOTDeletion(localfs, firstLocalCache, "DistributedCache failed " + - "Deleted first cache LRU order."); - - checkCacheNOTDeletion(localfs, fourthCacheFile, "DistributedCache failed " + - "Deleted fourth cache LRU order."); - // Clean up the files created in this test - new File(thirdCacheFile.toString()).delete(); - new File(fourthCacheFile.toString()).delete(); - manager.stopCleanupThread(); - } - - public void testSameNameFileArchiveCache() throws IOException, - URISyntaxException, InterruptedException { - if (!canRun()) { - return; - } - TrackerDistributedCacheManager manager = - spy(new TrackerDistributedCacheManager(conf, taskController)); - URI rsrc = new URI("file://foo/bar/yak"); - Path cacheDir = new Path("file:///localcache"); - Path archivePath = new Path(cacheDir, "archive"); - Path filePath = new Path(cacheDir, "file"); - doReturn(archivePath).when(manager).localizeCache(eq(conf), eq(rsrc), - anyLong(), Matchers. anyObject(), eq(true), anyBoolean()); - doReturn(filePath).when(manager).localizeCache(eq(conf), eq(rsrc), - anyLong(), Matchers. anyObject(), eq(false), anyBoolean()); - // could fail, but check match instead - doNothing().when(manager).checkCacheStatusValidity( - Matchers. anyObject(), eq(rsrc), anyLong(), - Matchers. anyObject(), Matchers. anyObject(), - anyBoolean()); - // localizeCache initializes mtime of cached rsrc; set to uninitialized val - doReturn(-1L).when(manager).checkStampSinceJobStarted( - Matchers. anyObject(), - Matchers. anyObject(), eq(rsrc), anyLong(), - Matchers. anyObject(), Matchers. anyObject()); - doReturn(-1L).when(manager).getTimestamp( - Matchers. anyObject(), eq(rsrc)); - FileStatus rsrcStatus = mock(FileStatus.class); - when(rsrcStatus.getLen()).thenReturn(4344L); - - Path localizedPathForFile = - manager.getLocalCache(rsrc, conf, "sub", rsrcStatus, false, 20L, - new Path("file:///tmp"), false, true); - Path localizedPathForArchive = - manager.getLocalCache(rsrc, conf, "sub", rsrcStatus, true, 20L, - new Path("file:///tmp"), false, true); - assertNotSame("File and Archive resolve to the same path: " - + localizedPathForFile + ". Should differ.", localizedPathForFile, - localizedPathForArchive); - } - - /** test delete cache */ - public void testDeleteCache() throws Exception { - if (!canRun()) { - return; - } - // This test needs MRConfig.LOCAL_DIR to be single directory - // instead of four, because it assumes that both - // firstcachefile and secondcachefile will be localized on same directory - // so that second localization triggers deleteCache. - // If MRConfig.LOCAL_DIR is four directories, second localization might not - // trigger deleteCache, if it is localized in different directory. - Configuration conf2 = new Configuration(conf); - conf2.set(MRConfig.LOCAL_DIR, ROOT_MAPRED_LOCAL_DIR.toString()); - conf2.setLong(TTConfig.TT_LOCAL_CACHE_SIZE, LOCAL_CACHE_LIMIT); - conf2.setLong(TTConfig.TT_LOCAL_CACHE_SUBDIRS_LIMIT, LOCAL_CACHE_SUBDIR); - conf2.setLong(TTConfig.TT_DISTRIBUTED_CACHE_CHECK_PERIOD, CACHE_DELETE_PERIOD_MS); - refreshConf(conf2); - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf2, taskController); - manager.startCleanupThread(); - FileSystem localfs = FileSystem.getLocal(conf2); - String userName = getJobOwnerName(); - conf2.set(MRJobConfig.USER_NAME, userName); - - // We first test the size limit - Path localCache = manager.getLocalCache(firstCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(firstCacheFile), false, - getFileStamp(firstCacheFile), new Path(TEST_ROOT_DIR), false, false); - manager.releaseCache(firstCacheFile.toUri(), conf2, - getFileStamp(firstCacheFile), - TrackerDistributedCacheManager.getLocalizedCacheOwner(false), false); - //in above code,localized a file of size 4K and then release the cache - // which will cause the cache be deleted when the limit goes out. - // The below code localize another cache which's designed to - //sweep away the first cache. - manager.getLocalCache(secondCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(secondCacheFile), false, - getFileStamp(secondCacheFile), new Path(TEST_ROOT_DIR), false, false); - checkCacheDeletion(localfs, localCache, "DistributedCache failed " + - "deleting old cache when the cache store is full."); - // Now we test the number of sub directories limit - // Create the temporary cache files to be used in the tests. - Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile"); - Path fourthCacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile"); - // Adding two more small files, so it triggers the number of sub directory - // limit but does not trigger the file size limit. - createTempFile(thirdCacheFile, 1); - createTempFile(fourthCacheFile, 1); - Path thirdLocalCache = manager.getLocalCache(thirdCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(thirdCacheFile), false, - getFileStamp(thirdCacheFile), new Path(TEST_ROOT_DIR), false, false); - // Release the third cache so that it can be deleted while sweeping - manager.releaseCache(thirdCacheFile.toUri(), conf2, - getFileStamp(thirdCacheFile), - TrackerDistributedCacheManager.getLocalizedCacheOwner(false), false); - // Getting the fourth cache will make the number of sub directories becomes - // 3 which is greater than 2. So the released cache will be deleted. - manager.getLocalCache(fourthCacheFile.toUri(), conf2, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(fourthCacheFile), false, - getFileStamp(fourthCacheFile), new Path(TEST_ROOT_DIR), false, false); - checkCacheDeletion(localfs, thirdLocalCache, - "DistributedCache failed deleting old" + - " cache when the cache exceeds the number of sub directories limit."); - // Clean up the files created in this test - new File(thirdCacheFile.toString()).delete(); - new File(fourthCacheFile.toString()).delete(); - manager.stopCleanupThread(); - } - - /** - * Do a simple check to see if the file has NOT been deleted. - */ - private void checkCacheNOTDeletion(FileSystem fs, Path cache, String msg) - throws Exception { - TimeUnit.MILLISECONDS.sleep(3 * CACHE_DELETE_PERIOD_MS); - assertTrue(msg, fs.exists(cache)); - } - - /** - * Periodically checks if a file is there, return if the file is no longer - * there. Fails the test if a files is there for 30 seconds. - */ - private void checkCacheDeletion(FileSystem fs, Path cache, String msg) - throws Exception { - // Check every 100ms to see if the cache is deleted - boolean cacheExists = true; - for (int i = 0; i < 300; i++) { - if (!fs.exists(cache)) { - cacheExists = false; - break; - } - TimeUnit.MILLISECONDS.sleep(CACHE_DELETE_PERIOD_MS); - } - // If the cache is still there after 5 minutes, test fails. - assertFalse(msg, cacheExists); - } - - public void testFileSystemOtherThanDefault() throws Exception { - if (!canRun()) { - return; - } - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf, taskController); - conf.set("fs.fakefile.impl", conf.get("fs.file.impl")); - String userName = getJobOwnerName(); - conf.set(MRJobConfig.USER_NAME, userName); - Path fileToCache = new Path("fakefile:///" - + firstCacheFile.toUri().getPath()); - Path result = manager.getLocalCache(fileToCache.toUri(), conf, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(firstCacheFile), false, - getFileStamp(firstCacheFile), - new Path(TEST_ROOT_DIR), false, false); - assertNotNull("DistributedCache cached file on non-default filesystem.", - result); - } - - static void createTempFile(Path p) throws IOException { - createTempFile(p, TEST_FILE_SIZE); - } - - static void createTempFile(Path p, int size) throws IOException { - File f = new File(p.toString()); - FileOutputStream os = new FileOutputStream(f); - byte[] toWrite = new byte[size]; - new Random().nextBytes(toWrite); - os.write(toWrite); - os.close(); - FileSystem.LOG.info("created: " + p + ", size=" + size); - } - - static void createPublicTempFile(Path p) - throws IOException, InterruptedException { - createTempFile(p); - FileUtil.chmod(p.toString(), "0777",true); - } - - static void createPrivateTempFile(Path p) - throws IOException, InterruptedException { - createTempFile(p); - FileUtil.chmod(p.toString(), "0770",true); - } - - @Override - protected void tearDown() throws IOException { - new File(firstCacheFile.toString()).delete(); - new File(secondCacheFile.toString()).delete(); - FileUtil.fullyDelete(new File(TEST_ROOT_DIR)); - } - - protected void assertFileLengthEquals(Path a, Path b) - throws FileNotFoundException { - assertEquals("File sizes mismatch.", - pathToFile(a).length(), pathToFile(b).length()); - } - - protected File pathToFile(Path p) { - return new File(p.toString()); - } - - public static class FakeFileSystem extends RawLocalFileSystem { - private long increment = 0; - public FakeFileSystem() { - super(); - } - - public FileStatus getFileStatus(Path p) throws IOException { - File f = pathToFile(p); - return new FileStatus(f.length(), f.isDirectory(), 1, 128, - f.lastModified() + increment, makeQualified(new Path(f.getPath()))); - } - - void advanceClock(long millis) { - increment += millis; - } - } - - public void testFreshness() throws Exception { - if (!canRun()) { - return; - } - Configuration myConf = new Configuration(conf); - myConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "refresh:///"); - myConf.setClass("fs.refresh.impl", FakeFileSystem.class, FileSystem.class); - String userName = getJobOwnerName(); - - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(myConf, taskController); - // ****** Imitate JobClient code - // Configures a task/job with both a regular file and a "classpath" file. - Configuration subConf = new Configuration(myConf); - subConf.set(MRJobConfig.USER_NAME, userName); - DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf); - ClientDistributedCacheManager.determineTimestamps(subConf); - ClientDistributedCacheManager.determineCacheVisibilities(subConf); - // ****** End of imitating JobClient code - - // ****** Imitate TaskRunner code. - TaskDistributedCacheManager handle = - manager.newTaskDistributedCacheManager(subConf); - assertNull(null, DistributedCache.getLocalCacheFiles(subConf)); - File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString()); - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - // ****** End of imitating TaskRunner code - - Path[] localCacheFiles = DistributedCache.getLocalCacheFiles(subConf); - assertNotNull(null, localCacheFiles); - assertEquals(1, localCacheFiles.length); - Path cachedFirstFile = localCacheFiles[0]; - assertFileLengthEquals(firstCacheFile, cachedFirstFile); - assertFalse("Paths should be different.", - firstCacheFile.equals(cachedFirstFile)); - // release - handle.release(); - - // change the file timestamp - FileSystem fs = FileSystem.get(myConf); - ((FakeFileSystem)fs).advanceClock(1); - - // running a task of the same job - Throwable th = null; - try { - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), TaskTracker.getPublicDistributedCacheDir()); - } catch (IOException ie) { - th = ie; - } - assertNotNull("Throwable is null", th); - assertTrue("Exception message does not match", - th.getMessage().contains("has changed on HDFS since job started")); - // release - handle.release(); - - // running a task of the same job on another TaskTracker which has never - // initialized the cache - TrackerDistributedCacheManager manager2 = - new TrackerDistributedCacheManager(myConf, taskController); - TaskDistributedCacheManager handle2 = - manager2.newTaskDistributedCacheManager(subConf); - File workDir2 = new File(new Path(TEST_ROOT_DIR, "workdir2").toString()); - th = null; - try { - handle2.setup(localDirAllocator, workDir2, TaskTracker - .getPrivateDistributedCacheDir(userName), - TaskTracker.getPublicDistributedCacheDir()); - } catch (IOException ie) { - th = ie; - } - assertNotNull("Throwable is null", th); - assertTrue("Exception message does not match", - th.getMessage().contains("has changed on HDFS since job started")); - // release - handle.release(); - - // submit another job - Configuration subConf2 = new Configuration(myConf); - subConf2.set(MRJobConfig.USER_NAME, userName); - DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf2); - ClientDistributedCacheManager.determineTimestamps(subConf2); - ClientDistributedCacheManager.determineCacheVisibilities(subConf2); - - handle = - manager.newTaskDistributedCacheManager(subConf2); - handle.setup(localDirAllocator, workDir, TaskTracker - .getPrivateDistributedCacheDir(userName), TaskTracker.getPublicDistributedCacheDir()); - Path[] localCacheFiles2 = DistributedCache.getLocalCacheFiles(subConf2); - assertNotNull(null, localCacheFiles2); - assertEquals(1, localCacheFiles2.length); - Path cachedFirstFile2 = localCacheFiles2[0]; - assertFileLengthEquals(firstCacheFile, cachedFirstFile2); - assertFalse("Paths should be different.", - firstCacheFile.equals(cachedFirstFile2)); - - // assert that two localizations point to different paths - assertFalse("two jobs with different timestamps did not localize" + - " in different paths", cachedFirstFile.equals(cachedFirstFile2)); - // release - handle.release(); - } - - /** - * Localize a file. After localization is complete, create a file, "myFile", - * under the directory where the file is localized and ensure that it has - * permissions different from what is set by default. Then, localize another - * file. Verify that "myFile" has the right permissions. - * @throws Exception - */ - public void testCustomPermissions() throws Exception { - if (!canRun()) { - return; - } - String userName = getJobOwnerName(); - conf.set(MRJobConfig.USER_NAME, userName); - TrackerDistributedCacheManager manager = - new TrackerDistributedCacheManager(conf, taskController); - FileSystem localfs = FileSystem.getLocal(conf); - - Path[] localCache = new Path[2]; - localCache[0] = manager.getLocalCache(firstCacheFile.toUri(), conf, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(firstCacheFile), false, - getFileStamp(firstCacheFile), new Path(TEST_ROOT_DIR), false, false); - FsPermission myPermission = new FsPermission((short)0600); - Path myFile = new Path(localCache[0].getParent(), "myfile.txt"); - if (FileSystem.create(localfs, myFile, myPermission) == null) { - throw new IOException("Could not create " + myFile); - } - try { - localCache[1] = manager.getLocalCache(secondCacheFile.toUri(), conf, - TaskTracker.getPrivateDistributedCacheDir(userName), - fs.getFileStatus(secondCacheFile), false, - getFileStamp(secondCacheFile), new Path(TEST_ROOT_DIR), false, - false); - FileStatus stat = localfs.getFileStatus(myFile); - assertTrue(stat.getPermission().equals(myPermission)); - // validate permissions of localized files. - checkFilePermissions(localCache); - } finally { - localfs.delete(myFile, false); - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java deleted file mode 100644 index c297ac06e3f..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEvents.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapreduce.jobhistory; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapred.TaskStatus; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.TaskType; - -/** - * Test various jobhistory events - */ -public class TestJobHistoryEvents extends TestCase { - static final int[][] NULL_SPLITS_ARRAY - = new int[org.apache.hadoop.tools.rumen.LoggedTaskAttempt.SplitVectorKind.values().length][]; - - static { - for (int i = 0; i < NULL_SPLITS_ARRAY.length; ++i) { - NULL_SPLITS_ARRAY[i] = new int[0]; - } - } - - /** - * Test {@link TaskAttemptStartedEvent} for various task types. - */ - private static void testAttemptStartedEventForTypes(EventType expected, - TaskAttemptID id, - TaskType[] types) { - for (TaskType t : types) { - TaskAttemptStartedEvent tase = - new TaskAttemptStartedEvent(id, t, 0L, "", 0, -1); - assertEquals(expected, tase.getEventType()); - } - } - - /** - * Test {@link TaskAttemptStartedEvent}. - */ - public void testTaskAttemptStartedEvent() { - EventType expected = EventType.MAP_ATTEMPT_STARTED; - TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); - - // check the events for job-setup, job-cleanup and map task-types - testAttemptStartedEventForTypes(expected, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.MAP}); - - expected = EventType.REDUCE_ATTEMPT_STARTED; - fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); - - // check the events for job-setup, job-cleanup and reduce task-types - testAttemptStartedEventForTypes(expected, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.REDUCE}); - } - - /** - * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for various task types. - */ - private static void testFailedKilledEventsForTypes(EventType expected, - TaskAttemptID id, - TaskType[] types, - String state) { - for (TaskType t : types) { - TaskAttemptUnsuccessfulCompletionEvent tauce = - new TaskAttemptUnsuccessfulCompletionEvent - (id, t, state, 0L, "", -1, "", "", NULL_SPLITS_ARRAY); - assertEquals(expected, tauce.getEventType()); - } - } - - /** - * Test {@link TaskAttemptUnsuccessfulCompletionEvent} for killed/failed task. - */ - public void testTaskAttemptUnsuccessfulCompletionEvent() { - TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); - - // check killed events for job-setup, job-cleanup and map task-types - testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_KILLED, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.MAP}, - TaskStatus.State.KILLED.toString()); - // check failed events for job-setup, job-cleanup and map task-types - testFailedKilledEventsForTypes(EventType.MAP_ATTEMPT_FAILED, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.MAP}, - TaskStatus.State.FAILED.toString()); - - fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); - - // check killed events for job-setup, job-cleanup and reduce task-types - testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_KILLED, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.REDUCE}, - TaskStatus.State.KILLED.toString()); - // check failed events for job-setup, job-cleanup and reduce task-types - testFailedKilledEventsForTypes(EventType.REDUCE_ATTEMPT_FAILED, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.REDUCE}, - TaskStatus.State.FAILED.toString()); - } - - /** - * Test {@link TaskAttemptFinishedEvent} for various task types. - */ - private static void testFinishedEventsForTypes(EventType expected, - TaskAttemptID id, - TaskType[] types) { - for (TaskType t : types) { - TaskAttemptFinishedEvent tafe = - new TaskAttemptFinishedEvent(id, t, - TaskStatus.State.SUCCEEDED.toString(), 0L, "", "", "", - new Counters()); - assertEquals(expected, tafe.getEventType()); - } - } - - /** - * Test {@link TaskAttemptFinishedEvent} for finished task. - */ - public void testTaskAttemptFinishedEvent() { - EventType expected = EventType.MAP_ATTEMPT_FINISHED; - TaskAttemptID fakeId = new TaskAttemptID("1234", 1, TaskType.MAP, 1, 1); - - // check the events for job-setup, job-cleanup and map task-types - testFinishedEventsForTypes(expected, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.MAP}); - - expected = EventType.REDUCE_ATTEMPT_FINISHED; - fakeId = new TaskAttemptID("1234", 1, TaskType.REDUCE, 1, 1); - - // check the events for job-setup, job-cleanup and reduce task-types - testFinishedEventsForTypes(expected, fakeId, - new TaskType[] {TaskType.JOB_SETUP, - TaskType.JOB_CLEANUP, - TaskType.REDUCE}); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java deleted file mode 100644 index 521316fa18e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCacheOldApi.java +++ /dev/null @@ -1,296 +0,0 @@ -/** Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.security; - - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import javax.crypto.KeyGenerator; -import javax.crypto.spec.SecretKeySpec; - -import org.apache.commons.codec.binary.Base64; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.EmptyInputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobTracker; -import org.apache.hadoop.mapred.Mapper; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Partitioner; -import org.apache.hadoop.mapred.Reducer; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.lib.NullOutputFormat; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - - -@SuppressWarnings("deprecation") -public class TestTokenCacheOldApi { - private static final int NUM_OF_KEYS = 10; - - // my sleep class - adds check for tokenCache - static class MyDummyJob extends Configured implements Tool, - Mapper, - Reducer, - Partitioner { - Credentials ts; - - public void configure(JobConf job) { - } - - /** - * attempts to access tokenCache as from client - */ - public void map(IntWritable key, IntWritable value, - OutputCollector output, Reporter reporter) - throws IOException { - // get token storage and a key - byte[] key1 = ts.getSecretKey(new Text("alias1")); - Collection> dts = ts.getAllTokens(); - int dts_size = 0; - if(dts != null) - dts_size = dts.size(); - - if(dts.size() != 2) { // one job token and one delegation token - throw new RuntimeException("tokens are not available"); // fail the test - } - - if(key1 == null || ts == null || ts.numberOfSecretKeys() != NUM_OF_KEYS) { - throw new RuntimeException("secret keys are not available"); // fail the test - } - - output.collect(new IntWritable(1), NullWritable.get()); - } - - public JobConf setupJobConf() { - - JobConf job = new JobConf(getConf(), MyDummyJob.class); - job.setNumMapTasks(1); - job.setNumReduceTasks(1); - job.setMapperClass(MyDummyJob.class); - job.setMapOutputKeyClass(IntWritable.class); - job.setMapOutputValueClass(NullWritable.class); - job.setReducerClass(MyDummyJob.class); - job.setOutputFormat(NullOutputFormat.class); - job.setInputFormat(EmptyInputFormat.class); - job.setPartitionerClass(MyDummyJob.class); - job.setSpeculativeExecution(false); - job.setJobName("Sleep job"); - populateTokens(job); - return job; - } - - private void populateTokens(JobConf job) { - // Credentials in the job will not have delegation tokens - // because security is disabled. Fetch delegation tokens - // and populate the credential in the job. - try { - Credentials ts = job.getCredentials(); - Path p1 = new Path("file1"); - p1 = p1.getFileSystem(job).makeQualified(p1); - Credentials cred = new Credentials(); - TokenCache.obtainTokensForNamenodesInternal(cred, new Path[] { p1 }, - job); - for (Token t : cred.getAllTokens()) { - ts.addToken(new Text("Hdfs"), t); - } - } catch (IOException e) { - Assert.fail("Exception " + e); - } - } - - public void close() throws IOException { - } - - public void reduce(IntWritable key, Iterator values, - OutputCollector output, Reporter reporter) - throws IOException { - return; - } - - public int getPartition(IntWritable key, NullWritable value, - int numPartitions) { - return key.get() % numPartitions; - } - - public int run(String[] args) throws Exception { - JobConf job = setupJobConf(); - JobClient.runJob(job); - return 0; - } - } - - private static MiniMRCluster mrCluster; - private static MiniDFSCluster dfsCluster; - private static final Path TEST_DIR = - new Path(System.getProperty("test.build.data","/tmp"), "sleepTest"); - private static final Path tokenFileName = new Path(TEST_DIR, "tokenFile.json"); - private static int numSlaves = 1; - private static JobConf jConf; - private static ObjectMapper mapper = new ObjectMapper(); - private static Path p1; - private static Path p2; - - @BeforeClass - public static void setUp() throws Exception { - Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); - jConf = new JobConf(conf); - mrCluster = new MiniMRCluster(0, 0, numSlaves, - dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, - jConf); - - createTokenFileJson(); - verifySecretKeysInJSONFile(); - NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); - FileSystem fs = dfsCluster.getFileSystem(); - - p1 = new Path("file1"); - p2 = new Path("file2"); - p1 = fs.makeQualified(p1); - } - - @AfterClass - public static void tearDown() throws Exception { - if(mrCluster != null) - mrCluster.shutdown(); - mrCluster = null; - if(dfsCluster != null) - dfsCluster.shutdown(); - dfsCluster = null; - } - - // create jason file and put some keys into it.. - private static void createTokenFileJson() throws IOException { - Map map = new HashMap(); - - try { - KeyGenerator kg = KeyGenerator.getInstance("HmacSHA1"); - for(int i=0; i map; - map = mapper.readValue(new File(tokenFileName.toString()), Map.class); - assertEquals("didn't read JSON correctly", map.size(), NUM_OF_KEYS); - } - - /** - * run a distributed job and verify that TokenCache is available - * @throws IOException - */ - @Test - public void testTokenCache() throws IOException { - // make sure JT starts - jConf = mrCluster.createJobConf(); - - // provide namenodes names for the job to get the delegation tokens for - //String nnUri = dfsCluster.getNameNode().getUri(namenode).toString(); - NameNode nn = dfsCluster.getNameNode(); - URI nnUri = NameNode.getUri(nn.getNameNodeAddress()); - jConf.set(JobContext.JOB_NAMENODES, nnUri + "," + nnUri.toString()); - // job tracker principle id.. - jConf.set(JobTracker.JT_USER_NAME, "jt_id"); - - // using argument to pass the file name - String[] args = { - "-tokenCacheFile", tokenFileName.toString(), - "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" - }; - - int res = -1; - try { - res = ToolRunner.run(jConf, new MyDummyJob(), args); - } catch (Exception e) { - System.out.println("Job failed with" + e.getLocalizedMessage()); - e.printStackTrace(System.out); - Assert.fail("Job failed"); - } - assertEquals("dist job res is not 0", res, 0); - } - - /** - * run a local job and verify that TokenCache is available - * @throws NoSuchAlgorithmException - * @throws IOException - */ - @Test - public void testLocalJobTokenCache() throws NoSuchAlgorithmException, IOException { - // this is local job - String[] args = {"-m", "1", "-r", "1", "-mt", "1", "-rt", "1"}; - jConf.set("mapreduce.job.credentials.json", tokenFileName.toString()); - - int res = -1; - try { - res = ToolRunner.run(jConf, new MyDummyJob(), args); - } catch (Exception e) { - System.out.println("Job failed with" + e.getLocalizedMessage()); - e.printStackTrace(System.out); - fail("local Job failed"); - } - assertEquals("local job res is not 0", res, 0); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordMR.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordMR.java deleted file mode 100644 index 8dfe18efcf8..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordMR.java +++ /dev/null @@ -1,467 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.record; - -import org.apache.hadoop.mapred.*; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.conf.*; -import junit.framework.TestCase; -import java.io.*; -import java.util.*; - - -/********************************************************** - * MapredLoadTest generates a bunch of work that exercises - * a Hadoop Map-Reduce system (and DFS, too). It goes through - * the following steps: - * - * 1) Take inputs 'range' and 'counts'. - * 2) Generate 'counts' random integers between 0 and range-1. - * 3) Create a file that lists each integer between 0 and range-1, - * and lists the number of times that integer was generated. - * 4) Emit a (very large) file that contains all the integers - * in the order generated. - * 5) After the file has been generated, read it back and count - * how many times each int was generated. - * 6) Compare this big count-map against the original one. If - * they match, then SUCCESS! Otherwise, FAILURE! - * - * OK, that's how we can think about it. What are the map-reduce - * steps that get the job done? - * - * 1) In a non-mapred thread, take the inputs 'range' and 'counts'. - * 2) In a non-mapread thread, generate the answer-key and write to disk. - * 3) In a mapred job, divide the answer key into K jobs. - * 4) A mapred 'generator' task consists of K map jobs. Each reads - * an individual "sub-key", and generates integers according to - * to it (though with a random ordering). - * 5) The generator's reduce task agglomerates all of those files - * into a single one. - * 6) A mapred 'reader' task consists of M map jobs. The output - * file is cut into M pieces. Each of the M jobs counts the - * individual ints in its chunk and creates a map of all seen ints. - * 7) A mapred job integrates all the count files into a single one. - * - **********************************************************/ -public class TestRecordMR extends TestCase { - /** - * Modified to make it a junit test. - * The RandomGen Job does the actual work of creating - * a huge file of assorted numbers. It receives instructions - * as to how many times each number should be counted. Then - * it emits those numbers in a crazy order. - * - * The map() function takes a key/val pair that describes - * a value-to-be-emitted (the key) and how many times it - * should be emitted (the value), aka "numtimes". map() then - * emits a series of intermediate key/val pairs. It emits - * 'numtimes' of these. The key is a random number and the - * value is the 'value-to-be-emitted'. - * - * The system collates and merges these pairs according to - * the random number. reduce() function takes in a key/value - * pair that consists of a crazy random number and a series - * of values that should be emitted. The random number key - * is now dropped, and reduce() emits a pair for every intermediate value. - * The emitted key is an intermediate value. The emitted value - * is just a blank string. Thus, we've created a huge file - * of numbers in random order, but where each number appears - * as many times as we were instructed. - */ - static public class RandomGenMapper implements Mapper { - Random r = new Random(); - public void configure(JobConf job) { - } - - public void map(RecInt key, - RecInt val, - OutputCollector out, - Reporter reporter) throws IOException { - int randomVal = key.getData(); - int randomCount = val.getData(); - - for (int i = 0; i < randomCount; i++) { - out.collect(new RecInt(Math.abs(r.nextInt())), - new RecString(Integer.toString(randomVal))); - } - } - public void close() { - } - } - /** - */ - static public class RandomGenReducer implements Reducer { - public void configure(JobConf job) { - } - - public void reduce(RecInt key, - Iterator it, - OutputCollector out, - Reporter reporter) throws IOException { - int keyint = key.getData(); - while (it.hasNext()) { - String val = it.next().getData(); - out.collect(new RecInt(Integer.parseInt(val)), - new RecString("")); - } - } - public void close() { - } - } - - /** - * The RandomCheck Job does a lot of our work. It takes - * in a num/string keyspace, and transforms it into a - * key/count(int) keyspace. - * - * The map() function just emits a num/1 pair for every - * num/string input pair. - * - * The reduce() function sums up all the 1s that were - * emitted for a single key. It then emits the key/total - * pair. - * - * This is used to regenerate the random number "answer key". - * Each key here is a random number, and the count is the - * number of times the number was emitted. - */ - static public class RandomCheckMapper implements Mapper { - public void configure(JobConf job) { - } - - public void map(RecInt key, - RecString val, - OutputCollector out, - Reporter reporter) throws IOException { - int pos = key.getData(); - String str = val.getData(); - out.collect(new RecInt(pos), new RecString("1")); - } - public void close() { - } - } - /** - */ - static public class RandomCheckReducer implements Reducer { - public void configure(JobConf job) { - } - - public void reduce(RecInt key, - Iterator it, - OutputCollector out, - Reporter reporter) throws IOException { - int keyint = key.getData(); - int count = 0; - while (it.hasNext()) { - it.next(); - count++; - } - out.collect(new RecInt(keyint), new RecString(Integer.toString(count))); - } - public void close() { - } - } - - /** - * The Merge Job is a really simple one. It takes in - * an int/int key-value set, and emits the same set. - * But it merges identical keys by adding their values. - * - * Thus, the map() function is just the identity function - * and reduce() just sums. Nothing to see here! - */ - static public class MergeMapper implements Mapper { - public void configure(JobConf job) { - } - - public void map(RecInt key, - RecString val, - OutputCollector out, - Reporter reporter) throws IOException { - int keyint = key.getData(); - String valstr = val.getData(); - out.collect(new RecInt(keyint), new RecInt(Integer.parseInt(valstr))); - } - public void close() { - } - } - static public class MergeReducer implements Reducer { - public void configure(JobConf job) { - } - - public void reduce(RecInt key, - Iterator it, - OutputCollector out, - Reporter reporter) throws IOException { - int keyint = key.getData(); - int total = 0; - while (it.hasNext()) { - total += it.next().getData(); - } - out.collect(new RecInt(keyint), new RecInt(total)); - } - public void close() { - } - } - - private static int range = 10; - private static int counts = 100; - private static Random r = new Random(); - private static Configuration conf = new Configuration(); - - public void testMapred() throws Exception { - launch(); - } - - /** - * - */ - public static void launch() throws Exception { - // - // Generate distribution of ints. This is the answer key. - // - int countsToGo = counts; - int dist[] = new int[range]; - for (int i = 0; i < range; i++) { - double avgInts = (1.0 * countsToGo) / (range - i); - dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian()))); - countsToGo -= dist[i]; - } - if (countsToGo > 0) { - dist[dist.length-1] += countsToGo; - } - - // - // Write the answer key to a file. - // - FileSystem fs = FileSystem.get(conf); - Path testdir = new Path("mapred.loadtest"); - if (!fs.mkdirs(testdir)) { - throw new IOException("Mkdirs failed to create directory " + testdir.toString()); - } - - Path randomIns = new Path(testdir, "genins"); - if (!fs.mkdirs(randomIns)) { - throw new IOException("Mkdirs failed to create directory " + randomIns.toString()); - } - - Path answerkey = new Path(randomIns, "answer.key"); - SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, - answerkey, RecInt.class, RecInt.class, - CompressionType.NONE); - try { - for (int i = 0; i < range; i++) { - RecInt k = new RecInt(); - RecInt v = new RecInt(); - k.setData(i); - v.setData(dist[i]); - out.append(k, v); - } - } finally { - out.close(); - } - - // - // Now we need to generate the random numbers according to - // the above distribution. - // - // We create a lot of map tasks, each of which takes at least - // one "line" of the distribution. (That is, a certain number - // X is to be generated Y number of times.) - // - // A map task emits Y key/val pairs. The val is X. The key - // is a randomly-generated number. - // - // The reduce task gets its input sorted by key. That is, sorted - // in random order. It then emits a single line of text that - // for the given values. It does not emit the key. - // - // Because there's just one reduce task, we emit a single big - // file of random numbers. - // - Path randomOuts = new Path(testdir, "genouts"); - fs.delete(randomOuts, true); - - - JobConf genJob = new JobConf(conf, TestRecordMR.class); - FileInputFormat.setInputPaths(genJob, randomIns); - genJob.setInputFormat(SequenceFileInputFormat.class); - genJob.setMapperClass(RandomGenMapper.class); - - FileOutputFormat.setOutputPath(genJob, randomOuts); - genJob.setOutputKeyClass(RecInt.class); - genJob.setOutputValueClass(RecString.class); - genJob.setOutputFormat(SequenceFileOutputFormat.class); - genJob.setReducerClass(RandomGenReducer.class); - genJob.setNumReduceTasks(1); - - JobClient.runJob(genJob); - - // - // Next, we read the big file in and regenerate the - // original map. It's split into a number of parts. - // (That number is 'intermediateReduces'.) - // - // We have many map tasks, each of which read at least one - // of the output numbers. For each number read in, the - // map task emits a key/value pair where the key is the - // number and the value is "1". - // - // We have a single reduce task, which receives its input - // sorted by the key emitted above. For each key, there will - // be a certain number of "1" values. The reduce task sums - // these values to compute how many times the given key was - // emitted. - // - // The reduce task then emits a key/val pair where the key - // is the number in question, and the value is the number of - // times the key was emitted. This is the same format as the - // original answer key (except that numbers emitted zero times - // will not appear in the regenerated key.) The answer set - // is split into a number of pieces. A final MapReduce job - // will merge them. - // - // There's not really a need to go to 10 reduces here - // instead of 1. But we want to test what happens when - // you have multiple reduces at once. - // - int intermediateReduces = 10; - Path intermediateOuts = new Path(testdir, "intermediateouts"); - fs.delete(intermediateOuts, true); - JobConf checkJob = new JobConf(conf, TestRecordMR.class); - FileInputFormat.setInputPaths(checkJob, randomOuts); - checkJob.setInputFormat(SequenceFileInputFormat.class); - checkJob.setMapperClass(RandomCheckMapper.class); - - FileOutputFormat.setOutputPath(checkJob, intermediateOuts); - checkJob.setOutputKeyClass(RecInt.class); - checkJob.setOutputValueClass(RecString.class); - checkJob.setOutputFormat(SequenceFileOutputFormat.class); - checkJob.setReducerClass(RandomCheckReducer.class); - checkJob.setNumReduceTasks(intermediateReduces); - - JobClient.runJob(checkJob); - - // - // OK, now we take the output from the last job and - // merge it down to a single file. The map() and reduce() - // functions don't really do anything except reemit tuples. - // But by having a single reduce task here, we end up merging - // all the files. - // - Path finalOuts = new Path(testdir, "finalouts"); - fs.delete(finalOuts, true); - JobConf mergeJob = new JobConf(conf, TestRecordMR.class); - FileInputFormat.setInputPaths(mergeJob, intermediateOuts); - mergeJob.setInputFormat(SequenceFileInputFormat.class); - mergeJob.setMapperClass(MergeMapper.class); - - FileOutputFormat.setOutputPath(mergeJob, finalOuts); - mergeJob.setOutputKeyClass(RecInt.class); - mergeJob.setOutputValueClass(RecInt.class); - mergeJob.setOutputFormat(SequenceFileOutputFormat.class); - mergeJob.setReducerClass(MergeReducer.class); - mergeJob.setNumReduceTasks(1); - - JobClient.runJob(mergeJob); - - - // - // Finally, we compare the reconstructed answer key with the - // original one. Remember, we need to ignore zero-count items - // in the original key. - // - boolean success = true; - Path recomputedkey = new Path(finalOuts, "part-00000"); - SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf); - int totalseen = 0; - try { - RecInt key = new RecInt(); - RecInt val = new RecInt(); - for (int i = 0; i < range; i++) { - if (dist[i] == 0) { - continue; - } - if (!in.next(key, val)) { - System.err.println("Cannot read entry " + i); - success = false; - break; - } else { - if (!((key.getData() == i) && (val.getData() == dist[i]))) { - System.err.println("Mismatch! Pos=" + key.getData() + ", i=" + i + ", val=" + val.getData() + ", dist[i]=" + dist[i]); - success = false; - } - totalseen += val.getData(); - } - } - if (success) { - if (in.next(key, val)) { - System.err.println("Unnecessary lines in recomputed key!"); - success = false; - } - } - } finally { - in.close(); - } - int originalTotal = 0; - for (int i = 0; i < dist.length; i++) { - originalTotal += dist[i]; - } - System.out.println("Original sum: " + originalTotal); - System.out.println("Recomputed sum: " + totalseen); - - // - // Write to "results" whether the test succeeded or not. - // - Path resultFile = new Path(testdir, "results"); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile))); - try { - bw.write("Success=" + success + "\n"); - System.out.println("Success=" + success); - } finally { - bw.close(); - } - fs.delete(testdir, true); - } - - /** - * Launches all the tasks in order. - */ - public static void main(String[] argv) throws Exception { - if (argv.length < 2) { - System.err.println("Usage: TestRecordMR "); - System.err.println(); - System.err.println("Note: a good test will have a value that is substantially larger than the "); - return; - } - - int i = 0; - int range = Integer.parseInt(argv[i++]); - int counts = Integer.parseInt(argv[i++]); - launch(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordWritable.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordWritable.java deleted file mode 100644 index 69d532ccf49..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/record/TestRecordWritable.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.record; - -import java.io.*; -import java.util.*; -import junit.framework.TestCase; - -import org.apache.hadoop.fs.*; -import org.apache.hadoop.io.*; -import org.apache.hadoop.conf.*; -import org.apache.commons.logging.*; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.SequenceFileInputFormat; - -public class TestRecordWritable extends TestCase { - private static final Log LOG = FileInputFormat.LOG; - - private static int MAX_LENGTH = 10000; - private static Configuration conf = new Configuration(); - - public void testFormat() throws Exception { - JobConf job = new JobConf(conf); - FileSystem fs = FileSystem.getLocal(conf); - Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred"); - Path file = new Path(dir, "test.seq"); - - int seed = new Random().nextInt(); - //LOG.info("seed = "+seed); - Random random = new Random(seed); - - fs.delete(dir, true); - - FileInputFormat.setInputPaths(job, dir); - - // for a variety of lengths - for (int length = 0; length < MAX_LENGTH; - length+= random.nextInt(MAX_LENGTH/10)+1) { - - // create a file with length entries - SequenceFile.Writer writer = - new SequenceFile.Writer(fs, conf, file, - RecInt.class, RecBuffer.class); - try { - for (int i = 0; i < length; i++) { - RecInt key = new RecInt(); - key.setData(i); - byte[] data = new byte[random.nextInt(10)]; - random.nextBytes(data); - RecBuffer value = new RecBuffer(); - value.setData(new Buffer(data)); - writer.append(key, value); - } - } finally { - writer.close(); - } - - // try splitting the file in a variety of sizes - InputFormat format = - new SequenceFileInputFormat(); - RecInt key = new RecInt(); - RecBuffer value = new RecBuffer(); - for (int i = 0; i < 3; i++) { - int numSplits = - random.nextInt(MAX_LENGTH/(SequenceFile.SYNC_INTERVAL/20))+1; - InputSplit[] splits = format.getSplits(job, numSplits); - - // check each split - BitSet bits = new BitSet(length); - for (int j = 0; j < splits.length; j++) { - RecordReader reader = - format.getRecordReader(splits[j], job, Reporter.NULL); - try { - int count = 0; - while (reader.next(key, value)) { - assertFalse("Key in multiple partitions.", bits.get(key.getData())); - bits.set(key.getData()); - count++; - } - } finally { - reader.close(); - } - } - assertEquals("Some keys in no partition.", length, bits.cardinality()); - } - - } - } - - public static void main(String[] args) throws Exception { - new TestRecordWritable().testFormat(); - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java deleted file mode 100644 index 8b45220332b..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.security; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.tools.DFSAdmin; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.tools.MRAdmin; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.security.authorize.ProxyUsers; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestMapredGroupMappingServiceRefresh { - private MiniDFSCluster cluster; - JobConf config; - private static long groupRefreshTimeoutSec = 2; - private String tempResource = null; - private static final Log LOG = LogFactory - .getLog(TestMapredGroupMappingServiceRefresh.class); - - public static class MockUnixGroupsMapping implements GroupMappingServiceProvider { - private int i=0; - - @Override - public List getGroups(String user) throws IOException { - String g1 = user + (10 * i + 1); - String g2 = user + (10 * i + 2); - List l = new ArrayList(2); - l.add(g1); - l.add(g2); - i++; - return l; - } - - @Override - public void cacheGroupsRefresh() throws IOException { - } - - @Override - public void cacheGroupsAdd(List groups) throws IOException { - } - } - - @Before - public void setUp() throws Exception { - config = new JobConf(new Configuration()); - - config.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, - TestMapredGroupMappingServiceRefresh.MockUnixGroupsMapping.class, - GroupMappingServiceProvider.class); - config.setLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, - groupRefreshTimeoutSec); - - LOG.info("GROUP MAPPING class name=" + - config.getClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, - ShellBasedUnixGroupsMapping.class,GroupMappingServiceProvider.class). - getName()); - - Groups.getUserToGroupsMappingService(config); - String namenodeUrl = "hdfs://localhost:" + "0"; - FileSystem.setDefaultUri(config, namenodeUrl); - - cluster = new MiniDFSCluster(0, config, 1, true, true, true, null, null, - null, null); - cluster.waitActive(); - URI uri = cluster.getURI(0); - - MiniMRCluster miniMRCluster = new MiniMRCluster(0, uri.toString() , - 3, null, null, config); - - config.set(JTConfig.JT_IPC_ADDRESS, "localhost:"+miniMRCluster.getJobTrackerPort()); - ProxyUsers.refreshSuperUserGroupsConfiguration(config); - } - - @After - public void tearDown() throws Exception { - if(cluster!=null) { - cluster.shutdown(); - } - if(tempResource!=null) { - File f = new File(tempResource); - f.delete(); - } - } - - @Test - public void testGroupMappingRefresh() throws Exception { - MRAdmin admin = new MRAdmin(config); - String [] args = new String[] { "-refreshUserToGroupsMappings" }; - - Groups groups = Groups.getUserToGroupsMappingService(config); - String user = UserGroupInformation.getLoginUser().getShortUserName(); - System.out.println("first attempt:"); - List g1 = groups.getGroups(user); - String [] str_groups = new String [g1.size()]; - g1.toArray(str_groups); - System.out.println(Arrays.toString(str_groups)); - - System.out.println("second attempt, should be same:"); - List g2 = groups.getGroups(user); - g2.toArray(str_groups); - System.out.println(Arrays.toString(str_groups)); - for(int i=0; i g3 = groups.getGroups(user); - g3.toArray(str_groups); - System.out.println(Arrays.toString(str_groups)); - for(int i=0; i g4 = groups.getGroups(user); - g4.toArray(str_groups); - System.out.println(Arrays.toString(str_groups)); - for(int i=0; i"+ - "" + keyGroup + ""+groups+"" + - "" + keyHosts + ""+hosts+"" + - ""; - PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource)); - writer.println(newResource); - writer.close(); - - Configuration.addDefaultResource(rsrcName); - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/HadoopPolicyProvider.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/HadoopPolicyProvider.java deleted file mode 100644 index 0e3cf4f791e..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/HadoopPolicyProvider.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.security.authorize; - -import org.apache.hadoop.hdfs.HDFSPolicyProvider; -import org.apache.hadoop.mapred.MapReducePolicyProvider; - -public class HadoopPolicyProvider extends PolicyProvider { - - @Override - public Service[] getServices() { - Service[] hdfsServices = new HDFSPolicyProvider().getServices(); - Service[] mrServices = new MapReducePolicyProvider().getServices(); - - Service[] hadoopServices = - new Service[hdfsServices.length + mrServices.length]; - System.arraycopy(hdfsServices, 0, hadoopServices, 0, hdfsServices.length); - System.arraycopy(mrServices, 0, hadoopServices, hdfsServices.length, - mrServices.length); - - return hadoopServices; - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java deleted file mode 100644 index 38b0aee9ecb..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.security.authorize; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HDFSPolicyProvider; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; -import org.apache.hadoop.hdfs.tools.DFSAdmin; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.TestMiniMRWithDFS; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; - -import junit.framework.TestCase; - -public class TestServiceLevelAuthorization extends TestCase { - public void testServiceLevelAuthorization() throws Exception { - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - try { - final int slaves = 4; - - // Turn on service-level authorization - Configuration conf = new Configuration(); - conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, - HadoopPolicyProvider.class, PolicyProvider.class); - conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, - true); - - // Start the mini clusters - dfs = new MiniDFSCluster(conf, slaves, true, null); - - // Ensure that the protocols authorized on the name node are only the HDFS protocols. - Set> protocolsWithAcls = NameNodeAdapter.getRpcServer(dfs.getNameNode()) - .getServiceAuthorizationManager().getProtocolsWithAcls(); - Service[] hdfsServices = new HDFSPolicyProvider().getServices(); - for (Service service : hdfsServices) { - if (!protocolsWithAcls.contains(service.getProtocol())) - fail("service authorization manager has no entry for protocol " + service.getProtocol()); - } - if (hdfsServices.length != protocolsWithAcls.size()) - fail("there should be an entry for every HDFS service in the protocols with ACLs map"); - - fileSys = dfs.getFileSystem(); - JobConf mrConf = new JobConf(conf); - mr = new MiniMRCluster(slaves, fileSys.getUri().toString(), 1, - null, null, mrConf); - - // Ensure that the protocols configured for the name node did not change - // when the MR cluster was started. - protocolsWithAcls = NameNodeAdapter.getRpcServer(dfs.getNameNode()) - .getServiceAuthorizationManager().getProtocolsWithAcls(); - hdfsServices = new HDFSPolicyProvider().getServices(); - for (Service service : hdfsServices) { - if (!protocolsWithAcls.contains(service.getProtocol())) - fail("service authorization manager has no entry for protocol " + service.getProtocol()); - } - if (hdfsServices.length != protocolsWithAcls.size()) - fail("there should be an entry for every HDFS service in the protocols with ACLs map"); - - // make cleanup inline sothat validation of existence of these directories - // can be done - mr.setInlineCleanupThreads(); - - // Run examples - TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf)); - TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf)); - } finally { - if (dfs != null) { dfs.shutdown(); } - if (mr != null) { mr.shutdown(); - } - } - } - - private static final String DUMMY_ACL = "nouser nogroup"; - private static final String UNKNOWN_USER = "dev,null"; - - private void rewriteHadoopPolicyFile(File policyFile) throws IOException { - FileWriter fos = new FileWriter(policyFile); - PolicyProvider policyProvider = new HDFSPolicyProvider(); - fos.write("\n"); - for (Service service : policyProvider.getServices()) { - String key = service.getServiceKey(); - String value ="*"; - if (key.equals("security.refresh.policy.protocol.acl")) { - value = DUMMY_ACL; - } - fos.write(""+ key + "" + value + - "\n"); - System.err.println(""+ key + "" + value + - "\n"); - } - fos.write("\n"); - fos.close(); - } - - private void refreshPolicy(Configuration conf) throws IOException { - DFSAdmin dfsAdmin = new DFSAdmin(conf); - dfsAdmin.refreshServiceAcl(); - } - - public void testRefresh() throws Exception { - MiniDFSCluster dfs = null; - try { - final int slaves = 4; - - // Turn on service-level authorization - final Configuration conf = new Configuration(); - conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, - HDFSPolicyProvider.class, PolicyProvider.class); - conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, - true); - - // Start the mini dfs cluster - dfs = new MiniDFSCluster(conf, slaves, true, null); - - // Refresh the service level authorization policy - refreshPolicy(conf); - - // Simulate an 'edit' of hadoop-policy.xml - String confDir = System.getProperty("test.build.extraconf", - "build/test/extraconf"); - String HADOOP_POLICY_FILE = System.getProperty("hadoop.policy.file"); - File policyFile = new File(confDir, HADOOP_POLICY_FILE); - String policyFileCopy = HADOOP_POLICY_FILE + ".orig"; - FileUtil.copy(policyFile, FileSystem.getLocal(conf), // first save original - new Path(confDir, policyFileCopy), false, conf); - rewriteHadoopPolicyFile( // rewrite the file - new File(confDir, HADOOP_POLICY_FILE)); - - // Refresh the service level authorization policy - refreshPolicy(conf); - - // Refresh the service level authorization policy once again, - // this time it should fail! - try { - // Note: hadoop-policy.xml for tests has - // security.refresh.policy.protocol.acl = ${user.name} - UserGroupInformation unknownUser = - UserGroupInformation.createRemoteUser("unknown"); - unknownUser.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException { - refreshPolicy(conf); - return null; - } - }); - fail("Refresh of NameNode's policy file cannot be successful!"); - } catch (Exception re) { - System.out.println("Good, refresh worked... refresh failed with: " + - StringUtils.stringifyException(re)); - } finally { - // Reset to original hadoop-policy.xml - FileUtil.fullyDelete(new File(confDir, - HADOOP_POLICY_FILE)); - FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, HADOOP_POLICY_FILE)); - } - } finally { - if (dfs != null) { dfs.shutdown(); } - } - } - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/TestHarFileSystem.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/TestHarFileSystem.java deleted file mode 100644 index a49bcf3e6fc..00000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/TestHarFileSystem.java +++ /dev/null @@ -1,427 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.tools; - -import java.io.IOException; -import java.net.URI; -import java.util.Iterator; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.*; -import org.apache.hadoop.tools.HadoopArchives; -import org.apache.hadoop.util.ToolRunner; -import org.mortbay.log.Log; - -/** - * test the har file system - * create a har filesystem - * run fs commands - * and then run a map reduce job - */ -public class TestHarFileSystem extends TestCase { - private Path inputPath, inputrelPath; - private MiniDFSCluster dfscluster; - private MiniMRCluster mapred; - private FileSystem fs; - private Path filea, fileb, filec; - private Path archivePath; - - protected void setUp() throws Exception { - super.setUp(); - dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null); - fs = dfscluster.getFileSystem(); - mapred = new MiniMRCluster(2, fs.getUri().toString(), 1); - inputPath = new Path(fs.getHomeDirectory(), "test"); - inputrelPath = new Path(fs.getHomeDirectory().toUri(). - getPath().substring(1), "test"); - filea = new Path(inputPath,"a"); - fileb = new Path(inputPath,"b"); - filec = new Path(inputPath,"c c"); - archivePath = new Path(fs.getHomeDirectory(), "tmp"); - fs.mkdirs(inputPath); - FSDataOutputStream out = fs.create(filea); - out.write("a".getBytes()); - out.close(); - out = fs.create(fileb); - out.write("b".getBytes()); - out.close(); - out = fs.create(filec); - out.write("c".getBytes()); - out.close(); - } - - protected void tearDown() throws Exception { - try { - if (mapred != null) { - mapred.shutdown(); - } - if (dfscluster != null) { - dfscluster.shutdown(); - } - } catch(Exception e) { - System.err.println(e); - } - super.tearDown(); - } - - static class TextMapperReducer implements Mapper, - Reducer { - - public void configure(JobConf conf) { - //do nothing - } - - public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException { - output.collect(value, new Text("")); - } - - public void close() throws IOException { - // do nothing - } - - public void reduce(Text key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { - while(values.hasNext()) { - values.next(); - output.collect(key, null); - } - } - } - - /* check bytes in the har output files */ - private void checkBytes(Path harPath, Configuration conf) throws IOException { - Path harFilea = new Path(harPath, "a"); - Path harFileb = new Path(harPath, "b"); - Path harFilec = new Path(harPath, "c c"); - FileSystem harFs = harFilea.getFileSystem(conf); - FSDataInputStream fin = harFs.open(harFilea); - byte[] b = new byte[4]; - int readBytes = fin.read(b); - fin.close(); - assertTrue("strings are equal ", (b[0] == "a".getBytes()[0])); - fin = harFs.open(harFileb); - fin.read(b); - fin.close(); - assertTrue("strings are equal ", (b[0] == "b".getBytes()[0])); - fin = harFs.open(harFilec); - fin.read(b); - fin.close(); - assertTrue("strings are equal ", (b[0] == "c".getBytes()[0])); - } - - private void checkProperties(Path harPath, Configuration conf) throws IOException { - Path harFilea = new Path(harPath, "a"); - Path harFileb = new Path(harPath, "b"); - Path harFilec = new Path(harPath, "c c"); - FileSystem harFs = harFilea.getFileSystem(conf); - - Path nonharFilea = new Path(inputPath, "a"); - Path nonharFileb = new Path(inputPath, "b"); - Path nonharFilec = new Path(inputPath, "c c"); - FileSystem nonharFs = nonharFilea.getFileSystem(conf); - - assertEquals("Modification times do not match for a", - harFs.getFileStatus(harFilea).getModificationTime(), - nonharFs.getFileStatus(nonharFilea).getModificationTime()); - - assertEquals("Modification times do not match for b", - harFs.getFileStatus(harFileb).getModificationTime(), - nonharFs.getFileStatus(nonharFileb).getModificationTime()); - - assertEquals("Modification times do not match for c", - harFs.getFileStatus(harFilec).getModificationTime(), - nonharFs.getFileStatus(nonharFilec).getModificationTime()); - } - - /** - * check if the block size of the part files is what we had specified - */ - private void checkBlockSize(FileSystem fs, Path finalPath, long blockSize) throws IOException { - FileStatus[] statuses = fs.globStatus(new Path(finalPath, "part-*")); - for (FileStatus status: statuses) { - assertTrue(status.getBlockSize() == blockSize); - } - } - - // test archives with a -p option - public void testRelativeArchives() throws Exception { - fs.delete(archivePath, true); - Configuration conf = mapred.createJobConf(); - HadoopArchives har = new HadoopArchives(conf); - - { - String[] args = new String[6]; - args[0] = "-archiveName"; - args[1] = "foo1.har"; - args[2] = "-p"; - args[3] = fs.getHomeDirectory().toString(); - args[4] = "test"; - args[5] = archivePath.toString(); - int ret = ToolRunner.run(har, args); - assertTrue("failed test", ret == 0); - Path finalPath = new Path(archivePath, "foo1.har"); - Path fsPath = new Path(inputPath.toUri().getPath()); - Path filePath = new Path(finalPath, "test"); - // make it a har path - Path harPath = new Path("har://" + filePath.toUri().getPath()); - assertTrue(fs.exists(new Path(finalPath, "_index"))); - assertTrue(fs.exists(new Path(finalPath, "_masterindex"))); - /*check for existence of only 1 part file, since part file size == 2GB */ - assertTrue(fs.exists(new Path(finalPath, "part-0"))); - assertTrue(!fs.exists(new Path(finalPath, "part-1"))); - assertTrue(!fs.exists(new Path(finalPath, "part-2"))); - assertTrue(!fs.exists(new Path(finalPath, "_logs"))); - FileStatus[] statuses = fs.listStatus(finalPath); - args = new String[2]; - args[0] = "-ls"; - args[1] = harPath.toString(); - FsShell shell = new FsShell(conf); - ret = ToolRunner.run(shell, args); - // fileb and filec - assertTrue(ret == 0); - checkBytes(harPath, conf); - checkProperties(harPath, conf); - /* check block size for path files */ - checkBlockSize(fs, finalPath, 512 * 1024 * 1024l); - } - - /** now try with different block size and part file size **/ - { - String[] args = new String[8]; - args[0] = "-Dhar.block.size=512"; - args[1] = "-Dhar.partfile.size=1"; - args[2] = "-archiveName"; - args[3] = "foo.har"; - args[4] = "-p"; - args[5] = fs.getHomeDirectory().toString(); - args[6] = "test"; - args[7] = archivePath.toString(); - int ret = ToolRunner.run(har, args); - assertTrue("failed test", ret == 0); - Path finalPath = new Path(archivePath, "foo.har"); - Path fsPath = new Path(inputPath.toUri().getPath()); - Path filePath = new Path(finalPath, "test"); - // make it a har path - Path harPath = new Path("har://" + filePath.toUri().getPath()); - assertTrue(fs.exists(new Path(finalPath, "_index"))); - assertTrue(fs.exists(new Path(finalPath, "_masterindex"))); - /*check for existence of 3 part files, since part file size == 1 */ - assertTrue(fs.exists(new Path(finalPath, "part-0"))); - assertTrue(fs.exists(new Path(finalPath, "part-1"))); - assertTrue(fs.exists(new Path(finalPath, "part-2"))); - assertTrue(!fs.exists(new Path(finalPath, "_logs"))); - FileStatus[] statuses = fs.listStatus(finalPath); - args = new String[2]; - args[0] = "-ls"; - args[1] = harPath.toString(); - FsShell shell = new FsShell(conf); - ret = ToolRunner.run(shell, args); - // fileb and filec - assertTrue(ret == 0); - checkBytes(harPath, conf); - checkProperties(harPath, conf); - checkBlockSize(fs, finalPath, 512); - } - } - - public void testArchivesWithMapred() throws Exception { - fs.delete(archivePath, true); - Configuration conf = mapred.createJobConf(); - HadoopArchives har = new HadoopArchives(conf); - String[] args = new String[4]; - - //check for destination not specfied - args[0] = "-archiveName"; - args[1] = "foo.har"; - args[2] = "-p"; - args[3] = "/"; - int ret = ToolRunner.run(har, args); - assertTrue(ret != 0); - args = new String[6]; - //check for wrong archiveName - args[0] = "-archiveName"; - args[1] = "/d/foo.har"; - args[2] = "-p"; - args[3] = "/"; - args[4] = inputrelPath.toString(); - args[5] = archivePath.toString(); - ret = ToolRunner.run(har, args); - assertTrue(ret != 0); - // se if dest is a file - args[1] = "foo.har"; - args[5] = filec.toString(); - ret = ToolRunner.run(har, args); - assertTrue(ret != 0); - //this is a valid run - args[0] = "-archiveName"; - args[1] = "foo.har"; - args[2] = "-p"; - args[3] = "/"; - args[4] = inputrelPath.toString(); - args[5] = archivePath.toString(); - ret = ToolRunner.run(har, args); - //checl for the existenece of the archive - assertTrue(ret == 0); - ///try running it again. it should not - // override the directory - ret = ToolRunner.run(har, args); - assertTrue(ret != 0); - Path finalPath = new Path(archivePath, "foo.har"); - Path fsPath = new Path(inputPath.toUri().getPath()); - String relative = fsPath.toString().substring(1); - Path filePath = new Path(finalPath, relative); - //make it a har path - URI uri = fs.getUri(); - Path harPath = new Path("har://" + "hdfs-" + uri.getHost() +":" + - uri.getPort() + filePath.toUri().getPath()); - assertTrue(fs.exists(new Path(finalPath, "_index"))); - assertTrue(fs.exists(new Path(finalPath, "_masterindex"))); - assertTrue(!fs.exists(new Path(finalPath, "_logs"))); - //creation tested - //check if the archive is same - // do ls and cat on all the files - - FsShell shell = new FsShell(conf); - args = new String[2]; - args[0] = "-ls"; - args[1] = harPath.toString(); - ret = ToolRunner.run(shell, args); - // ls should work. - assertTrue((ret == 0)); - //now check for contents of filea - // fileb and filec - Path harFilea = new Path(harPath, "a"); - Path harFileb = new Path(harPath, "b"); - Path harFilec = new Path(harPath, "c c"); - FileSystem harFs = harFilea.getFileSystem(conf); - FSDataInputStream fin = harFs.open(harFilea); - byte[] b = new byte[4]; - int readBytes = fin.read(b); - assertTrue("Empty read.", readBytes > 0); - fin.close(); - assertTrue("strings are equal ", (b[0] == "a".getBytes()[0])); - fin = harFs.open(harFileb); - readBytes = fin.read(b); - assertTrue("Empty read.", readBytes > 0); - fin.close(); - assertTrue("strings are equal ", (b[0] == "b".getBytes()[0])); - fin = harFs.open(harFilec); - readBytes = fin.read(b); - assertTrue("Empty read.", readBytes > 0); - fin.close(); - assertTrue("strings are equal ", (b[0] == "c".getBytes()[0])); - // ok all files match - // run a map reduce job - FileSystem fsHar = harPath.getFileSystem(conf); - FileStatus[] bla = fsHar.listStatus(harPath); - Path outdir = new Path(fs.getHomeDirectory(), "mapout"); - JobConf jobconf = mapred.createJobConf(); - FileInputFormat.addInputPath(jobconf, harPath); - jobconf.setInputFormat(TextInputFormat.class); - jobconf.setOutputFormat(TextOutputFormat.class); - FileOutputFormat.setOutputPath(jobconf, outdir); - jobconf.setMapperClass(TextMapperReducer.class); - jobconf.setMapOutputKeyClass(Text.class); - jobconf.setMapOutputValueClass(Text.class); - jobconf.setReducerClass(TextMapperReducer.class); - jobconf.setNumReduceTasks(1); - JobClient.runJob(jobconf); - args[1] = outdir.toString(); - ret = ToolRunner.run(shell, args); - - FileStatus[] status = fs.globStatus(new Path(outdir, "part*")); - Path reduceFile = status[0].getPath(); - FSDataInputStream reduceIn = fs.open(reduceFile); - b = new byte[6]; - readBytes = reduceIn.read(b); - assertTrue("Should read 6 bytes instead of "+readBytes+".", readBytes == 6); - //assuming all the 6 bytes were read. - Text readTxt = new Text(b); - assertTrue("a\nb\nc\n".equals(readTxt.toString())); - assertTrue("number of bytes left should be -1", reduceIn.read(b) == -1); - reduceIn.close(); - } - - public void testGetFileBlockLocations() throws Exception { - fs.delete(archivePath, true); - Configuration conf = mapred.createJobConf(); - HadoopArchives har = new HadoopArchives(conf); - String[] args = new String[8]; - args[0] = "-Dhar.block.size=512"; - args[1] = "-Dhar.partfile.size=1"; - args[2] = "-archiveName"; - args[3] = "foo bar.har"; - args[4] = "-p"; - args[5] = fs.getHomeDirectory().toString(); - args[6] = "test"; - args[7] = archivePath.toString(); - int ret = ToolRunner.run(har, args); - assertTrue("failed test", ret == 0); - Path finalPath = new Path(archivePath, "foo bar.har"); - Path fsPath = new Path(inputPath.toUri().getPath()); - Path filePath = new Path(finalPath, "test"); - Path filea = new Path(filePath, "a"); - // make it a har path - Path harPath = new Path("har://" + filea.toUri().getPath()); - FileSystem harFs = harPath.getFileSystem(conf); - FileStatus[] statuses = harFs.listStatus(filePath); - for (FileStatus status : statuses) { - BlockLocation[] locations = - harFs.getFileBlockLocations(status, 0, status.getLen()); - long lastOffset = 0; - assertEquals("Only one block location expected for files this small", - 1, locations.length); - assertEquals("Block location should start at offset 0", - 0, locations[0].getOffset()); - } - } - - public void testSpaces() throws Exception { - fs.delete(archivePath, true); - Configuration conf = mapred.createJobConf(); - HadoopArchives har = new HadoopArchives(conf); - String[] args = new String[6]; - args[0] = "-archiveName"; - args[1] = "foo bar.har"; - args[2] = "-p"; - args[3] = fs.getHomeDirectory().toString(); - args[4] = "test"; - args[5] = archivePath.toString(); - int ret = ToolRunner.run(har, args); - assertTrue("failed test", ret == 0); - Path finalPath = new Path(archivePath, "foo bar.har"); - Path fsPath = new Path(inputPath.toUri().getPath()); - Path filePath = new Path(finalPath, "test"); - // make it a har path - Path harPath = new Path("har://" + filePath.toUri().getPath()); - FileSystem harFs = harPath.getFileSystem(conf); - FileStatus[] statuses = harFs.listStatus(finalPath); - } -} diff --git a/hadoop-mapreduce-project/src/test/smoke-tests b/hadoop-mapreduce-project/src/test/smoke-tests deleted file mode 100644 index 44bcd791601..00000000000 --- a/hadoop-mapreduce-project/src/test/smoke-tests +++ /dev/null @@ -1,13 +0,0 @@ -**/TestMiniMRChildTask.java -**/TestMiniMRBringup.java -**/TestMiniMRDFSCaching.java -**/TestMiniMRDFSSort.java -**/TestMiniMRWithDFSWithDistinctUsers.java -**/TestLocalMRNotification.java -**/TestMapReduceLocal.java -**/TestReduceFetch.java -**/TestReduceTask.java -**/TestJobTrackerRestart.java -**/TestJobTrackerRestartWithLostTracker.java -**/TestJobTrackerSafeMode.java -**/TestChild.java diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-minimal.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-minimal.json deleted file mode 100644 index c4076d8760f..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-minimal.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "minimum" : 12345, - "rankings" : [ { - "relativeRanking" : 0.25, - "datum" : 12345 - }, { - "relativeRanking" : 0.5, - "datum" : 2345678901 - }, { - "relativeRanking" : 0.75, - "datum" : 2345678902 - } ], - "maximum" : 23456789012, - "numberValues" : 5 -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-one-value-many-repeats.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-one-value-many-repeats.json deleted file mode 100644 index 7cb38e3cb7f..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-one-value-many-repeats.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "minimum" : 23456789012, - "rankings" : [ { - "relativeRanking" : 0.25, - "datum" : 23456789012 - }, { - "relativeRanking" : 0.5, - "datum" : 23456789012 - }, { - "relativeRanking" : 0.75, - "datum" : 23456789012 - } ], - "maximum" : 23456789012, - "numberValues" : 64 -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-only-one-value.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-only-one-value.json deleted file mode 100644 index b24345acb5e..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-only-one-value.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "minimum" : 23456789012, - "rankings" : [ { - "relativeRanking" : 0.25, - "datum" : 23456789012 - }, { - "relativeRanking" : 0.5, - "datum" : 23456789012 - }, { - "relativeRanking" : 0.75, - "datum" : 23456789012 - } ], - "maximum" : 23456789012, - "numberValues" : 1 -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-three-values.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-three-values.json deleted file mode 100644 index 7722f796185..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/gold-three-values.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "minimum" : 1, - "rankings" : [ { - "relativeRanking" : 0.25, - "datum" : 1 - }, { - "relativeRanking" : 0.5, - "datum" : 1 - }, { - "relativeRanking" : 0.75, - "datum" : 23456789012 - } ], - "maximum" : 234567890123, - "numberValues" : 3 -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-minimal.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-minimal.json deleted file mode 100644 index 97791213c9c..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-minimal.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "data" : - [ - 12345, - 2345678901, - 23456789012, - 2345678902, - 23456789012 - ], - "percentiles" : - [ - 25, - 50, - 75 - ], - "scale" : 100 -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-one-value-many-repeats.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-one-value-many-repeats.json deleted file mode 100644 index dba44e74345..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-one-value-many-repeats.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "data" : - [ - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012, - 23456789012 - ], - "percentiles" : - [ - 25, - 50, - 75 - ], - "scale" : 100 -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-only-one-value.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-only-one-value.json deleted file mode 100644 index f020be8b4ce..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-only-one-value.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "data" : - [ - 23456789012 - ], - "percentiles" : - [ - 25, - 50, - 75 - ], - "scale" : 100 -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-three-values.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-three-values.json deleted file mode 100644 index 5bc4e9548e1..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/histogram-tests/input-three-values.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "data" : - [ - 1, - 23456789012, - 234567890123 - ], - "percentiles" : - [ - 25, - 50, - 75 - ], - "scale" : 100 -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml deleted file mode 100644 index ee86bd628a9..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_conf.xml +++ /dev/null @@ -1,3 +0,0 @@ - -mapred.child.java.opts-Xmx1024M -Djava.io.tmpdir=./tmp - diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_40864_job_name-DAILY%2F20100210%5D.gz deleted file mode 100644 index 0dcdf0e991a994f96fd4bd7bbd12778bc91cd19d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2509 zcmV;;2{QH{iwFqg%Ck!V18rqzVPkA{b97~LE^TFJVPkJ{a4utSZC^1mHZ?XeHZd?a zI5;w2YHwm+GB7bPFfuSWFgP$VUoX`vIgo5ZifOX|BWq(@7TyO2VyH!z#)QnHlZ^;U;0Xz198i zJx+iDHi0<23^PsJRTt;j$A04ToF|UsjvG4Mzgvc>>D}uRDsk1He)aCTU-ge#^SbmO z@7~5d;(5#~@Gr6jUXg?nz1*#LTGyq!rf!*6JwtC^QQN6Aaj%+nt^K(cS3~?4?ur{a zp);t4zB3ITJ%3W-`ew_xEdapb6>cWXSDf4yq4hZw~*PxKEDr;Q2Es@g`tA^W9hF_rb};Ooq3MQZeZDSrb)!>nXRO+;_q&bTt)XKi5v9agx>ax6oqEr_ zF5%@*qt)y;T2La!eod=x_2FmFX!WR1%L%@6R^9wWy@v23!L2>9dHN42MV7Ypxd-0O zmeq$Qph08%UOi1b$XUft2l9^yK32dN-wdD z0Ts)rH$JrtJD#5~=N`VgzCHDZuD}b5k}zXd6XwI%dvM0jUJ%w0mIqZn`VV~&33R5! z-C9=9?&*dF=<~e9UjPnKz2zJ*`wf^HW{JDex|+E)t1#gSf?&JVO)G|`CQC3l0)G_v z3rzb=7)97yEpTdR-CwRtfg8?eC9a|OyB4s>b?MKt=RXaq&dhmub_dls?4Xe-?gmvl zLI2V4=2M8^N*Un0GTh<(!R2mz;EwN}|Fv>gHhWsT-D<*YZ|FwTtdz~!Xgqd@oZ&)8 zhC!vwSHArJDD#~OL4_jtT0_mqM9Z-l{cacT5GzRh21vj@iNO|OL0%}~WiH7ud-!-N zVO|z@O{IUP)RN?EwJ@}fuo728g1Tqkvf+TT;~*#DKx1mvIbNh?nUL73hcnh=#v+-x zqH!|L6gJx8Q|KELBue-)==)44uuzYHWO97+?)lz>8w9{MndQ}sx|#>BU!xKZk3DBt z3n#Oh^YGvKC>Vt!Z(93N7~1!6gT8Lb!&+3?{XL`_5!b#sQ=mV`9?mDTfEL392!omf zaMot?dmB17iQ9Q;p{aLUwHVb+6Ppg;f18YbGa9>Qr2ZDn3=Er4c2~Y_-kA2s&sNI> zZmu^gKNcV&%at#8;HTsAlb^&-@#;~E{su1ETI6AIgqR!&`5j>AL>B9kyuE?OlW zre`ia$7f9OnKL|}Exq$6PVRZ0Jrb5EnW%`+q*Vk}5mfIzsA4b^1j15lF^cJo8o}SU z;w^|Y0?(_IE&z*Uk}_}mH#a~GFQYh}A>yS{t{w&Q4o=-`5UtF_Ry)^$GML~a@pl|x z3!eMM(EB~0t%Y68a!}-`?6I5A8k2=K{WxR2&4%3BH)XFf3<)=mV{wk^6~d9e2FTLl(#D94#0v{ z&PkHWi>Y!Bu8Kkm`AQ0+yPV#voWlYWyT|64*e=LDFDvWRxdIV8wJ9yv!OQO2j0a|d9-DCY?1UMhQ&_#)t6E#O~oD^}`4Fmxl{fOss)D&&%^Lb`k>beHKa`L2q&?z%t}tejT9AAkj` zd>7-qnr%YP!hE~p*X3ymI>xplky5?%IvXu=9M{&ip>5ue$k5L z_wl0j@ULtcq&zT+ctR;(1bh+jMZg~;;3r7|-)Oh$-IU`Xm!ZA?EA&+>jD28R^2RBT z3`7U$Ww&h)Ju3YjwRnNSoi{8-3c-CjR3a%-I6es%in zs$~5D+MBx8UKyd)@0AG@(N8tkk-5N>dYMNH;S(P%bi{AHv8WOzHAct|eN`N9biv0P zv7A+uL*@XXEp*O>bH?E^Ts-26I%7m)MM4R?}@5y(A;N^}{WwM-h9OdY?ZkcumhNzj5Xs9F=AyvV0l>Mye z#QXS&{1;hKDYu>Z6#6k-tSWL;;X$wC&3W~9TmQ}8Xm?f7@vd??epJ&O3EPC~F XawmSXx;R(*uKx5FX0lqy+-Cp)^q}LX diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml deleted file mode 100644 index 1a45cb9613c..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_conf.xml +++ /dev/null @@ -1,3 +0,0 @@ - -mapred.child.java.opts-Xmx1024m -Djava.net.preferIPv4Stack=true - diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/counters-format-test-logs/megacluster.megacorp.com_1265616107882_job_201002080801_50510_job_name-DAILY%2F20100208%5D.gz deleted file mode 100644 index 15a89c6d41cfb33b4fefc11415f2fdecd18f6124..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2502 zcmV;%2|4y3iwFpU%d<-W18rqzVPkA{b97~LE^TFJVPkJ{a4utSZC^1mHZ?XeHZd?a zI5;w2YHwm+GB7bPFfuSWFgP$VUo|i_F)&|hZ(?6=VQpnCL_tYRStT+?GB7bPFfuSW zB{f6r-(0wx4ZRD>$-H;)J)^5r)$lt&~|1|ovUVDZGWw;ihbj*xRD*$ zgKFT}GylQ$rX{X#wDjxJqvJe-N2C9_qnehfh7}?#QduHI7I|KY95uA=&2{N%^x%K0 z)u#4%Hk>T{!11nTGc>3^x|7kMI&~iHM{m6RZRmRQL3QX(!F=vHBmZ}>pVpr2k?YR4 zuO)XdygaD>Hl1XL!|Lg(rmN=H>rz+KJN0(PqV0OW+qm87yb>CfI8!t0ZL3r78P_Ge z`lz>>{YDF}(AckG*3CYA?CGsu=+m_QXU?n}UqY_|d`NI>O)WnBLrRgQWq$2}ccW$Y zp$e!_-+dZkg9NzMYpW*s3jK8Kc|8CK=RrapjQ;DU+3NI6D=cEimF_EL@hPn&NCZ?& zz25lJ(ye%e!l)ZQJDxRjM~;9+BpetspC<0mo;P2*jh%r-1tMUGxLeIIV{~dr!O%MbKQcsj71&(#K`}1|lcY?*d#5J^j*91DbF8xtE9df zVg{jHDFcL8x-(h~9qz^h68YhJ&y~Bf(No*)Rucw&L(`i^rEJW{lZi9pbO#zT@+)P& z^6h^o8TJe)0*pv}yRK#=q*Y(^ezyxxh!rG$3nU<+#F&e)P+pi}b39S_56kg@;=%I3 zfcnQvSuDN`MMN>lUGk7mR!~r8P_S@vP^68nfup=7v77;vWJ-%V%s7772#rtSc1kH0 zc)k3X@C$2rJ1=D%W9{y)B_Q%24oz+&l z(YlAH^Sq-TYf%yBw;+^|xc1$i0V&$nXfd7p;bj;AfnT!$%i4T#Z$XQuaXl|JG_`K4 z7DKviVudaL?>3L$j3dRW7nINKVLvZk}BWs$~S7e zwY>2w2t>2htt4JU2oYfTKgxjFCWJAh5q^W}cY87hC;+z*+o&~=Usl38y7AKRQ&n_V zR%i1?(6c=l1aUfw?y0lw0+q~mh!2s8tV4A7)wbt8T34dmKbC1yckI25%VZfgFYQ4D zt~>G9n5BTcV+7gV^}?Qn5%V`R7~Fx;;M=E;I~qTXol)iB{bh@eml+MM^I7e-EUj<$ z`sRz;Xt>k+@yy}eMF1#q-iGM7D+AqGYIn;Xs^`YDEz|g+9@a5vUFWh9Q6R)sF5Tb% z9yp;TQG{*(8>X@Mr6rXj!&UEg8oTP;PJ?&bvDLX81HZ%cLWeDIbW7#@q^Ff-q$n^BauZWVI*I)N;*!@TzZbrnBp^Mcs^Tt=g*wn z^UOUG)+!lzNT!Oc6hT!4)yEI2x042`Yu$wKdo>a-OG;jiBx9alnnv1P>;4jrwPb3N zcv+UVBvMvLN?c3VH9c0waR?>!ccriyXEqtT>>YUaqJ*im_t_I79eRV4_Ux&kh)VLa zzW_Lrvy*Y#zqx^o5D$e`a}AZKNRk92*?^8D-j%VGHi9UUaEi#5L;$On73p1#?Zw3W zPcHr?X9C?wfpNb4(*=wLru-S3Z{*?it)y#25H2;{Y7vR=$)^VmqE)%rYVR6g1`~Xu z+#Sc*GROU_YyBRitfgI?;T}DAG4~^nIaB@8$10qbNq6bQe_A{|Oq^Bb#dx#GuA60< zT`xGBO29d@`W8mJZP_V+b1a1=uTW)I%o37$Q7Qty2>2r4-z?xyrDB9!Dn{5N>Y^&s zMI91QenF2YIj^WY0uDw|C-GPi#WYa|cPSCl_?OBGNfveCmZ$Zus|--Tm8wIOGQE+F z&9jm`R-}UDGnkxeMJ7ARqgNq$L=ij5qm1uNUxDON4po=@@{B>%VWc2Z1bh+jMZmvV zz@JLSFz&?l4s1yVRToN?eao~8&An7dF_F)!>W+YeRn=ih=A~3s2Y0EM!oNfy;a8!O zR2|_r_tGDR`pwqrcqWoxnrEK66Izh`UwP`z=OFnDoVp*sZH+uSJerMAf5RI>=AIVsyc$OlD<|)5TQ7Yf2kDe!Nj#X<>meB zVW8h^rB2376ZhCWGsR<6Q2bw*;*lhO2#P;`<359?6L?DV?pQ1Wz6kgt;4eqhF`kC9 zDwV3~L@FFusgsD9SJNSxoLSQy0SBw5ld&j}R80qW<+PPLg$DxO(sUxAgcJ&zju*n; za4jp=Z?;rN6sDPbY@V6rX)$xpVnSY><;m;J+$mvad6JFg$t$ot$sy~Kzb45b>lC8M zs0jEX;ERC2?0}yp&Ep#FR=t~Ys^BWL4}V{IT?=Cy*uI96wj?Mif)MVodv*Ev=3ubU zG*!a$0nVcSh7-r0y_drUQ^r3FloCP0MISGi+U%V(Nq71Q?$?KzuCHvKXL?gt+iN4V z{=7B;*Ws7@Hj%l&)OwkR(cm*5Msvb%2V<%bro}cw_AH+j2jX1tfjBIu+vJctKxj+7 zbK#tEyd0+|+z-hZ(HU0HdA2PPEBmYyEhN&{UmD&XljmC-hDmgeZJbQ8eeR8%aVTb1 zG$29LS-_%R>O`9Il8TxhSh;CszxLlDoK}M&Q7GL$!2tbdVJ#gi>#=W z+xB7x?HDar7r9X3L93%B`I~yXt^I0k)w?cex`936g7Fj{+T59iP$+8}w`(clR z@$8YaAE3!_)pxw_;b(v120k|!l&Ah+P+0<+z~arNu&Lw&2h$CJ>+ zPA6ZL1u9A(_)COzEuSt^*`1F0FGD3}&nc(Zlb8@OSN8dc`tw!tA&XwUoZN}utuD@$ QzNZggdGb1igbb960qa$#d-E^2dcZUF6FTW{Mq7JkpK zF#Mdvyo%HU^N?vWi_Ru(k)*S`Xp2CJmBv%wR+ft?2L10#T}??8DVdhzrfPx4Ai`8dvo&c`tA8iG|XJIpPZk)zWnbf84-d{gOkD7 z3%!pYPs6~QuO1h{EE#9!EpNTmmsK>-hr!>XNi?oPF9>fZ$?y<4ltY3r;1C}t1fgI_ zCmwxN%=~V42jV|NiRJd&GMVw!*}JUW3qyal3=0zw&C|t$7fu%Qs5xtln!Rs+kkSsB zBfbyB(LX%;7_HY}IA4r?0tn}#!>PV)O_@O%)no*r(rze&HGocE^cl%ed?8l)ImAP<77TrJymE3WOy^o)B5po z>i-s#`s0Z31a5j4L0Hq3u73I~)8BND^A2n#|1^GF{rWBRSJ%E54-7G5)!u&#)err7 zN%G@l>KB+*eG)5#aPxhI*1n!E*WtA<7JMA*0|avK+jS_q<1vznxun{y<`&QHelqvV z;{)k=j@J)K|38DpdbujiUh0?lD+D*mGt1{#A*f7Jex%FEbn1_5zke(}ADS`}_kACH zO5eI=;6M6-kaL;7XO;J_=0#(X;)DD>Lx9ny@7ed`Erxg44u?n4Tp{B5_>j1bVPdm* zb#Ejh=>r;0=0)wxUW=qd@zjQklEUWNp7yACB1pXny>PuMfXk7z7WGihT0=SPp}akm z6XF`7oFNAG59RO<>YyA!rzR zSR85SU3n@o%*)0e(70n~zKTKs+NFSQy%mOO=0}vgl^ppi-b#(LlH+;RS83p?Boj>) zsc9@Kd@a=6feCc6==H_<`&Spe52IOZN`W5U5#=s+b?}}OIu5Q!dWnjI_Mjg$2~rK9 zk1?%BeFPMpvS%ap?;ZCI&J8l4e!0I4`Utuv)JGc7M?1iLJ){X8Bv=V)`P3_jhSZ3b zD?t^}kQvbo!I!`;(i*$e!B>x62JE6Pkb(`h-Z&gOqzQI8jw`+2b=W1SDA^jj^ii1! zy9m)>7gBV}*oDZ;#V*V{#;)z+G5{FtFfOC8pb)CKjADSb%D>~dEa>C1mmQbsZM4G? zuCdYCE85sb=T@{E8{Hy7nkqHgmyO0C`sAXGjdtC}k!KSdoe3BUdYNuBMo{%j&4;%^FKiAF5p}e?(>Y;;E=KecRF1nfdJ!A+ zLN2REFQ|ex+4G`@o7pwRji#^>Qe)hxJ$ckQdPO=|<9QW5vg)iVdsUedD9+WG(JS4n zT7*x2&}^$J+D&V-RW;MOV>cJK@B7m>nza_L7S-b!!T`f(IEY~7cl8Ocy&t#PQNAlW zX{Sc6lRDr!sX?w%yNY95zpD`1HU}wnzU$TH#o3!3!_Rl^-F|r&ES7!{P7$eS@BH9OTJX(xD3fe7vWDh*qc=t> z{|-lT+biM3l{mF$*#)q$@i950>AEa#~CEsy~A)3dQ%N)xAd9g4~N6`YG6Mwo&B?`MCUC9Ps%{fF?}U;6X0HxGYb+{>CMzR-ghef7AG zWS=Ft^y>V#qC-9N{+i6zv#8BQ2xEo-b~q)BQ4TUcekOD!B|526Wr^9qn}3RBqm1t(gY$cjoF4bvQ~eclGQ#d)-ex!9wiB+g+D*7M!XYHo z-QI9FVz(5pG2V^XP3D^(Wg&B7J)6PfKJCX*EPA>0gWF)@uWlDN5tb)evJa7JA4mU` zT>wG;Ti`up=}7^$aeuZ*S(Kf${%`HC<6pC*zQc#cw<;$>0fk^UkCjwox*|mbjZ872O zO2>$zOz`JmvQ5faA!O7_g7nU|Njpa?P7%@Gg@|Z4;nEd{4%5!X$J&<_Indm5mt{o? z??U-lAP~yf;*G1lPb&#QrM+%gxUbulh*sZ1m!}u6r{e%nIBI&JHAWJAM~$wz6@Z}2 z8Y3?q--U7@*MRR1=iJ#pzW2384iDM8sx<;QbGKD=^R>3vkHJ`bADv+aIA_nI=l%Bi z!6|x&?uhj8l4DG=4!X3> z(`)3{WI{%*Bv|-03^|Sg?=ivLd`g`~rPlI%lLGoBD(bYi0)71v8Krl9haVWtz%$+t zjC9aNSvsJ6$0nQlv)3OLNNUe=g4EF zkeP1@GVHjFF+&xSn2IB|HG*0Bv3(ViIs`*y0SAU)kOPOQ5iFbfvsY(G6CLfnu#T%U z$S$;71Te~X4q%ChQLFLoJTOzljVY5It9lO%L*zOp{1Q}{wyMxjeAZU@)mLcv5i2wZ zr6GXQn&9$aDl`-mwJq?g;|dLNp>^m+=>!DG$I^8h;Zf`a!c9+XTae7wwomF(9c? z)9v_5bb3D3Q6kvkRBk9ai_O8R>&gu;KA2=KAyOPs^%DXXa~X!} z?Z<Qi#hx-S4dUjF~SOYh-X{G=3)+9+D&d>%)!X5E9PKkwi2x?=3r*F60IxdU}m-w zt-Bm*X0{S-$85!|E9PKkwi2x?=3r*F60IxdU}m-wtt;kWX6}nQyg*y@^N8QFn1c{a zfm@vv6s|0!%4sNWTT8^-nCvF}8l&Ba-BP^9a5rMN6t6Mfjo5YK33n{2J2Yaq6mQF} z<98U8k{E3bc_Vg9@fu^^h+Qk56SBQQZ^UjXURUIy3434U;n0da9Q_~2jj)6?h5!I2 CZPs=G diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-sample-v20-jt-log.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-sample-v20-jt-log.gz deleted file mode 100644 index c68f42383495d85e1d61fc552a524856cb93a241..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29932 zcmZttV|1k7(>)HywylY6JDJ$FZQFJ-NhY=@nAo=MWMZ2gG~ zuByH3^hF#C0|QC}G1k;A(8|WM|Ic!OF~F&2fyz8qY?@JKy||6^ME&#m*E4>Q4+uE>8XL~U zJ)AUc3pFPZamV|`DnfwQm!Getr|swCz7KCK6rz6VKH$m?WnPH7>`TNuiQ5MiRATaTUrf8dwhmE@=8g)4t1(r7V& z^D4=Mjwm-_Hlp|e@Z{|1`?Fbhaoxy`vkfP4AMj-L@9S<_$0cm<6=24^XyTYxqc1c^9J$OOCB1W3^1l&mm9^l{XBZpqZt}`7xzrW#eeEHtl&Aq*AJX@U@84m6^xaXZ2 zwgm8abv`(B>-#Ku+;4uych>-(u6`MwByKto65G44Zl0ZO))XJK`to%LI8vduciTyq zz%7W`6J0viwhHC>XY?58cX!{-N9DPd)kmCMl6?^v=EO5W56(-woK(3kl^X)EnKXrv zW=0OGJ}gVqbNzghtq9+U-XphiL$4KXo&ir^BA$+>*i`ME&RxB)jozK95IW7i(B1J{ zU)v*l%5YB~qt`2)2h%mLRO8l(KKR|Hp9jXC7tB27j?AA+=k^4jSvhWHfz;j$>Epy_ zXDLiizP-ffDUR0;_LsyLrl&tV9*~72_)Wym4GFKWA0Nk)(XSBya$4Z^RcW6oUosjO zuWoMeT>{Xf`#&l$Mg3I*eus=P^O$&=KCIR_1pf7C9N&FZ{7Zy7l{c*vlng0ap)EL1=_3w|zR-?O4KdK^o&xh--B9hksc*6MF#N*30D^yMw)A z6W5C^|Lfy@BDQ2CUuUQ8O;d{_k=qVox1X~cM^{_atd73FdP?KjB=+Hv(AR8YkFF85 zbl;$(XKm~jtlM(PVfS$D8I5Cat)k#FEBV8CZm~4rAJcl>7r~!Eq&K{<#&2GE5*IR@ zG}4SirJ6^j)*KgKtSF5Pv~y7F@bx)3*D_3~`52VwCkyLv)WEI*hs(I5t4pgY=tdb~ zj0kmy1Y=L1IIeE&5bmFb#RyogF%8EU66#iiG{|AWh{enjV)Vs9AenKj@s^3%ij{zW zV7);!6GRNzbzQ`#L^n4VC#KM-6g*K)vou_+)0O0Ho)FJI;+b+^<<}U|lQ1eJnmTe; z%d!wfU8!(kv1(&RrOhFv;)6*;pSU=znv}zz;iw61xe!7wROrskWmQRckiub5B*~>9 z6*)q%QA#9Sy_G17-zG{ZJU?NgC>4c-$#oLIMPXHJP}rw@H7IFsU@%f}K9W!J!A1E5 z6om@}LRH|P$V_YEWUJ#Xos{6N6j@bJ0g&Ju%w!2+q_C_6kDx2JHXM~>7&SpgRwT)F zutqU)iHD$Ykq|X;sW7zN++9p91 zvLd3w=yBygSQ3I=LuuXouE~$%*?6#^KU-O6@W}HnkTUI5A)1(18LGLzq$t7$ z=1P!1kI-53nHTGLqaaGi30ib3!Fiov$oj;xnETAWXm6BUjtoP_#i}=wlrs6dB|Du4 z%A^#)E0jqx&Fn;qgH&-~fS*%Qe#sZ_ilEe4XFfU{#>gKR*8JE$ln4(e#_h;x>++JbnFHos>?g{^FS%@mE6j(`=Lz7qo&6FkC(q;OLOw7*j!a%_pk>HX{+ zKvOGdM`-4fEMA9-D;++C#l^x+WH?8@$KrbpGG`L3lgTWK?G?2*Z6Y71C`v4iREoXC zu_!n&)s3TUxMklrWu`c1l$EepNOHt*Osr6Ef|+Nh zPM=Rvg-^bcO|7bSzqqC^1rOMYL!`teQ(ZM&op|$LSG z26;9KT!xM~r;LtgID}%#OX<=a&=I)q=w!@n_*UH*2#Ts=wsAW(5gO?>MdWNYsaLRc zy7of&?Jw&vIYLQMTbiO?@nmD{zji0jSK59Usot!N%wo3 zcMYtlFl*#ZzoN`6pG3COTH}V_-n{C{N(@^PKV+C~WWhcFD-JSf(ym_tQPWOa?I0mY zzMOKM1xo-+slHrFj3XYqUxb0mcCkHoqFY~)Uv_goSu;RlXmFP73$as6da7ft$%@&U!z;74+ljWeu@6jPjkjvL zvbF`f?xKM_KPVdwLX?pRE~)VIi>okhDLG?=FgT~)!ST{;d(&(NB8+uv8iJ(GsxK?hmZDlxpS`V|1 zria-#p7gDB?G-Oru|)g&amI_S_Kf7L@>ihcD$MO|FA{mZ ztK-i7)N31shUP{%C%imsxt(~+a}MiAV%dL|Ey^u7BI`Du)OrKnU{vfe^}`P!13fO3 z$D4BW#gah0svpsB@gZ^`gA6XZS>s)JUM?)jT4%t7SUg=u+itevxs;dc72~LX_-4C! zNcK%=na|1baJ78LXOyi}K{sE8%lhT}6x;8Fw@la5IXAc7VCfY^kNCnmZ$D|kOxUL; z>|@`@?{r)A@*o{W_&z?|J+*NDZU=Sj?rhyAflCp{N_ezS{m?jfW$C*XLxbqys?ME~ zzm>QB!j`N}zJqh6Qh6aM!TX6{?n0I1tmN5C|4yHj{C+R$y4B8dklz748o5E{*u%xO z|1kdYCNn6~V3Fvq^Ji|F#{u(br1M{n+1uUQU}Wse%HSy#6Yiu`rey~zoxr?_rDSwO z!Wv)Qr9TcDPaWlP^hcM#atNy!V>q2T=foEEvr4Kj5kfFt?38e43*S6MrKK~=`*CTgd7RNRQJ!^z|iI2{+4*Me-_jBmP z8NBlL&!1bN@(dYzl~3s#kD4mQ4rhxy0h;7H90@|-Ut9fbI~MGvz2bKF)9jf0TXrtMy+Tb!K2+du8((@kteh0 zrFy-jDZhm*n~Iyu9{_%={1;C#}-jqh8tb^n4WquHsR$#wl|T%eN|6p&^I z_t*U#yq*a@j@Vhd8CnohMl*#qzWWWhHwD^3}_ULj>{IXHr^ zOlLM$8tQInp?qJE(Ti4(XTdj4H5!5OMM;{OlEIC`2q9CBu2bz{Wa0U^cp_D+yBj z+jMsl&Y)iuLT~buR%kE_HKx)7r8%UaWLy&|IK~PoS#dbKp z<&oXaYjmk)cyHT`fkSW}1x_E$X^7E2>!ix~5!vmFY5m5de%ILYjAjCr($iq9hi%7r z%53(&op{vW7=;9Qlpz03HZE3C6fU*llh4L&_a%+uA~*bCd<)|4>i8u~#^0+r2@at_ zNmKDESRN10Edl>9?dZ^tDf8||V(qb}*k4iCl5HC^M!}Pl{r-{CX^C!P@z8!M; zHoeClo)e=_zg$Pp47EvMnY`J?dpaz)FZjr=98s6G(dW zWcKWlI$#jS2)VmuQM3=ztZ{NZAz<@RAi7%P@kz~j&m(72p=oN#g5bnpfcW$*vC78W ze7F(B5@a^wGEj{`_XAHt#1#c?C@$4>RR`ZS>Xx1=1pRUMCO+$jDg zj5_WUb}P71;@=f+s>6-KngO2&!-+zIkfp_254M`HHYVqV&H86iG|Ycpz`wIul@YCA z;GCed8I23gm_vkaunjNL++-#)v!8(M>2(FJI%FeO2o2V6L#adlJ}TM>wthAN7u3^P zcxv;r)mrlOhv4?D&Js=xR;#U{>=P?Pjh5|?*a-{c#cg8?eDOcDJU*6iy+U z6vK1|aps|->yt?!800j$FlDc>mi}N<;!YxPX6&Y9+?^?c{K^i{V67me)X*Zw?!jnF zZeAgx8ga^TtQf;uM@H;KFnGJcN>U_B=Sny!l6y)?CILn$V=Uf?Vp2)jQQ-AqCN<^eKCVz+%maWKDPC$hRw=q70@Zb#9L<*{%t#<@w2LBdmWySx z6;=NlHb5&XF_O)qbt31t=B9eXKU3`t4Z9o!kQ< zS&Y7l6?+>0dZ_rj1W}9%K^#LseWtL4L{VBAUO>GMMf%j5Re1!1Q*!h%YJKV zn=OhKPblMHa2#qgM6#mak03IIIk0RnnJ@S7O9MpYw8a{2ZR<3}oQ6jrlI3aC;vCA< z_gs|Wv{Hqs{kN2bQ2{7ch^=445CbT)Q9GO|-(LHBh$3=HOWeExlHKB5Y7oe(&A z$=0T~!$gpcStv!U10*M?+`;mlqA>$*c4sKJB-CwPY5rwknp_Mq#^Lmv2B1GvjZBPsx^9X{T z%R<&RZ-QoOfUsbKh8o-Ov`}r8d@muMDKd!2m`(Y1Eyk?PM6?j|Y#gT|oU-p~jeg#F ziNH{qet>&qzbx(6FV?)tst|!0W!>tRF}(Q*W3B`DOAIbWPbHeVT6%*b)|Slp)vs8% zbOchvKj2k}U>wvTdlGAKCKb$8C4PI-_@Z6v!tY;rq1bB!o%QiFKBJpC8TU&iqYfXs zh+=;|Wd6zgZQGYA#D$d1P227Vi@GOFiv5I;gU}|6a9BAZ)n5gKQQp-55 zWMmWJ{c8%!wRp?1tBnn#_+ZWb2>0n@7=UJI9UY{jN?I#WLgFDHvSJB&iy*X1YS%Pb=`Rq z_Jeu^0(}rQlUUKi_qn!1D=#^Lu@>hw12vcyr&a~0QPH00EFvav)}*N)16lK2IL!X| zB|(>LKBVR-XEs}2HL3VBh4^b_Qt||T2T3lr*w6UzKirDP1#{>Glt@Ntk@~FZDWbm_ z+O>Ut+)5QvAe@SLiR@}R9H~s4KMClu^;;>k#6M%Oi=P&N_7!G`7x<)wvpCtA|ItxIoVC|zvGil08)YiRgy0w= zQ68~}={HY~IgYDc9vT1(xxZa+f{d0)i2s9;%ExTR?;Scurc#k%GrQQ3Zgw&iaKrK3tN;bS2Ja81p2Nsv4!QQ50k_E@wMVu8p_+5 zO?XbB7D2gif0-F{kCm~YKq*lrB4Gr=lhFq;ucn_$YBL!l6w9o3K(z`M-gdSQ(n##0w3Z1zSuls-oALr5cos_;?kaC!`Lt2TXQ7`6RLrq1N`hWn_q z)R<~P!ge?E%sGQ=g&Jc#URIV?%w%WkB+IiZ~BK z9YQ`47wWY;(mAPE8J?>n+jyt%UAI5|&4p=mMuh4w;f!y= z(1&&DAzkN+2uo&tf?MAUE)O{dr4mrvtpnjIWYaOYO>xf@B)J^+ zh*+v86*AB{SM-3FP>TN`jvh%DZLVOm2;gVr7j>(kBxb(-(Ln|0K1(CSIOa`Cx$7a@ zJsBB?*|Gr>o|P?TIj;;z$gd521p1LPGz%h4)Xxp)&Pb*xcP>}lhoS6i$N_sI*#^}$ zYc)=7^z`rVwj-o~hw09_%s*C!&{^I$YW7eZzUvfD#pzi)lHEdeXlE^cAJztCbPb4f zTq8e1`%&T@9>JAI6#n9JG(XY=9IBYwUad!4aN_qiKjdXz4RcQ+NX$9&w-|fLF7hrn zy`vtEg)1Q15YSJDe%&#zE3j7jS3+}mbpAAE4{NC`YA4WSSw1BxKxVwiP}Y;OhFS7K zV-Jnws8ppzTXao8-5Pr)cx%mI34H&EOx*7Bz8fXp=JyiKwGMC?@O!*mPuU50z1%#D zIf>k9bDD20H1B6Grq2&}yxS4-e!Ds$mjAw=zBt+W+>Bk_+3pX{u6pm_=<^Gispl>6&G-A|t}5SP#qj$8I`MN$sOS4a zQSkG8v-iDsHqQ@mwMa$mf8RgYE5)OGU97u&O za031PxbqG8s#^WN$wtoqzWwOj>3umK+!6ZTs?rqld)OleF1`)l^t?Lx{Qh3OGZ6IZ z4JiJ8Ka1V@eB6r7Rx}j)K8O|S_W5)KKnZ>Aw+241pD6Zx?q66M27Fd;FOC}We;!sL zZ-2im6Mw(Hr!zSid_EpW=5PP&63jRZeI9fxUPkWuynIN>ukHOheov0(rH8FJf~tzrp&Pu-hn4Ms7>-$JvwCq76mIZI zs#;P(Z4ZcKL?a=$5OFGL?W{-hl$XTeC!YrvKTE$jy|36k!9ewAOgni_ZZW7k@bC+t zS16qSoOX+#sEMB=@mR9drgSQwgsIs9OKZ(qXd>7V56|hleV<^HAL4(d|nq6|NxEP}7_{7_Yp5 zxH8tN$g^m8H|F*zX;R(|pbw}>+G41kWQY_AC^Zd(Fo(rzfN~96+LpB|Yx02~hCN<$ z1fA3+tY3ELU-dq{-Z0Z}p{-H;p#U!v26ut`o{L5OOSz-lE8_tmAvw2nwGe!`}apXX+Z$ zQYYV=IuC)O&xQw|YDS2N~mnlYRU>Y zes=idI=PpoJBp{#U6DCo6sF#SVqG=T#F_pz zH}Z$jCWaujHJF+%BlQ4o7YBeZR9e95B$dEJ;k5ITdA+^iOt2+(5PfR-Jh4T$+U?TD ze`eik+_{JIr@E|hDR`rn>Gz@mAP7WI|Tf z0TdWC7CAuVCBfAAlLUdq7Ic9nocgq`ydbQmRj zY(FCvyA|3?iwLQ7moO<1J<5j8t|1#de1-~MDn-G;pC~yhk6sLq z+>&M2aB}oz6xmeyoJcBdMEgsWL2n#XabF@R8Q2fgNVin$;!dl${|B0&ZPns1kIWG? z4ct1H)1U|lw}qehbwPjbaLS6Wi={?gz!0_TW?V!kwTdoCf(DIY2hu2Gy~3?Oe}fmjRQx-oWI_^_n|QK+*!J+-0N^~nJ`=x=It6uobg)h z<@KpXYxE9oT3O8E|8p}EAz3=0$YT-pfObW>;Lq;X_FqZ~s+J_6Ke7_o|BTss%x|n{ z=M|Mo^{cnyapTVppmEg$!~iwRdEqGrRp4NU6>0gENQ}@(KQ!|SkzjFg)L#!zTjN?f zkKEgjnB53{3oh1Vth7-ya0!8xofISVWiC8UtM67L9SoT|27lAa{-XNUkktF$&|D!Y zRQ_-116G?6&B3p5^?pcv(yAZOs$dLqysMsB4bOG^PVXAGf=E>ClDA*iX_`lAgMjGA zC8~w$IhW++FpbY~y~294S~y@CF0*e+oms8AQMdB+OMoBI{sDjY`z43nW=>+VFDTpU zYKQ=UV=aJcw;QGGi#FneC+SJH26PT&R@^b0GyR;$|F9w<{+MJQ_T=GC+Ykdw)1NmR zl!8rJ0^HmW>Yqt3Z8`NQC%sq6biQzaU0q-jI*(( zi;c~*F)L%YZk@hgOu!n}%iHQFz=br^C!?6}cMlj<#ET!a_AdFVVhVg%Z#)kgnFXCZ zkvjV#VE0pqwwz^)qd3!^7~Ze2TD?Y4zs(Bb;*&dY7!=f{Ce&lXrfov{O}`LKLbVQE$55P7`XQQ3?9Me zkXbsUjfQj`Bw!@_D&N#J*^#VIa=(T;L?8@QJWnF5&bKSZJ_Lg5E~?<*AF{iS1I>r7%d z4zH^xX`@rtA<;(;UW8djCZD&wO?^Ymk3xy&e-qHD8y{ThO<@qWZq=F|?NB<5qpi-? zQQS7;=>_nUuT!YB71~fcdWjoyoGcOaPfoPjlsvu!E~%p;P>&d z;!f}D=k%l|)Az&hg(jC!kN4McIkBU`=Op4S7xCA=V_t8&c1>^I*W>*WQ2hD5yp^Zl z?0i4CLHFiV=m*@j-YiV(?O&WE*oXTl`x+LTX>F~J+2v=B4+R|GfgxxK@1#)&A7|Ay z*E%aEx&swrH6zWj5<#{LM%{6^RS50w$HR@!^MMtg`MX#*8(jhUY=(wyed_59w7J1S zkFQY2n=?RDVq3uFc}zW#ApW2o99o;nhUmy>1;t>KpWTv`mida_*Q^1^yfmndbTZZL ziJmAMk}5BFC|T*_qSwSB!@D41Pl~VHjKvV!4x%oExK_|e6>pAcj~lC8a4XU{(+?n+ zWc@UyZwt?0*;K%06DMRtpxPm!Kbqnpq z`xc_dlxEJ9ZwB>lZb}%}ERP5z3zv3)_cXG~tn2rbk@APr>mCzWWjfG`kHd!3o2cPk zx)~PMBc0eK@8p^vsX`u!U3iHELH@at1VoK9vury_iVtYQs%ke`XMJ=OFRt1i`)MLreDKBj;#ZZX+k zW1{}t`*c%75haCObq&TG9HKyS^K2l(;CBwdat9U?ue-IAH>RFcA&4jb)t=z`%ZuWR zTQBOdz_%^lF{YrP)fmB|lJD@aNkGU5OqK{;s%G7Zr?3^sZe*~kn3}yBhE{&)1dE#C zkBmbd+9$MhvKNG<(oF#V;g?C5b44~2?hXxtl<#v9Z2vFtINOVCc+*#V%xPV`1T%zh zB+Dhc5KiZ_54F$td=Z7Wh-06wL`>_pn?K?q4n(?nO2nK+M66K@VgQ)(m5mwK!2FA(4&`E z+pLzTeqo`T+}Tm92xMBynZV;&qq|0}dvp{OuKYBr8KYMET)8+K*2V0vUUGhRd8Ddu zdOx^Moc!YcCqJ0yfY*W_gP~bZ$de(an5 zZ602|iZwK|Vg>WFa+trPio`P|&kGIG&Hn;OFysp*wm=kLN}%Z+Q@EBLgWHJSV+!1O z6}P>q1Yr-Myv->8H~R&f)X9Im&v@1*n0L2^}}6hlSbde-(H7KEDMb< zbXUtK(Q@wY_Hm_R+JZc!n9%e_qO`0h;m!5F5A~1_)f||Gd3pY9+Nfao!KU=OIF-b9 zp2lVw1z(S{YKqL;lVu(yMRANmuD9Z}KpbNNaUA<77opBR>$xyac_x;hc|+hPHt=U!XjAPExg!W#>-n@5)GNo3r=qA zp`}!Lc+mo0Y~k>x%;D$iVCt=?P?L}l1ZWq+r1NBOd2Qs7==TYEfP#IC2>M_{^3f)qgSy|4&`)vD)QYv7k8;4{EhqX@S#FTP4XG-7CS+JAI1f;7JZimr1zYQTMP-e2 z6wyx*Q_-$t*CDwNTbnBDTlfq46CV-gotg%}HIcYBD$3_2WW{5v@chO+@_rWWi(j(< zIho*rIKwV<&ayfDllB4p$Rz^>-K&&au^{v;_nD`F7lD0wEKxNgA;#lEt~MAbpCnR? zC$2Sjrgy@WM_?CRAc4dUAn8aZg-MIt<~G)&>}UV!k^wX1D&jLix zrdha3i?rZ;iRt6DWc<2E`t~*peW41JvE!QSDCau0A#*XY^zwD0=6>Q}snl!~b!%tr zQQgn>jA4jr+4-q?SvimY<%mY|9qq${=ah~?Fqp5w|ArI%a*B!RFWV6wn|u+4YvDlSDqiaYz%@u zXj_~!GNf2CF?r01(v!hjcCmaoob^Gx|8=&<%5WPEzC#$Ac4V?k+AMD`(yWiQKfs;Q z0trUA-NbK!U7r@~0j~xcdd#A&NKO)+DnqgtR_?b}nX5r!zA=&8fI_oXzBz;-n)mgu zQau%gGF%Ak0z?E->SOAOYeX}pe1BW!{SpMiu}pFaPs(thL=aBAGXv_k>one00wNT^ zEqvWy_-27nEA8*Km8s8e>9 z{84gGPya(I>e*p>EaDf@;rLzs`+@q)ffLG&^Mw^o8DNWI?wWCe zfx#OYo?&RPehw9YXM-cH53>K8&=me_sNbU_qmT+8Sp=Vr+?DKpTj+|Um$-I}e|YTw zJe;um553>p7&l8}K89x9HABf|{~4z)2-eN;l;Gsf3&|GWk|OeF@GeaSPpE(LiLV2xfkqhnt-W{tp%bH$5HS=sIg{~Mb3Y{_M`tMF9lC&^Go zw#MyF>^WvQ?OZJuiuYF80oGz*5yBfXMc(D-Bz;zK(b@`B@Blu#rIK^V4lHicrbB-y zc0?oWp z()sqD&Pk0~Y^0cS1KW~@zLG46Y>eFIeS`P(&3A4N1Ew)H3L}HnjJj2OKJnSjkRNjit};-nUdq z50qcAj1?1hfvtIfIC3@6_gJ)@2UnJZ=%m?%z6>eG$RvNO9u+=AC&qXK!BJ&vr)mUi zsxfDny8XCL57|n%AB{CkbR)>c7=oq%0apvW(+35VqR|B_xKzmCKK z?f*9gb7gpBUlq*;^o;qKL)on?S9|Uaqwxa?i;OJ7Nx{4C?{)u`cVGvRA15(#N{h_% z_-hZn*AyKwW7jE592VPAm~x?<(fGVz9r@P+XUc-6JA-i$-Y8KRQXHZp1j&smvRs@ykIDlIk#lSYWXhO68 zIgs%88Ix&Zi;?3dc1a+J+~jKT>GGHKNzq1?d`$yRPbU~8!FfI8;?93w6>4WvTd&A8 zuJx-?oT_HUXY{FLLDIsDHKr_7+vjmC?CxhRps6A%q^4jA=xKI?TWSXU_iPP{u6*o~ejeB;;=+REaKAkH?RWRKJe>&CGp}c( z*_z)j;2e_wpSwVUA4 zd^xGeSHPQLT?a&3P}rmDFlrdxApi;s*vm8HXHbpM!FaUhf)^ssvGU1L}FGvD+z#yv5vPodCA&gALV?$tvJ@gyJat;NyF+ao$LyI>{Wf z6ha#dTLQI<0+Pjkr`i`VSJP?IklwkjB$*Ve(Uz>>iQ-(q{3?=7#wO|C(=g2)=GRm# z4R-DLEuN673t+<`-$a@Mkip;=2q*laV7&m74*79!h$+P_9j$|5!N8=Fu-kp>?+0d@ z=lkCCc-$Tp0QmQR#)%O5JYp5paD$dsb25TX1)=?so|fhSvf{E4bE%5lq2{Z`w9~r0 zNr&*F9}MYE)zX{PZCI*s=HPLZs9}e7wKY5}s|U?{LwZ&ASJ9c_Z0NqVA;kH88K*{a zkwuie_uz;(*WS6;JpWoQP)7$__({^5Xluyr(n<@G$7$vfwfa&|oiy?s*%@!x3R`L# zlaEbyl`eQD4mD?&!6|-Dm4)-hal9NNA_$iyYsvqkKpmUIsbowI>91+ za8d4^@Ya~>GuJ@Vs-6$I-goasKDIuG*AZG#(;9KE`9r2sCe^2UFl3wR_yAptFMTj` z=9i$m=j?^s>|T$)Vn*b!9MuH`v=t@y?vR0KC^%)E8SjjjLiq+6w^gG|?mH(RLmkA#mzofM|tu#~1pmM?n=eHh){Jg+yMI;Bk zXBrDOw;u4eT)jf8sU}qUX8#qSa`Mqzy<`M{MTM@32MP3SCq4T@{ zrBEbjq=0ciaEM;XoKDe!3DYTeamw}{@D8YR|2C-lKJVpY6SITjpqiyfJdbm@8~~;$ zrAu=(*H9B8!_o9shaDul?u-Y2rcp)-~K9?qn zM6>xOie{6%1LlU$DxoqXM83MJPbfu+>HhiggIROcwLFKLKe#yyfAJ;vKIh=k%X7BM zZPvKXSTQxyBM+XWDrY+N65fp?#MeIN8K{R(D3#MnTaLQr?;zjlsXE?daTe_b7JVep zBGf7{Hluk?)QsSNg$YaXfkY=Si?^d@P|%fX{K{l5^g@11R5TjaMWX81hlSq$9|Um1 z=Lda%+MhFlDA;;U&XGdNakGqd>?#49fmbK^?_&edoybCZZA)AS?He?gwp?xv!ck&1 z!)6nLz#$b@#~__QBcnbh2@~7{QxPL`O)I7GKk79oq=vg$+`Pc(5X>`PCzXFX|6ls7 zt0AwUY&yx(TQf|9HnclRH%ow)pXmWF0Rei+SP156^lhtcdUdlCL5!3#i$M<84DIs5 zWUGU<_UNAbIa2sZX7QxTpOsDMNiO9CfA?}wmOLW7Ot2N!b<+SZ5G$Zt%-apzPs^VF z3)Ei=8AT4&4S=3bXC_*4lEp>`t+YUawg;)vsi*U??l;A%Jgx$h_ zmHQmEOd5WHpZ)?;zXjWkrf|^!m_>4s`u@GxP;Q=H|G%cbGA^oc`&v4LA*4&Xq(Qp7 zTPdZxyHQHIySuxG?hfhhoB;)-rQ|(&@BRPY4}61p&a?O0d#xRFm_Yc!l#4-S7< zVPv=Wzsi=!p|yjA(m;n8>BiElgs_%7WV4>=?nCM7xM^hqXn(HqY;RsMZn_YttZ;q} zyoF~`5BwlWlio=3VV)n?77mZ?bR_u*-kx#g3@e8z&BD%wJb$#Bq?O~D0M-Q6mY&Yl_dgc{<#z%7KGpsU0 z*1N07)s_W;UE?``^^!XW`hf`$OnC*)C(Q&ypr0x^HE;c?Gy7pFC-DiJA}E@IZ7Fmc z)|5G95_S!jx}bx0`XQJ~DqL3_CFYxp@r&G_C-@~n9zp-0O(BoFrZ&j0-{qwla;M%k zTdUqV3P(c$Cn5UqDL|F+N-0g^jhdJ4G;^AT87z;{qA#6e+ZHKN&?w$QT5QX_6s8cF zUgS^>WGR8X{d~ziYC9y7=a-H8fBSWn*Nz{>5}R!bI6qGrQ<^WaNRR4&9s|0K70vx> z3`uJ(-x;$}q`Q;hp({67qc^hgcAs+-IBdmoUt1qwYUYG!pdfZ5%OP8@G$)0BoAK*` z0$4J(OKOP}@;U18(J%IiLA;+N6%%&p(AKD9qgso6zqhxL1Qg)mE${mG9@d5U69G~7 z!G?sQ^`p)E^rc|C2ugb5SD)V6m7-Cs{Sp)71J~xpuzj;UpE3Gw?O+i5r|JkLW#jpR z@yUsRF3sC9usoWAp)O-M0_PFWV@zKbc=j}No2cr;7Dx4JW^D_;@WnJWQG2SA1Ky52 z%{R_hsacX7&p>2E&Umki!pR@K1c0?-$=N=h2{K<|y3YTO80OWaBQb-8%v@nO3c}d% zsTQa2F1o8<8O~T0jZLkYxY|g3_<*b=vCN4N*H}WUL=A+luB7eaaw<{I!eiP7;Nh{v zYsDN>imBZSV`bUfRlVV#MQNOdF!Sr#86*=JnABYve9e34+7oNbj(gVoYI%1d87VSoHm+By^j|A&N=&M62&xQCppzj!fE zFH(gIP2+;6z<{gA+ca~%#W*2bKka}lPo7BdqZuszb8L|XZSmDGoJ$xAkE3Bs*o|SW zem;VNK^bNso-?zk#{GGZ_-j(T2{5aoM-Elb#jpQ}Ftr?SZNaiB6LIz`dru@Tn|G8%ANX6=ahD1RrLLE;beIX_i4?z<@hy ziQFcE_;>$5(fI>!w12c25sd$3dO%5Vjfav2?#`5G*SzTL7iYD@-%8?3I1|mFr}Zwh zCLx8Nk%12N_nl{jZL$^2L5au}bUPd-lOKz?In_3LbMA1sxMEKc*7-*_zGME0 zyTQ)JN??KRKuDjWJIyW-5>K9PPQk2TLjTR1*XM2kxtNGrr^D>EFI`FV7JcbT8QF+K)*sMid`0hKa-9X#nPdk|SXX>Tn)68-Z>g z3DVk1+8p1_Ge+&dDk2++&0`&SWvW}{dABbMPmT3HFFHc3q3DnKTqCU*%b!925xTnX zkm30(lMbVQI%1YQ*LxF6AxPJwI$$T+Z9<;#ur1=lN%osCVxXMxY64&)GLTqgVRHr`k!$C1)h*KdwzAQ zhC;|96XI&t5Opv~mMAUIdJ0f9%&6k&NVuCxdVQoboF*FLozqbrO5#r9pjaR)vy+Tc zL#xLNS{8^F>oT|5ZIzNuYOa!lg%w@(3!4_gnKaU79Z4)Q{P*~~^xG0vxOntw15H0W z151i*&*Y_d4kY#Y{uV}sviUF^;dJw0smCf){?T~a`heh=*Rp)3CAOIkqnh==I-Q&l zHKZsxjp?R-A)i`(_6jaSmXW4ong?K~Dc6Bc{jvCx{~R>SHZ9a{*;$4LmNpoJi6ped zrYd-o2;kpIK*=6&2C~*A33Ag?=fv16`#%16 zi@G`F^fka%qh=Hdn8tzJALt9eCilkkz_Ek+hkLR$dPX|AjlTY6srzPx<^De+f{zaj3&Qy56J5LX_8S475GZC>{|gwEjHbM(9u?CIbcYOYUWa$rA%i^E}fq5|1lH39!f|7D}}9@6HbMk+<&%44LrYnYjn&?u>0M{=KN7kp<4Gf4pZ^x zhT=)5&H1xMIvMk1I?2^3%y;CHdd5o1OcJ1FVX zpULTN?g_}9C&zcwDzwqfUfAU))y}DVD{*&j z@AVm2z?44GSzw&{{GLJTV|}dJ&!DEsgTegwhA{jz`E*t zFGkvaN*I4;#|nrQY)y`~zp(4mMOWPJK zQ`$3fsph@#bmv#w(_2g4*$9Whuhjql)}BW^YLf|HzW%~f3Ej-`0}tsW9UcqpSUK`J zPXLAUvV=(1JIrS^^WrybKLXwGrr*V_xCv1$rt*?CbzC=t=4h?61J(3?9OTuy<+WR? zF7c~x1`obPddn~WtNteHg>lH+%a^-QWjKXRSRm2BOw%D1ERe4}Gn*-NQMTFL#r!8OBlpsI9Jgqb%fEEqwogv)wQrcP zT74etfxoP5MKE&BE=Tax&h~FY8H0GSF|d=Gdi}vri588L#^;fS7fTpGa+p1TF8zi9 zZ8wId0Z_+svn$_vD4J-i_u8TnH=;cXyyXS~6GU{kA1Mi!@SptrMddGrVb+HoQ|g5zAU2_*LX@OP{e4^GBuvZ}JnJ?!IE4;BYw zo?xXt^9uwwE+opC#9614dA_bQEOT=~E+UZ$)SOlgjn_Q=I8f$Hi;@_bS{PLeQN4tO z*5O$P?O$f>k~sb7NYt-S1R*_>QG3`%3Z_zbrrct$Rf2VK9??F?02}cmbc+ik^EoqP zRdxV*$YE%>^<-TCegS(h3#Af|xaAs9c$%>$9DCoG#J`H5rgNuF$s1wuC#P!(>a9oP zdw8)*xV!AwCcQ1p*gTl@&*Y6;+4E4r|Ka*UD;rgGKNuC6l(+VcEDbG3j0p1A{#O%d z;Y434jeVPyE8to*FhsLOQ2Kme32KtOU(j{GuP=L%@Tm-MA{%f&V68*-Lt~l2%4yvm zg_sfXITp`dnVD`4p_4)OGNK<4to;hgEy&t09ZypNN>vTSxjuL3WKHiBviu6)qivMW zL>g1!X*loUY!lP=3Wyz6bE}|-|Fzh@^RrM-g3!;q`x*9B5W30Z5&{cXqf#LPCUbCZ+S7^`)y$5-o^-aCTgLxw4N#e>Hl+}+swG&m$bK%d=S72lh zuj5Flp;zm`h7$Spm5e!Rj!`^ig%cG;t1eIfaC+||C+cUxkxNIj#dX%1n3;{t;U=m| zde*Ba`Y9I&70F6bvGynGq;VBW5d=5H#FYDy^F-+Z&MDkAaVGob7t=oI+ROY}YbF_U z=;4+b$3yu}-z@q27*fnKS1n|24}Li5We~Z2yqrnqlJI+VGD-p^@viwC&xLA90r-qU z8#KjS*8R63$if>#78eXj(mwkn=*-b6m`wDsh{ByOkU(Ph)mJ-xF8r7o`he>pA$@B6#xP^Ia zj%m;n#Y#0vqIaZYN_uyC6F+>`8&=tclilB{t8^h=OihMOrdl($6;k?AG>e`1VB-n~ z{vM=>(ed+ws(z%4WupZdTRSy=E3KkUUOcTV1>Ho&KFW5544G@Y@s9ib;`4`eZM+=V zXcSKo=Q@FqS0LFBPDDM!7`xXUhFTEr6B;BxHbK@3qXuD3mT~7)5l(}?zixr$IYW`#=Q3YWL}P z|Jl2i4mE^#g=6}*eQ%JxyA&=8SvuCS%C;>tP>EZViwT*wA}bITe>T29^pjrtI&<+` z7)oo21RH@t6tqo5oHS5S`^9demg|^UB+5u*)}>ItH>EE`Y;2Ac(=wqLW=E&@HsAY@ zT_+=;bjA?gGRVgy`!Jw*`Ma8L^fKv#xZtDjD`#s#T%RR(s6bFjyCr$f0KyN8W=BDo ze)UB5Ur!#72)Z!qe#`!eN%y?bMH7M|%2yjokQs4ETqZ;hr_QYa-x{BYW}zDnMroJ= z@ptkXmX)h~pd9uzp#wRhiwnEpHWFD3+=qKt5Ba{p6y-An$EmH?gA;7ivfh*TbXH z_}HpZFO}3v+Rh-IXpH*xm76OL#?#gZYcsven`kB5Pmbhn_SAdd5HZq>o1S2+1qZiC zihqfEg#l)gzuGuG(&KPV`mUCcZ_GgMJaEXrm)Frsid;q%Pa+ukECfpQO>&#w(IDqn zv3)KIIzZZT$5gUXo%6!>|3!flHLuxdrBpqGwm4BzyD&m}amO^u(v$E7A(L=DT{VzfoIWU&&KgN@CkrM0eeVnx1cT`gm zMGJe7eDe~)KtxJuumK7<_HgDJxeY8bVJbM?@|={8)W;EX$y)$cNh#@VN_QPK zoOR31H;aQcbk0qW4D9spHZoNow4M|`7Zt6a{%-eWhL7nXXHfSy$~+qBE-QPJ;(HBm zcDM84u=4W&QK?xY1qzzi(%ZCB+jIGlc>`B^@3Znk-Kp7r7PN|2YcK)ErEt4TMohr3 zWZPeSqCM^S)=Hg;AqDSpF;~)*(79pXmC}45ti588t>REUqAVLw~!+Qjob?GH!j3V2X{l!@IK$^ z=+bJuF7C0UJM=|4MxY7|5;M96!Q+(Lzis&@#-V+^2}yZW92>R0n?9`D$5BJv#Ki%b*(wMLm`aBvlipSQ}=+1!*DQq>dzVFXz zc)fW%jCgjcmyuDpGLnY}>{>4Jlu@|tug8EF7a;{pD(VDwO6OF z{Ts!6>Fks#3MtP@Pa1}_SL;*?JMr^T&1|gEgau~ups-WdzqgghoV<*sg8OQ zP5atjwe0}#vK&Gdn)SO^azV2^YAXo>`K}M6I`qY{$fDat@Er1lW8tjqE}+QnZ5IVDrQ={&0fT)1c{aA`?^f#+ zZ(RXRLrp&mJ}-B$>|Gr+^V;Efx=%3@8*h5UuXZ^qlXpn7z{6XHr&VOUPR{V=9aa!m6rjCZZ$vsMctz5!ksg@2!JMk%96YyPii~r zBJjduv}lehgHBQyr4~uXfP>Y5K}#P@&giBs+Q9MWIJIv`>ox>I%xZJ z_mJKFq+C>_(xCrCH*Pwu4Ty+;JI24?sY6H2Ndmf`*Md85JVkpYM3qcgNni@?_}F<> z$hfhNrE1m&UJL~C2N2j-C33l~r*q56n--QA>rsUu<5T3agpGCPK|-eXvqqb?MyK?T z7Y*99szDhFacXNA0+=%-twIhrw7@fDVA;%81vJ@Wj2C3&ylqp|&t)(oOu||Z@b7o< zIa2D$?khO|EUjQiUKF{G-&NJ`MNE8Rm>p%UXe}e^0+Q%RXBab_^~lw;KO*{B&T1QCZ)CmVOwMlKaB?hVU zWR*9ZYO!py$Wu@BJfQ9}=j$)?T>-(L+^y~vv~~AQBW;R-O`qwhD)3vO;GxR9)eIwz zUci?t7ILzVW`Hm6Y}+hH6t7^_bm~UlTMBy;xN^1upD+B{;qjCU&+l8hZdE!b>_|xg zsZ~x~8E!W&r+j)jDeMsNCf;CAW1^@U0aD!vG6`kWJmab`BR`#T8d`hL}=GepJ1w}S^xmgl{15tc2&wl8}_i4Y8c$ru>_;R<%)AMk6+Ez^b zpd#}8`^o?!d{XXnSRbyt`0k6(>}H_-<{Fsi`LyTjpZ&KlPoZv2OhNDvvJ|17la{l4 zB~fYd|#zq^52Y$ivI@1~cusm%sUUlN)<3fMol z|LUe6fD8#Jv727I=mixHUYxp;FZg~R4&}j*!1V-w4*OnwSD|ppsamTZuZ_+2QK0u)SA&%B&Ml;XqaZ za9G1_VpZ3h_~V+_=aK4BWN&L{l<;Rlp$EPNuzRlL*A)DUd#{f%H}7!@1;iVtpwL_b zAvoupStUm#n$1LuMZzira-c;3>e}4w<#mPVB8W3lCtpjJCIa}@bneZO7 zaQ;V;58}RfnDP;(IKHtR(WRMOS7f7U$@z;TJ{=Up*@++yf#?&7w`v8K{+tISoB*w)G_KLSaKfK@7Uu)^$5B0o9ig}yoC=wkNz zX3q=!FpGpje!qQTv+94f7D{UiTO&!zJ(Q04)kL?26Zmw z)P}==x0V+p3R=XPcwW7z(7B3i3MS*nMxhQi#rse0)@J430^IAs{ner@@*wWkwpVQVZ;K{7s&OO~!oxA3)0bUC>S zVupOM=%?n2o9ESC79dZ3R4$QtxflF+_rKr*F2zJdeX~oM-?4|@Zq}8BHucRN4({5W zgy>;fsf1kl$axbdSBt0W{rFf~A<5S!*Um<%U{4%HI!KX-<|#j)bcQM;Sd|23A1DZvvMb`ri!VNsEFpN8xqKxA1rv& zUIN|tu5qG%?|xSWy3^kN(+eStQn+-yB@l-A7vKQr(F<-ARr?Hwq3SpnBbkzux>&ZW zMU|ePpdA3Ppknlu==OW|_PDqo4Pg3bNARt(;F7aLRQltUPHMj6*DX~Ll5OZ#?s>16 z0ca%gMec$RLB10Fn)Yxv=WLqz$1XVuSBzAes%ok~;IZqfu0{*ZCtcQE0*uf;2=?r=WqZ!dE)_UXT&n;g*0rJrdE*N#y$30W#f*rOeC;b|hNAw_z=X&YMt zzb-AQ_k1m^w*AM~FDImR;fpd-bM702&v4Td*iwFR2z-S*%SgQRFme}>E+s>KVrBS~ z?IHr*!t`9D1Lb z+}D{Kyea<2&b&6c_fPs;F;|$^f9aCoIq$(;;h?(Z8>Cxuw%=rGq1M|s9J9TUtGZB# z(E7qDvD>j4So#;(7y;NBZ}6s7)Pw5v-d}f{7uySw;kvN+%Y}p?)U)|+lrXsX)y~>W2ZzC?fb^-z#JE2;nZ)6B# z_l9OeR?P;vk4K8x@}w|jy>Y)VYFP)wbYWf)q* zWvqloSxzW4x3iMc>`s2|0Ud1XFg| ziq$=K;h0Ves|ktAgm=}KPEY)`{NG2WSm%bJLXvwMqXn&hAu_G!Q)*n7vSp)qn}-yH zt7IcB1eaY<4C<2H4hRxHCMyY}Tx}Xo!V53_MN&W)(ojUfa-EmC@4eLYx$OtG621y= z;YsSKKM795Z!0_*4Sb9T>-gx$#{CY@BYZ2HJ@4zh&^{rXf3UB=RZc%jS>2}PgA+o8 z=2J7gF_BTfYBNmZdW%@Hu^uqPQ|*U(LOBmg@x2cxg<4!cNzK~{9@$T!g#R$()MhL&h&%6hDjrynm5elV-Rk$Key3?I1 z*~FeMrK_U}6_BQyg3o5vA5b?Fm6UZ}18>%!Ik&T^)!cBw?0HLh_OZ92LDuqf!Kj$B zuAapK`54&;>Vq-IL{JH?(<0v}01cNofKo2nT=QH=(w+XPx6Wpd2n9XtskF&fBon_j zJ0(#Oq)jR`JH2(7;6I3)-xT9JqW{nk<~?cx|363Sj4Vc&S1#5AyMfc?=R6r zG#KO`d!KtUPwKf$!Kg)Ift1X8hE?4boWHtzE;Rg=ill2xqp?iR|GODwLy9IRt;SjW z>}qP|o~TF$4Ppyh(ky@sg(0yTtZX}eY5aJ|EYny;E7vH3bZXGahM1^8wejbIvNxCR zHPk|%YzKTil47s=m4xlvXZW3|JJmqzL=(cG{{ zx_Cw`NI&UeMCgRsZTw7OGIi~5o-bXp2(_R{#kace4Ep_UJ>bLHf5};xOPbl65q^7s zHWz84h9Ez&-uhGvo2U{842MLT-C;#o3VVILy;k=b2Qo=5VTkR@5-a7z7xvm!*-=&P z)35Q0?Npiw4vGs$anuBYUwOMTKld2g84OYOD&bNd8wcQ04yEMzOF43@+nrH4((FRDMQrPZ`o_fxP_fRT zqaIcP2aOuTW~<#0XVs7V=4Ge+-m*#0(p`ThJ^Nci9JrOP^!ZFMi@V{QhyUjOH}TDr zuOJ6Z*Zv%_LgU=|`&5)|Fv!WVf0jX30W>G2rCu4-CM>Ylx2a!L z>~bOo*+1F+8+l!TK0WEx9dBpQ*(RIZVANOjLq7@jd0H`rW;K)YLib(L>}lpM+7x2V z`aH(h-|k-_2~Qh-Qgz}%DsLrb4Q%%}`&=;F9AYJZbrDnw4;mL$=BHdKvOy1Q#_jph zxR0a)kO1DGTbTns56AYG|K4-Zgjt<2X(_4xqx{WjZ{lrU1!alCv|CU7z8QhA&p8+MCYeu+={ zE>N^yO~^p!@;}#^FuQFgF*wF@e_(Qq?Wo0fq7%^^^h3f#hjpSzbg)Pz46>Jyzf8_W z-QXn0hjo=fa?4XWX&BGh0FXIW8|#?6vmY0mAwV;%W8bDuZIky!b?9{UB8U;7Ho`86 zfR1eC)xRPBw_QNlfLE*`ucE`bBwtSkH0hRlL5q$6h33qVnjPg$_{w!FLBFdSgC1Oj zJ`wGydp)viESXlE&hxdV(x58YO!$Tvf2=#A4@VEO!xhJQ89j+R1W2uI1|R^}EwS^( z{?YXR6OU$_%Q{Ucg6Zhh4h~4-Q;--XT2YTaYTS(=QE@fIfp=ZY zXoJNR&=KT8xiARPKfN^-6#=X!?Swle2tcLTbrHc+@$eisxZ+@o5iMi@8cSdg%&o=L(ZA4FHGn3Q*meAN2-YcnM(Y3S;0=Y}Vo>a4r=rUqEx{ z8a7Jv_U9#2&_=%@Z)O6$NHM@eENTl+;(~Lz?1la0Zx8l+BW&6=srTEKMmB-6a~~)> zsjfljP&^shjlUTAtGL3d%$|6PA$b!1my#(V*dDiR-~PZ6c99~x1+#o~PT<%{q&fS3 zKZyGvi1d7c+SN8J^Z+zOHk}05p}rgKMO!2FyPOqY!oU3@cuUs{-l$p{;iFru(7Ce+ zeQf=gSlEy@_j>k* z+n>J~^8sa2^^yFFFN|PUgypWCnBxPio+Z1N(brx9m91MHZmZ^PB(gjiP*z z#61L^ZBPE91XA6F5u2W(*ciXCqKRrd&gj%vSvxcPOBL&ml(+Me=!KhDVcd%MKuc=} zH|%LIk+u8J!$j@N1wKT@SLK7aa(MS#*{4^kJd6(n1!zJ8&HJi7kWu}~;T#AiVI8E} zB!Ke@Cz8j8Jw-itec#I+ngi5I7uks`Q|E#~qhaTrhhTN{X{oX@sNKRF@TfCECEHv~ zP?trM0+HQZyppEBrvI35vs zz7FkqK1IksM%rSyNJO3j8+>^@UO(Ud^00X>?DN~_%gqE>PE7BOAIC=9b9(#t^+`i42Zu{snz~i_3|7FaXKb`c$#+*@w+hq z+O02-yPGfkESxp`4I<<^PPBrF_m>Xa_ehiQIQ~Huhv~?l$ww+4-tj2yht>bsl|g(h zYA&k@1obLPCNzoA?4jDH6}iFya2n}tSNRV HNU#1M&Vs;I diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-topology-output.json.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-topology-output.json.gz deleted file mode 100644 index 5f590deca97f8c2a3cfdd2c836c73956948bcd9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2290 zcmV<_a95x;{45D@83N=zJIv?{r=(o(}$mac^p4`_w?Z} zX6=Us2=9=@swZ`q(tS#hQ=n47#gc9jY7=fE12MgoA>Vz#ormm^S|ZO2}W{(@6Ljo z&VtC!Lh8=K(_Mvn3B7MqXJ==1o>Nphr)UC{X=jzXB=mgFFUWL8&(W0Z?7ZMb54;;& zzm{L@1qIc6Lvh*OP?WPb6f^9_ycu!N$`l2KsC!2F#a`5L%P7^_n-W~rl`iVzWp(rF z!l^hCt%obodhE*6boJ?bMV_Wtbf$VmF040ZW8}j6t~724IO}~u<3n)l^YuMl$cwR{ zzj0JhVN|GTR8dV#=;p@6Y^+X2L2fsqnq!PAB#oOoZYtJ|S)Gc^ZXSwDqKeMrN47KFnNjn%0@DD2Q!eY}o*Zc$NxsmPg^rfyzC znN>jrp`kH}`u|y5yTaZ}RTX^_0)eK^3?xU$?$lZFdy*?gG-hX%wol2Vi2ZkZ{hZ8R zNUuv$u0`cLMMbuys4yg{2S>rfMS(6wWoSh~7-dr$t2$s&bilGg*}N@SnPLUaITGK@ z`*(pmq@a}{8J(&j8DSt(vU<=Xdfnz1R1BZUi2e#RonMgYoK@&M5=?eB*m*`+@3aOE zP~*h_byK0prb@zFR%~NcwBJiL)&2s)Scb;vP=$uPZcx`CsO&u`j}QI+F90E3i!pDB~G|jnhs6@r1@pZqC#Y ztTaYJ_|Vumgrs1bVG8x}R`Ax_rZCS$Qh@2uSP4e;ucV-{p|SdSEAZpdo`x&XSZ4Ih z%s3aBK{gqY$EKZ`WJXra=>AU03c$*Y&5;@JDKpMRW+eB#!qqw&_nvoXNu?$!AhaSy zG`lFNgmnm3ra;qNhhP+{M>Mo?95)q|AuM*-m>oBjXRWkM+%|=>@d~neIE=i|O_f=F zE+a2CC9B7=uyJ&SO?b{b3l-Qv!Q7QKN?nGp#4xS`n?oDf3XRAvu!<$e7hG{?M<<(|<}q|d7wf9cva80>t~&L)>MiW5p{}DI zqM2J-niSR5wEC{*E_FpX?kiJ_4iF^jl{Ysw#b9Y_&~8(MaGIh)H$}T>YUX28 zgsGBtkmy}d+%Y--Cjq(*%t zHBKcdI4mh-FsbE@NzFG;3X@MO69W$K$~ylr?kYgsRm#{^RMZjsYS>s+kW^~cRR+~n zqSIAK+g0$WyC|U@)avN+u^D7=ks;doEEdHgJ;yR8nY^; zW(5RfwWThr-8dPw*pB0-OriFXWi`7ttF5D1VIrzmacHcLo0_ee6&WY1?P^)!r%OG^ zM_9nu>fj{=gmDkatY0QH^p4!)urXQo@YUMC3y)Z0Qydb49*1NUGEd1&XZAcMwh1Qb4JF&f zDU3-KDU%{4C&k)NZ1}CJ*Cs`-PpZ!#sbnlEa7-;39U3bJhwAW2)lS2HMhnl!_*E^Z zsfzPoS6+jgf>n!2Rk&zXm|a!ST2%;GRr_%2%DufUSRFS7UshG>s0!b&DvqkEhEn;` zgmx{~&8nTPs2Zo5yD|i;QiLZwzk0d7T4Et{_F{Jc&5&5gdcix=BZ=W zV5Q%@f7RkuuK zQ(jc#P*u#ds`}I_8y{mPFMLHyy^w1Mw(&*njGvMj;g#{50xCre$t<8&>*MCBV>+W5 zs#7wv21^hF>O+~5ot<&hTfElv7TE?{HN`NhakWwLdZKE`imH(%u2jH|PBl1xSes&S zG{re-s!_SAPX4Ci=B6^%rYQDJvAvpFpU~6{sHXD2rZ|92O{8cF6=*97z@JWc{Q2e8 MAI__>1pvta01xMU3jhEB diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz deleted file mode 100644 index dc033d68bb1a060c802e79020e8ae82cc24dce32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28321 zcmZU(Ra9L~6D^9nCIoi~7Tn!~ySoH;cXtTxzLAZ)ySux?MuNL-AZRY%`R}K>zO)mqgxYgUs)BOq|ofE6I1O|4uUja^MF8C;!>P0Sb^+*}>qTp6uh9PA-3^le-= zIXgZ8hKaYwYXk2MY_I*X$+=C^*~zlAGqxVQ4bwx`uD54Ni&F~!ULmEGVGt1@V3X2* z*?TOcz!*eQ-t6}Kyk94mD$ZV|Cl5O}e7HY?9TaE%KMq0JldBf93j_i5HHP|Tn#()> zk`PkD7@;|HJCG2}4o%wMC#$B9DW9P{hysXblP>Ny$B$R{A51@5D-RYu+4TRlb*FRW z#9j-0K;ho9H%OBSLL z;Z>Gz))aLm9}pETDdpD{XP%&Za=P~70jHg(fs7VI+POe$%eCJtS?j;UziGW8cda`h zf1URmyWFbn@B1BFpzeR_Ka6sxY+8lnOU8R@I(;5%tgU;qb{tl&muw8TYo5zeGDJwYpCys$OH^UwL=2vak70z{n7)Uo>v>*hq{8!HEXV@7pg8o z-@4uYTx?APX)~?tmwSi5S$om`W7$xDI9-mP-q}#BHR$zsYTn#l8UDE0KdeoER)0WD z%6?FH&YN-=y?4#4yfCbsj{VZ}(j*C79W;nVCH#KF*iN{E9Ssg4k`c7VZW3+a!in%sk^FFsD z(NJ0OX}Bzyp~#hAv-6c3C)|pLSYp;)6-V%O&SacAPI51rTlSqC#mQ>biBYK~j?I#; zG&w+sy(^dUK&UJCy0dmQUE-1Xrkt*a!E(;)>HBg{JblR}3qiTkeA<%x9_HNaGd39~ zn~WKo7q-@m!3PU0GU2U^8QN5}$GB79J}~5>P2ZRlERIN@H!>v1b{bN@X$UgwM(k&6 z$Pq>ke%7Rt&lnoCsgNZ=`v)CKl*?MlcbJ#~7?QaC-(GC!T*rjM;Xb2IJiC7b_wnP9 zL%5ZN#rvF*3uie`&+^jxa&QlFpyb46zCm8(M@2gL5b&{c+oGKCc2oXrzqAg zZOBgFP@rA47k_m8l*|$1!R|vn%`qO+zD`gGcR7JLg)7oTz)Eu3Zd6z&?~`(Dv#^ik zW#2{c_70pOAQAC#o^Hnbr;BQ{8v73f)67O+JWH#Y>q&b;GzC&3Uh0;ng_gv27ggHb&-Fm|5JJVem0`Ya)m=MIk z8w{ifPk>+4u1G)41$K&^z{P#6kqRhksy*0$Ph{_qDToO?Z+Y{-T+oedmJ;!c&zrn$ zaP3-rg*V~#k>W~k0W)HArQ3rU)l?{bGZYXmLDR4ekc;qgPgH)kKa4RsQ^9r3DL*;) ziegWA7=r}7uOcyXIaZ3IBNjVD*Z9yDJGsA)?zY2h-|$S42*)$R@sr$a5eVQ!921}B z%Lpv&DXyODPPof7kQn`MD`0ZZS!_Aod7f@5GiNuA% z#tQ{5Y3aWX-YKj5!B8Fy6sPD1{RR|qZhBhFid1w|EB_6%qpR&ky`lMM|A;@nxBXh{7E{Gq#Lqt!FOWtUFU!?9`cdJ1Zcuc=!1K!7Gpx78P8y7AODV z6|aN;53lGIB#|XHn!p0GpQm%3@-TAda~nHNeg9EC^Y<_c5==%{0;+M&Z`A}u{l`{o z4uz;J7J2qSRm<`6C~LApGaYM?{#|bmti9fR+EiTUl?Q^EJ8{S7Bsi-tQ)=k0z@l0~ zyM=xxmke+JhX$Dvi+v&L#%cO)D$IcliUeX|#ACHUa-j>k>D>SDiaN_=`u=nJ_?wHJ z+Z0#hx#{z{XUBLRmRgIfohKwR#OmlEf=lHa$iSo0+^^aS4oS6HqnGOXPGNlvC-6x2 zbzP);y5XxluOw=0;eP`7DdY=zJHyj3+I>dd;?Ac3dp!N*QKHAHXfZnXu}xm#vNXq9 z!B$-##=D*9=L!FPe*lKxt*}2RP?J*t=H{(CH}E)H*1iu2SUrQ{r|0k8MD*O=X6&QD z6^8&2KW*dQ&^*4`Q|~ID;_=orR&sqkaO~Spk$@n6l4pb8HHH*WyoBNpaQGbV`B>xZ zVjgAhq4iNLa7lIt!mkSsohKS$M*!-1ad!YP?|xf@6exjqY)gzj9)k^+BAW%hohCvk zr$k(BCX)YCivLrZ|0z$?>$a=7T$Un_t=+ip|0&S_)H_R2t4GKZXPJs*@{f=SKRP=r z76Ia(s>he^Slr48cXWJnMZH{JrDy(7FrI5>`AD=3+)dmqkY*Hr4dt)Jr4UH|X! z|Gz54Q3ZHwjId6f=$C}*7m!CUL{B|_Hu%R6>A1cB#&DfNaB{B6A77csfN_*7EKGxA z{0gVGypH8S^IxMh=XwBkb0;!I^%w9_#D!J9KTm*sD&J>S<`Iir8QM|Os+OMjmfAhj z_8~0P0l($qF_nbe;k0ZRCrWqbH0a8% z!=kJwwI=L_A@?KUR`Fsd$OxA|=brh)nkiEb)(fK}@s1rvk&%h%1Kef1YM~^CpjNb6 zzA@@VlE6*~5?0|omMo|*h6%6pX?4ot7R*3J=WkYT1&=Nz?Vp}N+>IqoWP%q#j966^9122Z;nQ--mjsMYPYSz!F@1$^!F7CkqmlLW4T zZfI*(c0E2ZpkxE-`Bbnk%~3JVrHd&%?+V(s9lfp-5veRHjcxC(#H5D|DvU0(}yKpF%}Vy+0&8>6nF*tDrL?QMQC z3BGUA@3#Tt50-91xmB~Zy}5=v^w@?}<22uV(Z-%R>kUKt;Nvh*t373ZH@SeY7o5|r zAt1w|2-?i8QZ=^Cw0peM}o(nQ#fqZ^0+1^KYe7p}+F{GqL*cNjQEqItS8EGGb zN2-Zth5$pFPdhdFE2x73Yk)}u)<yYM?NIyrSDbB=L3hr)j!I1$=a~A~#R&CD z#Po!tIK>ISS1+AtYOml*blc;W+-%ffPZ=OZzBm0m5Rr%s^9Ex@liUE11jN|dAbzi$ z@jDcMQX-NVct{)$xdLCMpder2occiG;a|sa_w_~EvplUxewMuGUSRMn~p?Dk;f6iO~m~2cdQFT4}7#-QXLdDMhG&!ju_K= z!r4}Vr0=|ea9G-vimtH5#-hC3W8pr;awk?@N0B22LO1uDxaPOsWPDlw;~7KThdE&4 zSj%5Bl7WeU!0#Yz>Y5s(>kgAl2I1|ST>R2{qW?WaKu*HRk-q)}X>u9(BgsPI{tWg+ zMM!6is7wJu{k*TRX+9n47u4Um_mfPyLo#|gyr0-KBPIIPc*!DV6sah;W+sW^U$*#? z`=OJ02FAB>l8ukhpha{=K33k5TZ&TZWhHNwf#n+E8x%Md1i{)2YVbCyKY&5qVbdum z*rVp6vd7=fNfvt6V(e=AAs+F@%(Y^t+sq{E@MwzjrXjX<{VMKcJJwOYe{tR@4vK<4 zm4nKkfy!7fe0PKX+BaumBr5(F6IyF}!bBsu<(u;`^2WQcfUM9@BeRQE1&Z zUbhZ3bT&`N_(#5N3Bzyd zB^sfW$(s^vss&Aa3W~uL?_@i?jFtgO<0J}Cl)<@k%0SD19d3W5jYG>5yqE9N5(!nC zZIDjbnMLSf>oVirWkbKxW2W^XUY@ZJVRCyhss^Ay?;GnFVTu~R^m3Z8g|f>7FiNtG zy-_;^G^oZ9yaH31>oS4aR|55!ANO8iH?c|{nJ-tkd5oG1OZCGOW)-@@nQ)Sl=e+6V z1KrZ*bi3G+7GSL#2wrF()oIg<#-m+vC({WR)i)YBcbMXSLT_g;#D3h<>;>U?>u}^i z=F6#>AyNy4QNx%pfd3lI!xD`AmJS|G9nMBPVC-bpXnwrRYckXJsY?e|rf=Ultuu`8 zRCRy$~bm;;$h)p~iqL@%W;NmxotO!HT_I1 z0|`gLSZ8rgZg3^CyQnhXYD>jv4DO}_7DL4H+8)vx69s&DV(udZ{;s%Y_#+M0Oct`R zWjcCeF}r0nkVSrw1H&e01jbV*s)TwV zJ-xOofsPw)NwZi&bl`>SpsoO6$V&8#cmqR6r`Uj+b4Z!Ct3jq*8yqE=Q7M~H5#h%U z>$IeMYH26mZizIQgbbQ(aa$%Y=}fZnY2yvlVzqGPP_u*qApEW zD-f_tjCvu*(grbvj~LByyEqWYj4EwfUAh#kO?A7S0)^+5*~6c=mg_^abPAXT6LlxY zxksgHG3R(=6H#PZAJqDhJkm)?6UcYnHIE^J7Fz)t!~;p~>7@OnF;f`HHNhoe+@jT` zORWehR4aVx@b=9IeI`97#3uK&4vVfjPqV+}ijSQ>ke&}AS(bh72GuyBWseL{%jAiH zkln+cI^?>9@j+P$q{NG*yd@O^s@w#Hf00*5EpL|u*PW2?@^pw~r(n+B zNQYYHL+s`YERTd|$XL>7q0sFwvcDr@QD~#SlPYtey#^JhBT~Ck(2`-6b$)?vrgWt; zBtxM{e?V}SeSj2*Un{@=wz#9c|AWwlQYjko!uA}s^$^F1iDXg3314@_&4O_6U+Yt| z{K|nu>drgc8{VQ!v4N0xeNT6{j*S%Z5CoG?oPx!Kq;y>(Ea-B;a z4=Q1**O#JQgC3Z9DNXl%iHx4$C419e+FaMKKhgOKoZgF6Hem8{!4bNb*Q?TsKirh$ zy{}J~)l$#sgc`C}zzGuG+?WkNW}t&X8O9uY$DJku{q|e^)l8}F6)4b^_^0(V79m6{ zGVu|AQqVz(6Sz2(*GxD4jCp%-J+&t}VPLhKu)__yeU4egIbNON+3b@0+9sG9RXpU! zXOM4nI;jd?U#N&trz&2~yxxuqULvju)X3p8zpwQm|E{$mdoiS&BJ=>W7$kWc$#N%k z{0Nx~{$t5?z?kXp{lHx=b0vL`HMTNp)=Z*JYztPWf`1?!MI!;^KcIJ zOv#Xw2He|uV?+5v0F<5(35N~nyr`)`hQ}}y{M45r(rZ4}zhwXv5m8Ib06Qs@-OzQd z9N186iSWhdPjR1OJ%YOSvXHR_$RF44V**R?n&ujC986vwNqLEdqx%Km?{j*H5f6yH z0N$P5fw#e}fz8U(jophh^{~Ox0oaTYn0M)==@YyBK5niCA~%D=xf^!Cn)}mTAnthy zFH6+`b2z;bN7VV@Q9gpv>qfOB(d;-B1sIwUnwK8RHED@u>hg-=MOODDt^-Rcipc%j zpF1CwT}Y7jrAi5F{UBgwj%E#nS1q8MZXI**HEb??L9v!ZsRu?m-JV|=6=VuI`K9NI zNsi0Q()eozOz=6f>Obua!EI~9C1Yyz43XeCQ%5e#lI!L{1C_PsSf7G^e{xc#95`{a zEHVzV&?X9j@1d?CEj{8#6))a_Kh=UdVlgbHx9WrVDpZJ+Dr7=bt)JIiA%X_4I9CI` zq+_BU%Jobi6<}NFng^yp_ltz>73?$?xd5I7b`|An*>8*m3y;8L3am+` zReLO^W6DrL1jQxLt^K4kuTR>y_=Go^PHH`$aw37!y>h-Hf5(EVb2hrBCR5vL5HYI) zzQul}Ye=uWX6a2y5L+`)bIswd5Euy%z``k5XRz|e$7r?;UrPD+f_hr6yFk!XE&mIP z@k?Xl@-mX>q?vDaS}ucwbl4nGl-v?2rar_zUmH&$shLI7Y_V0i;;xnWU|Nxc?j_Ww z+D^DZ7$Se3!SiYM*`cgbVHb8QZP$!A)d`P)n}e{cv`RY^xI^G3bBurMm!bIMf*aFd z{2k%<8OCK-rvM8_%Pc}$W*qLt`bl)?;3fd1n}B~hq`>zHAF-Aiekwf zJMiU~jKPqJQfcEY9R7eMunT8zql%}OHi03Bx#-n6bVIm4M^ui& z9!ULiBG(MJ7o0M^0~_&06~fQc+au`iJHoMbD+{cA zZKp81Dtn@uI1{Lr*J9}oPwh?4jM%ZhR?ZCVG?0irKoXv$SVi~?yRLIxcxQ1kK+XwY zElMtEDb(R14!i_fK;3xxg4dQYmVp@_&syT`^G)^N-EqKjp>K_FTkXz)%j&Em?smZK zS5*>hRmFvH4aG%kPuq-;5#7J(BWoTO=Pb+Cg)Geq>L6z^$w30%LK!|#f7b{Q@hzhq zZNf((WkY)#1_ki3ftA6hDhzYJD7}`MZ?m%c{9yo8{lg5HXXzSkSj~$V2UdfBdX%Kr zdQB8d#Hy&)Q)d;OVaqh7080PevOiRY{>y3u09|qc5sM<$xX#+%94xIrzV*le^@ObB zUSu!QTl$DV!@EGELdw`UT(E&U4I!iv(jo2Mv8T8W(m@*D{eEMuJQ!*jSspY zQddp`VvBnT&!K_v`ULaye0Js9DY$(B$|GIVIZG{4>)WJ#{JI^@VI%EP`v{r9e5PMK zj14~vm+>PCy~tBMxoou(a_8A#`8NiolKyn!GkhXPRFzK z@(Uhbd}90->AQ1z$Cze`NVe;cybZe!O6ag6;6h6$>b8ZUFg}rF1i}W; zv#Heu&7y)YCxd%rKqQ~?9pAjzgIWb4kXTfw$hEP4jo7W^GwoC4Sp3LqYS`6c+T zhw1ljE-PD}nD4l`4Z&SOfN_U;PNx%$yt0R__Rgf92hz-3{cDt!)FctGGQZpGA`6UI zz`JO4&cL2yjy~5W%WXt?@KGe+i=9Bqr>A7?=mKLkF)0rOY<|V=2Ytp)na_Qwv z3-soFTl!4jjIj&6_Y-aSkW4JNAj-`LjIJj5sugpuR+h`4Sy^C&?u9B->|ba51Vd!) z@HkWEG~Tlf^h~CP4ojjdE{S1f;c1-q`-AThUK3^Da2r1!Il-c(q`|-K>=J?zcyvVu z2h1)hOVHv8wlb<_k(0NZ_u>*nim_5-jLJrU8r==g( zmAgG7I`CsL7m-v&` z9sp=A-KEbOCe$3cRvZfCC8bQ#SxhSVY%{wR6Db~m6=jFsXLhVr-n${3Z!8e${rTi0 z*9{1YOoL-m%q6M7ug;!ADD&e{!169|4<}Xh7kQwHGw}A{rECk|4aj2H{wV;?hT$g* z`mip1I8*VUBWFx4!k;ExVF6arAjlKASkhq3!h35T4$xg-T)AkyCDm^un2#%|QZ)qb z%D}gtSX*{MzWl}{eT)=DW5@X0ww;T7K~l3p-lSB6aQicZ2}iW14a;^G1lSl`&Su^Y{ICcpWs$aj=aQ%5t>GkYe z?nxN;WNsw-$yp^kZ|Cq3Ym6|jy_bbt!$fU5@`syrECx?))j9@mN!4_Oprv~HS6>x- z-46}?7BCv)-_=Vm6q0s|_<931=HS@a9{hoRgh#g(%gUqmn+>p8ion22mKxB-mpB<9 zyqC4~>kYz7?&Hf62W_Zj%IUO8pF#(fy3QY@wQ1?UM4bH=#P^Pse$yPZPF*K!t3k#l z=#6{TFsKVXb}V8xW4}-Y6m^K}^BDneHGI_ic@BG8ICo4u9d5gK9|^0?f{>mRY5yh(10 zLQ<)$9*f$v+OH9I-L7Wx@CRuVlG-_57}DbeWdiqy{GB7=lcLxo zy7@8A!cEXt+y+^oeG2#5fVzg9$?#SxO+G?jGSCK{GocY<;j&40xbcY&)xKjasm$DRmoPoq^$D^-tynu6};Opvv70UTybIzhHOmX6(i zz<&pR0=qH+T-!JUq`#ljYo1s-EuFQx_`EGMp&dx;7TaaLV3ahV_(Vxu(uuauhhQVX zMm6RP)`FF)iTs^AAlug3Hic9hkgq4a8pD`8S=PeATOT-ET&AIwxs z)GmM(pp%!3Wz7eEqEeJIebet9yZ_#6<;xex^+AtFvv&-ldOX-;Olm)x~u9 zeDsp$WQn#Uatkd@qIYNW;}yzvr3kyNe)iQy1zFoB$N-$Jz2|I}(R3nYvk;+sQfaO7 zN2;EA!S-E?LuW;_p|^E=fEVK~O>J!(ED`S|>koyo;f^P)={1Q8I=0-@pV0-=MR#h| z)*FQ*DvpY+3`AUw>2Q*WA@-#rpO44-uKdAxFVSJI#X{AU7&U%=9Nqhx4sHz#NlGIW>~f-^S+E`NmF~5sRm?nwySt8g(>@X6F|rcZIP&WZQJTu5Mvy!MdG!~#Yo+0_OiS)TE~DMkuee4HICBcHKq9mn#dtpZ zrD`e$CSEDecgmN9o{lJ~LZ5aKnyb|wf17nCH(r1E>)>HPw*2b5Z2Ee{hVvFgG9$#Sl{cl(C zm88u`4wehc%v!GH0C_ts@)~U}~)a!WX~u2L~RlLwYafw-Oc-$kQObl_oYe z64cMfbuxZ;&(;iSDm|^{mq9dt{T|KYxKrr84t2&Z-@Sz=uAjxQ)xjvxOH0HmeaTqp zmBZ&Vgdf?M+dFAcqOCq5dDVAUMoBVS=SG}9DKF`NMZ>*BCpS5vuXEd6sSY>}{Auv< zoh6|A8OpLRwiGDiioFW8lp{mdJY*2v(wWbX$c^2y|@dJFS#v@S{bGB1>MZb6>gD) zO+aQ}FgSmP)LW;f|GQojumSw~r2F-%!i0**DTTt3s3WSDVm1kC+LE3N&dsZ-c9U27 zskE_3Y;XFfbSP&=Mn+ z>jI=K(+xrI;c?KzIXb{G`d=LBQS!2k)I4@@~zH_r}>*sgse!;B=g9K z#~NbZlO%fU7S43 zH^IwNG%18RjR*7%xd_8wV)`M3gk8jS+NG>ue)G*x?tgcNM^%xuh?9Cg$}fuBrRm6) zhaO!FH}ORm=6VrW_SEeu^I7&>UlT~MZ>Gf0R|ii%oX23xEM4lj5_e+8B2EM=73u)7 zL=3iMmts|7(r~wE?nu(N-%qpSKi}U;Fn5s-ZikjA!5PFY$=6p#D#b&UKh)Nv3ly?h z8r8jy=8kKkxh~`6(>NVz>6Tb2S9jeJZnmURQ>5aUa0#MOV;ieMUM9RR7v&IZ>*=DZ zb=SSa=i{n1M&(hr*nG!*D`s$(R4|!~JcATGzF>fI{&B#Lq{_-UZ`_BlEu~|U4~A^; z4c8P7E?sqi(dbi#ML6+UhJLwpxCUIUhUi(HoXA7_qi#Tg1+3cz5bQc_A6CN7+8t0( zFk3r+pR~zX<+^KJ9Pl}J;c~jeSCn20-INi}zdme0()=CSH6OO5F60#ky9F@l?%~FA z#3iFM%kZffOSdi$C$c4^7!C&mlC-^%gG;kY2 zS;+S(-mPGrKWVx$GwFnck}BtQqaNuVwo>}e0;{qrEf)?>=^~0tY@OtLk}sdC_`^=v z!%$bV*!qO=TFl@m`DI^8T0r75*T8-=+SlsLkq@s?cVssj%^Va%6&BqVv zZ~=EOrRE&nrxD^sBM)lZ$2&^WFBvGs=P*=aUUWfG!(sn&5o6lDcsb4y{uWmSC2a>W ztTJe7L1!jGkaSYS&QFYsGdrZ)kb5!FA>v*iwQrY?asFbr6t?bgy=KuB4Dfn^L#drX z9)UFp|5kF~Z!<9JF1cs|R{p}$f)rPADrIe1NAP815+s63cNyKv86Q;|b{bd+<}HDL ziaTQQKS{MRCLRpG5&`3ej{FjJ9iw{El+WdboX3QuChw@XRnJ=S8~f&BE#BHUx#?$X zU0Y9^>T0c(rGT#jhU} zk?9BJ?&o!bdWjwzuA(#6*UIwF3`be;73e5@m74XevGcFz55I5G=?zi+B3bU#aHk;? zdK=xgQ%&W94C0Mn1L?#)0&4{N(1JZozOmIDGdRu1(9&ijq;D0$U%!|D>|XUIYn>O-$|ath}dJ(mdFnd@?;zzdwjbV9;=e9+wN3uQ;@*e1Me)9Cg{O3kha_HT1gNHIz1F9Y18yLu7iwS~?G zPX0$)B>;M~Z(GJB^c1|ElFl{d9D^3WxKuL;#n&SeD)$&@7Yr z*-+0ltX`#t6Vr02hND3*xR}{<9>!q# zjatIUB{eiENc*OT-nHpG(fCD+X3rO=@}Q2XPR!a67l^UzNp~otMUusKnIv;L!4FHzk znmI47l439EP@lya+Qt1Rg~y}}&8K6MgPMB>@d?J>J_gtit8~_&v(>;p7eTWFpIGO$ zVj#t`vQzm(Z>XQ+EIb84!03&9xu|4r17x%Chq5d=ov_$uMK=6ST)ty)V*`eYF}tb~ zC(}wf-}2z%rGf5H%!87JcNRt=$tR?l62V^lg|N84jown#L9@0fgS#{gU zEACoISo>_fVadUIXpS$2tUMLHgQd6?T(>Sj_cvi-NnGc-c#T5bs-t$df*q7Dq@4A_ zQhfUJofqQX-@covWb8c zymx%wHwfS7w(AJ8AIKlD}LTJo+CDheisYSlQ*8~|5lq6RVYVRl*W-W1E-xOl_h34mxa zYO`7IW>_pn$0L|$lDo+-+?`+5Ie~|XV}Og4Zzsp=lmO^#f#Kr5K2V+T)^Zm0SV#bI z=%}24xl|Z*`n5W>UN2CuH##1am`#8Hl^?N7T60xh%=A?%;%;Eb-!Z_f0vhpZ zn6aFXX&s-cCXNtb(s_R!ctuwu>Om@=zRZpI(0apUSMS5iq}(C6(WC};Zap0X`mBNf z?X!1usB$i0=fI+>6dbw6K`ihg#+`5T3#fK`!UvxK(nxR zSUrOX#tgu|R`s$7bk{xM@h1Q~h~KzI5_6uAjD?;0>#Dty;GXVsfN>{A2k={n@)D^! zf^qw9Vrs{kx2j)eX_t*W!-=X|(>Bb{IoGGp+xRBFj?mUb!Ey5_A#N(0E?CRa8&h-S zQqS9uxM?$MpV_dt$_9#?29)&v22#)LtSiM4yGx;JD|}oDeAv>&m*4J9;ApyB+4fB&;CF5%5ZrtWdFzc{;*$|OR0 zj@zY`)}al_shr=iPg`oFhJ^ zyKszA+U)XAz!U-JtuDtU)cAT}myz&XTDS0qctDWkLsLthKR3`bf;R$@nEAaNZqR2O zZLZ_GBI257CI)mp{z`}E_$wskIst#Z@j;gb`f=EB@Z}@yso+KZ6(z?fcF2&bqF~>h zhb<5twCDy_CR|`PO8dc19*}sjP#Ze0Fgedr(p>$wvnb^Yk$3Xw7xkdq6g@e>hh=ph zfa0@dT%dxl*K{P0Q6kA62!z_wFD!AOC5YqyiYgWL#Ah2k`{iLzWxnS+0@~is=xua@F8iSaC#}T%Tue@yWuYTQ}+Kt@d7O%K@G!^rN}%T>ZX>xZctmY z7^5uSHmIC#jqHmQ{c#)_R-(8=dRKCBDGVLuYvD(vlx1LVg5D> z)rmOb)c(RLcjTp=(IgEBV5f!P$}$}^e*|s%VzuOwoF@CPme|J`G?A#@U?1ngDF-d& zc^vZ}Wf#Fc*&=G@jQYJN?+Ti-o>mCQ7<|{ZK68+1P0_Mjv;S@Loz@k;$-%h{swt$! z-)p7@c3x$l=)m;;_ObgkKeZwa3H_8MacpKf6Ix)KqloH>*spAT`xYF3D|6=>r2~L(4seKD z4h^`Ddnj_W@{L!6qbg+$ev%fG+%u@9WOS9JgC-7Bqk=h4lT{2!nDBJa_iu@Ll3}IK zqOhK&OAl^fL4d+%^x2h8&iF$VNhrai=HA4wBAv+Mgb^DSM;!i}i3k$-&->Jk^}0xA znhVjms-DSsfd16(T{YQD1U1mzPyH3`Xk+)5x*q@Nz^6*(c}`!w0%tvbqGK5*2>cIu z?h2S|8Db$F4-a zY#on$G2a~acC8q)VCFVE2Ko3DkX*GFFRY3J(aM3D2S# zSsjo!&%5${-_;xev^BnkaCUy~pL&6TlAY;hlKwVf>HWjuYURgMwx_%Ak%$7>?hVcb z;r%vLmsJoa@1RlKSoUIT<<6%y=seKqXAi6r4y>1p(8p{-@#n&NuMWinV?C^TLT|)n#i%egp&#EWciN=Dp%^*j> zFF9$*q+%1q---&|wHdzBue>7VgKy#)J{6G`FA|pBW6}x&vbs3qR0az_%lCE%9k$Er zJmj@c4u1BR+mYQ$v@$B?1=RJ70?+i90s%9I6BL2W2$L8cGlsiM8s?!=3(G^WyF{tw zBB&2ljK_+lMGUk~KchCzt3|p28>!{tV^k;jD*a!08kwvAvV-{f zN>&SzeRwJs>sw;)9#erBw{);o_4p4OXZ;x%>5DKeKmTdR3T}xSu1hl!EXbd&+1Wyn zx8W6&E0xkS)MPSHZa0du!v}~~a}6F{zERDepZ1QbiTmKA&(3JbYx8w#JdpGOrmsvxibpDw;Ku8l8f7k6!o zySux)dvS_WiaWu*xH~QG?ocFHaSsFvv_POZL5ml^>Hod&{ct~Se!Dw4JNC>obIv$^ zPi(=yl1xJ*P-RVhq1p;pG(gREjoU!=crY{Y^Lz8n9M`D5PQWX%5l*vX^>x=!J?T$f z_V1?gK|;B1zOrAb`1)M6f6b)%>#L;|RbJ`1kt$x9#FHMAOh!P71sDu+CciURMT#Vv zI^C$yz(B)Lr6>or_q<^^<6j#F-7^lE&VIGb2hLTozQP(1E=_QDF;y250Lc0V!d=rX zmQo*RGaOcLo3|snReE3tH>j|Q4|L7l{9W*7SX=HgTU=QUyLLaf8MhhR@jTZqn&17f!s+(I4Tm{`n9C!x)nCs8MkUUUxOW*2Yt5 zw4Vg!fr|CdLqCTmJtQO9PPU8@X|$o=?S`}+QUSEOy>bQPBgOSz9_%Aj9R;qQ7Tsi# z5WO8eA!*NGviwR&ZGzc*x}c8eI%~itoP0~b<3diUSant|u`l|gh4h_ic1{XJzn)a* z5Azaf-KoZ+(4?2IkgM~T3CKese&Q1aTKA0$5iXMNs({D4 zp6vC-0DFM8!nxt-&%>TH&}M>$>@SByzS$e+XWa-TZh3;8+N@}kj};eFHIgCvpNx)P z2wy6Ad{DcpUf=Z9Exs`Z#py)&0_GXz#dN-`(2^SXKf4m#P3ah@7_qNy-nA4ryvGDgrQ0 z3h>eETJFsP-QPZ%XN_xl7pzgjT2k-Tg8GphnjaKxXj5;!cih7OSj_xung-+KXy?7Br;fWr^aOk>qN*8yz-*0;R{QAt&ZJ7%|bkH&l zqb%kG7g;vj{WhJpu3jXZLHlq!O_E@?6dJd*^6|q{-71i)Y0!d{&Ucp$g`UtT7U$_f z_}>1=>Rk9&E=2g-`= zMSroDzcc#Z92gRRhrj*gJ;7%w$^YfJIP+>gP|5@+hjJuFkLfN7SMnVQmlvL+#C9V= zRAZ*CKxPu@1heyu`AD|~LSrAYw2Qfqahlz*x&5J)BYF3lbM5)swRbot?9`ove)O9rhtbE4<%kRRv@TqJh%Znx)dXMakt2TMrobhHF6|V$E|UVeC8WzZP=tP~_a|kfHl*LriIgtC?Eb~Y7hbg1DXh^QJVSA# z5#^Ek@?TQ{M_bFbqMQ&8=h{(QYm;uy3%nXwgs-DmbwHfdaYG#>gV5&S*17=2z!rUp zXYNxD3Mo{(coT_;Rc_6&Y7PN@H_y_|D?Y8A@MH7nQ`bHo=l|pN`wX7wF}IGk_0!2a z9ij+a&oCOVFNBL~_l*+;)f8D(@G>j2sX$g0uW}}3PO)|ljgQiOb=zn2Rz2@Me3$DB zFsuPbUJ9N1i^SrS4CUKgA3@{8r&Zr-$FVN^tiEBB7grtW(G?Asn{KK#zZ|ltU*I6= z8V1_gG-U!ukjMeX7M)09&sx7+Z3gIP3<@I{OZs3%c`K|a_bKe>2yPX>q0^>lECCKF zJSzr#(M_t=+8i3-A3GbmsmBsi0WAg69|W!N>|rnOb^IEhgky2w3EJTqX&{&*tdXSyWXe*xvWO7w^P3^$T&?c%q19XAEm; z$@$IvMLVy*PLm~oilmSueEdhv!^kY|t@Ou6p=fmH6!b6V^m@9$4JZpcW+hE2a*?8Z)iND6F>`2{xc!qtnr*u)w% zX!@+UTy2}pu<)3qUS(U>W?ZCis$|{`e$=x9{kTOi@5WzjN@TBOBSKm3Cj8%V=+ zX4|lQ7?ZX#P`SWRS(R>2ZCtz#YH7kAj>6#}()Y}6)cn|Y`7A?>byC-veS@Ib2Hrw5 z=JLv-3#&^^v!5MnK|!qew8Q=xejWkBvv*VzYo2sG@i^?)qq01wyNhjd)s>84JaFTZ zSP)RmZacdCu(J~|nN>~7jb?c*Y|6wA22`;JIH-9E^P6IRzTf!W50!Vc#E>UlyONLv zSpNCp5p41D5fw$z^3irOOyTE?>I&O*?fmxTBFJWrKKbuITIot>_ARNHu+e!ZXy<{E zCK3Oj1&J2c%3c)OF0~Fbt6EHr0z6)ORc2H0CmFc3@d;d-3H0`eT0kbS+zBJ2+^j+H zrk#JJzQW^n{#bEef!0xoDIdhPE?xy+D@xwtEduTYW%V6wcA4OKGz#FoM=@d#R2O|l zgaN%rKuSYKj-|xG>Vz`Khh*4ncE?}pbh1I%w7x}})`tio#xZ}Fv1(d3yo?at-;pG` z1fQ_a`wFL)s^(1NeXFOzNjmTJ1`N3ZSvJ#v%sM3=ZATv@T3R?>=6>vLEkBUHrxRI* zl+OpG~D*qDEc2curHv4Ak6I=BZ@EmqBT3g3S;+lp0hO$k) zZsA;_HZS>Aua)F}-q(%`{rk6^R-*N4t~&U2f5PtolU7RJn|iPY5w+$XZB-{8^CM6q z7=PxpuyA6cnOG+92U?#se4YSSRi3BYYPzS7vC*n*VQ}Mz8&lBB*N&jw_9&lG;&N5* zJAZ($yO{JKNZ4yTPKOdsevV=Ip!UEM@lMo@D%CeKy`WDM&PX6a=1~%ttg)^F;pb0^ zc~@tjtUy|)4~LvhK+HW<9&M}j7rq8c1%@AG=J0jej?7= za%xDQWr(;BuM4>Z3;SyHHA$Wo1{gdtds^b(psSUo(@K6>`u zyQ+-7vuD<9b@WSNzlsVHUDoYYCTZ4U?*{*oa`gVQ4!DPRPVgR(*ok z1hH%#z?w+fPvTXkC;T=&@xISJ$Zn~GG92(!1;4=2zNsKnM|K6U3@ru zxRx&9wNGa?@>YYstAlCwPZ4kX+EzEG(s0e#HC(tu6Bnj$R8%?~s+_x+9v-zX|1i0mXCN zw0=@q=Kd)VM_aeb>!Fe(T zUbe^;6}k!pM_+xK#$E!xC(21!7pUM2&^dbVlu=IpHfGl1ur&?FSlD+NX{Y;_-C`K7 zA;8Gbs6WNql^J{-!57r-Bn`{ybWPtzP90iQB;j(>e0Kqi4BVjW3NC{!TSm7vauiaP zDK^I{{0*y%P(gD_*MS)!q1NKj{3J@~eawG)`H{e6E zFTA^)qe=`!0!g;(R0pmmmepidU{g*47TT4^wDrEu@BH9+h}|k7gx%MVe}K?2Im=mY zYKiZNqDVmj5r!8ydRGk36a*moTSnVru`~#XKIJQSWDA*FeAhUTe~o5@daOT$4RRfP zq1y^49JWQ!m<}i1L_Q+Sd!aro1Yoku!G?q%Gt&O)qAVTi>aA(K=fOJX$W`~X@A3h9 z0KVy`+BiqN>tKaAV?@>RvwbKcSfSV=OV{|x5GhR_eO!zlK@=@B5%MNDb9Y+&$Tg9P z&*z?z=C2vd7Lk9zCTIHoK=0lm!KSCYlBSZKO5g)w6q)XM;l@d|A1VUO@-ZO7@Hd8SWxQ>gU!ly- zOc9;sPP%Fc9s#1FKnmRxJcM6&Hy=7}Toa0N3w;o0l4+w8CUEFd`xIoj-$t4V(-q<*YCuVa=a6!5&z^Y%KsQS?IL+sxZ`fh8BP9ue*4rVqM( zZ@CH4;u|9=J`TJk3gV=zPnDP?7OWm%sPazi5f2jeOK`VT;Xp;<03r|H>2~&AG6EnT z%}McFFH8Dq^(5m_7jcOZ5j+7l^!~P*M&d&+5;BTbR?=-#5AH(Kt!cY4eN;(YK)XY8 zUjyaRctB=KuvC2yvGl~GoVGu>1!Fl0?wE&ED($-yQl5i)7NJ-u(wJHo2?|4%o z{z4QBp(v|q=|h0e^_>$wKr;Gv84IMu=GeWuA~{QmXJ{lvf#~URPOzr>bDuzz!u7KXw7k-V62}PlG?MN5wN1X zO6-_%ZGpWV-K2r9r>&}07T~aJOWGNiRowk0Rg-(@Y=GH6}R5 zm{LmmFG_qE3sTgie8am^`BC+5G2-Xq2- zB;w}jrK1I*$()L*^4;Ll-}uA^rTC|e{!>{MdA|$i-Mx*Q?o`rXl!o;_PJxz%n}u;a ztGWK8W3B39lPmoo94_&4o$Z&o+Utw#huT`$&Q=P0gWC21SMpsdMI~Q4@~Xw6J#~I+ zF*G!A7)7PiWEt6*?v+w%p2B9qchgMq_aRyS-;c8@Su5vFOanRBb2fHKqb4LK6B6Dl z|;`|8Jp8HNqXYALoCJoS`2(kChj8hhi--+grB9+Gn@J}^z! z+HM9%RyKmrJSXM$j$O!YH;H#6PpA6K(MwA^Aae|#pm*_F&%Qhl|7yXb>Nh1TI4I>u zmVHNr6YP1cf$3!8apnkq58r1o*d#>8Mne!{(92#U%VF8`3U>)GPi<<`5wSr0UGb^RnuA9)H)&OQR;KeJp1*D63 z_BihuoHF89Ord7xA41>t9XtKW>6oDSs+L)|kVMiY?gcO7CltXGVBX17EPci+Xd>vP zL)j;NcDWDbQi0Xhyg3c5qrw==&?y|n$u{!v6;~%OlM^ofLio0Frs~3TJlO|Zivfb# z^MoQmJw37FUQ!3A#U?=Dtfj zr!S3H#mzB}a#AgK-hM)AV~lrwo67{*P0L3Rd2?tetJU{Co8PCHXKBfiNdp9bo`~mh z06z*PFq7xl=b%x1aPy=#QC`&H+=@_kZbjo6b|WpENICXxZDy)2skwvZIXA=fwCp}s z^YWqorxh`4ixz9N>uIR~Ujwos0=c**vT%@N6%C|qjh`vM&19t88$)Qz^kJhQ(F?D% zpXlJPx8ab;+cMnbVUY?|6Zj~vJwDtRFQ9gr_;9!N{tni`_c8Un>vsq7M(;obxauP! zN=`BaT(ZAS6$2%=P5Z5x^QR?O!@aOLBpcj zD4NZjf%pi=w-!wxXfx@B@S{EMa<|W?-IXNiW4hVzKwKNfjF;5(Nf|6kfx5SChLx!T z7hfKhVgg$lryqXbWOg8B+WMQZ4#tf%%^b(uQQ{nX>yg-AU>);6&`~cNReb}IzNs=Y z94T#X^s}U0Rsi}M_M(LY9-EZ*0k@_q46iVGQ9g~oVe(QST9e0;(M;A4eDUU;gCe6K z$R;<|RX&s)n%N&>b#;q;M^iA|n`1-=*{5Lw9O2A~Q!W@&$d@n5JlYgvK)rvA#TyF6 z7$GfW;m*j3YNsgzpA;O+r$hQkp{)dR)MhW>$288`V`V!#Ix`gl=BDD{X>z|c0&jG0 z*S_W2O766^ zhE6E}hXOEiFF~a9P3|~t-O5vVU5FuXH)y1;p+FF^&_y!`LWXYICx{6p=`{PJ(ikW# zIELq0;GJ2@aBe=&t?gz-BTGS#f<*uUjwF5jtQ1+M|;KEa6Z8&J0uRuZ~AM2{oG|be>w9 z$;hz*V-+a&M-i`5HPahV2sljH zQQ{s4)xQ^{ZCX@yR%D8dZ3!1N8KJ!b!esduNYz=k5tUp5*P4_pA^p}9y{@nJM9XHb zisOJ;duz(tQO@Ije`Y~XLrpu_H%2?*ck(GfF&GoEX?y!Z3mOGIEQQ;0LAF0>a{g%A z<4cM0qLFgZ^x#CnKj7v_K5|SR>J}%A^SVcT>j5g5$GjveaX-5~#=kl4w^q(ylUj}P zI*lN(5JX@nM_PE*QzNb;42|h{xN-WolhMp5hT5tC(1?9puY+x5SJt1SiriNwtx1!& zXvI2Mk>41!ztxPBbvrI!KL*tJX+8s8xT{HSRFhCQCo$2zGVTcf@MWU62F4W0hp)ylB4xm zT>&GH)8K|L%}f1Y@-$I*cFifiu-?<1F}z#N?&VBT*azGr^0g!jiZYu=P93`rP^PV% zvPI7v-=_FjQdHO7z2OcfkG&BK^SB&D9Y7THR4UT^#hff5`lmkNPY6?5U@Ncho3jla z>O*p5Uedl#?jU64QDEv~)|@ywsKQNu^7=3F1zI!#=vVtVnR1yeyJxRhs3k23n(-hI zitpI_cze7gV%A|jad8H71!3PCO@zu6gbPt9$9|0OQF&WQU@DHns=a%(Q1=9PeT^>B zIbmy9zYI0zpl(uEeE>4i=+QUW%2Huv0VztjGhP}<_${k=xvRyM(eA0!%Xj&5KD*AQ zUkTcsO`R4g)#i5uaD(vps7x zeH;r@N0W57krwouM*vY2`U~eT^5lFMpFQXmN6CEu9Lw<;Vor?6O8LdOyO3KQW{H;~ zp`vtYO~kxKS9G7~;kXPNG41G2c#7oq#ICw+{>#$BL|KQ*qz!YE-`Bg!JTHxWmG?&b zu-+c#rz%T{d0I>O>=3F7;s0m%g0*y!_%;y9DPg;%+4|3e-MoY%%cnaS`8^tE2EsXg zXadW@k2h%fYlc`N%YC8ks0TB)(UaHcaVZt-lnu)ZNluXl@MB@v!Uv1^{=SkcVifQb z3F$00EY#v?H%8WLl2{4^D%m#Hp|sDk%MHmz$;^~rzRne_ckOe*uFrx* ztu1)GAx9|cN@3lmotj0-!VmVDf|iOY+gLDO8AgvBf?bCA-k)MD3$>`3PTV|znn47m z#zZZRUZM!?O1GS{WfgjznDuN${?heV1PqbiMY-Cdlh9kHo|x;e5S+UFEE$&z;vDU` zWUUIMY_Sj6qDMrfA_TAuEJlHVMSP1h1zaOL4?P3l|MqNEY1zs2ru-pY{o;+XiiI@P z_x5F8jApBhcGdk?WMopb{chIVH~*3YV6~3<4w|{ZyX+AAGj)ZaMcK+lfnSix(#+wF z^Z*(|{lBlr2n$ilR;MsKH}Vx|T3lqJm&~oRdmt^yNClB>{;0VE9io7mry<6TGzDk* zDf4-X^f^l|q!Q^robP9bX(&pQF4>J%EqRAW$aKRQ8Po}pqE#-w&*ZJNeO)S+= zD)Q^f@W0Tv_9s2otA<^SmUysSEh_-=f32E`5%PbnTBq-fEqbW?~_ev67<$80!=MGo79c5aHDbPTHdWRMg;Z*F_m0R%fBBT(dK1BCCGUC8Y3s#2OWu3 zt_nt62(LlqP)?2u^BzUyt9qP!@cxyMN}l0WK5c+C%R>_3y>$es3ps$sZe%9Q0R2S@ zvJbx3yV@+4Os)yoDaY_+yzPqhE)Znl^c(kqoj0t86;EIWy!<*YCrWgZhG5vHZk-Ac z2B_Dhic{!}l^QX^q@8TdsymaMb&3OYUaTsXqOsb3XZDQebncwn` z;-2+?uJuziyZ^54a`X18=SiXZLBHVG{S#Ysy~PrmVu3-lT_OFTYUp6(JTYER45S7;f$Ly8paVyneB=#L&)c@521|!V$Iy6ds_Jc7klOezHDgzf zK|=+s%)n!k+vZl8p?~HuENYl1rF~-cuF4{*l~M3VvjTgr zL?T^?~ekW(^~u;P(-GKklv0ER{4V$huJvO7YJb z>b+(i8L{(4^St)CwoMN*Bkyu=rdtF{?KjLNul$5_LS*m{ywS3b4b28fb9C|Ms-9r^ zCo}ZryLf&Nw8!Aa>6Z3KX6PTg^{t@MRNa?Hr9gL(*&fc$5DU7y{(ID7Q4Wja|E3*} z3_oDuq$VhLUd9B6OnOK{sm^avhBrje8~ogwHKINMbC`?#h3kjg4L-VYCob(Fd!T;k zO3Z&$-fdB)ihBw-Hp-cz)Z<^?DNh-B0pry3xqmZ^!91Wx+hY`R z*4(aMA5rWlJ1FHd?+VqxM5BS7eKsmG@O+c>qiU^iii4PQoBWCEa7ctKtj^3#HKtrF z?!P#JQd_c2dSKnC=PrtyEodco3WHF zF=?Gno3z_Ma? z5<_nzp5T-uuQ>F}bQQ}3-}^(#azri_5XuUxh*P^GH4!oHojK6zfE$lF!TB?a`omX* zg?|`j0XomSf!)8b$(RaSBUG!VC3Lv_D#eWp?G>r}7c$f(-3BPby&vOgG$i{-!LiFZ zWZ>iIaC74VR0kJgWN(Yl1MLcDgCYm=`B&X*A_-oUWSr~uK6f}0)>Zmb`xxWtn5Fue zLK`v>JfZs+?wq0wLZyqko{BlihHP&Yr;W-J6<)rNB;tVFVRfeJ*ftC_)7J)%rt%U# zDy?j<%htV3$Pzv`d*=~$1LVb}csD%$;|5p)Os(Elpv6mGZ8|H0-f))3X{ zcT>H=`Hot)85L?Jtd_WPtS4OPk)7(x1Rpw6CKwiWFI4Pm+W$%785&LCN=B%xB$WWtJmV^L6n`Q&I*a$9im*$pVjZ|VU{><#q znAzj6Y}jvFH{od17nYvUUmEQe$o~V$S4^^m_X*m*Zw!jC5%$=kdozAY0pCGqU*y(7K{JvyU;h0XNT$PJ6#8s&!$>3Dz>4J5~fH9-Z8o zz*BXXeUx#M50Gh}_s$G-)03OrE(1O?pI;&J@(@5rx?Ewm6>d{yUPo+y9T%JWd|~Yc z>oPFpuwx|QU!3A4up3{Ze(EUE5-_jqfHttV7w30|1bVdiU0sHX@? zJMEZHGCg4_F#B1Z+OQjx8CE}E_S0$Ho0^68yzKzdP_JKKcHbsqQXMls;IZpU!NEJQ|(QlmU39^tY5$~ok-C% zec|tXt;egM!y?=H0uEF{+Avo{0>OBZ5s_RI?Ox?1uU}2RBi+`cc)~fM-W;F?GK2sQ zF5HC+{AhSH_5&d7U?t_8+(m$Wq90Doyg?jB@qq8S*r&5KE4FvJg$w|$a?-2DkqJa zyI2zNjeB{Qr^y^PA8u6DTV3fKT<6v4I4J78v6N~uZW?QU&=>$2K*RP<|X{c$^k?JmZ)vQ z2>@ZlNUh6Ma%pFF@l4}Hxa#$m5cGek6j)TF>lxHxZGXqNB%SfB`6=}0v#*3?Yg2EA-AjlC)wb@(=FrEBmCffyO?$jkr(gNS z7voY-m71RtUw99m7DMlE^Q2x*J71q>n>PECo20K6rGj6p+(L@Xw;EJ>Ux&E{W^)o( zLsjfUpL8Dp0}XI4k8Q6|?5)>_gV(=TT(8%i1CLv4n@>=p&6kTcsVCU$o}f(wuYmsj z1-56#sMpTd=bhV~&==&emu;d@W5(&t;47+u$FuQs^HB82JpCWO4irJp9mpClXTREzdQx&#dTyiOkZ?}p1be-Ol_(tJ}t0141J1${plN+ z4ebaGyj~mjJFYO->d~;Hhg< zH`wDK0Rp_XV)&@5b)MS1sH?Hx^=>CnAIhMI`rsh(z^4t6A%XoIggmKZH&FlG*5&^h zQ+Ontx-yvH3`M3*;PM<*ri>kJQv@-B#UedK7&U^Cm~QareN}^(%-^0kYT_Fbv|PWp zb<8q<8g#fKe!m-$DoW`JbG@M;zTSc(5zik0S2{5Gw?tPg!uB8L#_Cdc4x z+rer8w7cN~1eh+QNz4p|7=le5@h*$9S)Fj0%N7>|5D+uLB}a#qG&%c;cS0|H2Jact+=eHzYEHDwL$9e`|DNIwJZ$-H zzN|I%zn+s7hn8?Y&7D&4$giWNn2%P2Uw@%AI&cAhXDySZY|?DEn5b;MT*kPCT#YfQ zQ29UKxrL5)iY>xC@b%^IX-d=Uc^(K%Rz#2du=aZS(3twV!xcKRz4>%}H_$KfuPe)hiiy4St#wPlLI#}D1RtAc@TFcXx@2tJ}BLOzJ45Y{qlP8`e+`d(R+Q~8OA@F?wWb~G+6BRwV3qD+RvPh zz{?})plrr}r{@1USCv+ytEGcp{yQvr_w`QiyXB7StCPDq$Q;; z-UJ*nT^~9;D!Hy>|L0S46l1WzbbWv<)P4MK#btl_ito~#X-V~Z=H0vCfXl<)3!l~C zpzEv82AzMUHTrB`9m(_pjLD{ z_8?X1>U?7{Bj-9>x*+22PT9tO&ei!$^Se$7sq??R$1Bx;Pn!3oUz(mxp*Am$Pb4op z3C+i^FAuNlP{CKcZC8g(jKNpOf7YJ-)(E~DcIh}r<%So!(*4Ai!^1W;NR;$3694L6?o0o36(rIVsE(Zv#Af;_3B}?#&QnR-|HaX>+O<(eFwrT&#Ol& z!K-JXpO|Qar=&mnUHmzW{@!VE_2LntlDcI1a-a8V^epdyQ2OfRP@eB7y5#P!ZEHlZ z`QxhTwvuY^-I}M)IV8vB?^YRt#hp>t6h_(VgR_euN3@tV&&rjDRXo$b%dnH_P*BB-46l6T;I&&B+BI$Bj!q6O!g{?!FxlW^X2^ z&WB;sIUVPJMn@@|g2euE+5Di5ew12!@Z=~jwf0*Nws9cCb`p8N%nW~^n0kQOXMB{| zh?65xRb+WehL*!R60+-bWL**K(0SHtV19M@fn^>weEV;bS>XBEVegYaF7Mu5ucW^6 z?$16u7h?E&pcL)0y{sO*vOf6nO6-hv2G#j{NMuHk@UfompQDofpqKTctg9Evo6DVn zm)nb{Dx-H_u5A;2JikC^1&QI=;y)@{ud7d=Pnpa+vpVsDF13!SGMR|Xt^b-Z-9!3-)h})esI1hz0`hT z^5=V&#=N;%=jHJ+ui8Jh^x%n#h#8fDg;#Y~gK^(`*XA*Nt}p)%pUpg=>$I6MT#G&s zHrLJKKS~XO?vw8c@U%v>sSZ9`eS|7gX&Lm)aZPR520uHsfuGyZKX<6K*!xrME4=ja zWq8yj9nZ^hWN;KyK5(Y)DaAaaEOvGK16AD$Pqp;!pA^|CZMEd_U|>qAB)4;>j8gJdLoPeeU;@_>Inl(t{}eWHToQXk2#@^WVjlX|K6AVHXEL4 zP3|p^RMa`hol=iKVD~{5zhb15!zEHdq!s}zcvfvI0ZKi~FA-=o-j|0rSO=GZ6bc^9l(6%cEYZK|ot9__AEo*(w)OLb3TA!!2m% z@cqfJ{p0xB4Sg{Srt9~j#}A`NxCiff(XoEwouYU6Jnb84Q;3tW6%mc9ecsbw9ay^@ z>k#nq^L|uxm~YM9@UU>m+DBUt)sNn~#1v#tSNJqvj#T_pQ51GkgGRzXS&MJ7c>D0a zC@^M~YdUG0Hx{b?NQB$XI=?v$2|i0QyEn+2tX1;zi?PGYgSQWsmAJ3#SE)(h*$=-y z?Nb!BR>Y=z6_dL|gfMw^yWKT)o7^oFjeV9!2k7AR;i7{B^2zgcOBV>xBeJ{i-3nd5 zLXBG$Ojpp6LBBP1%(e3M?{68XbUKs#(f+U_*-!5m(`rivdQ)*DeQcL^YrOB>>Fq7S zS+|UBm@yfxJmPcj>Vq>16;MBbwTYf8P!zc2ymAxZeKwo&+R~ZPwZq$xXJ(7Oo-KQq zeYlKUomypDg5Q{SJ1@w8{n1rDk@mwoRnXT(8hLp!b9$vuD){hbJ#wE3bs7IN#?DpT zE#o~FuxSjP_SwQvy%@iIuUwnzbC*uxjpE%(p;jfMyB|+l%4au~-be*LVbEdA-pA=#c*V{{V zXgqhD2;+m8`VRiTKz8|d4}4>VhYk(!cI$BPjWB@LDnzm0_-JN?L9_YXFNODis8Zsd z&Apty4$%)%LnW6O*+<&>SH6{@N1W(kBg~4-uih6cF4v-xH!)h>j>~Y06&(h5O=E~` z0;$LjawN`aEpstzy4Uw+zn39Dove2+^Rb%`%{IB~e)NQosA$FfjOl|qCExP<=lZ^C z^jDN!-x@_R&%VRS4qGpGhm(0=hvXHiOACtZ;j@Jo zuhG#&{{=R^q%WZHu~MOYF3MD%Ac7)h89fatcYxL2S5-C*D_OiNo z|5S+GE17mqeN`Lv?3o8MRpJkpI#tzGAD&6$TI^qbg_co7@j}`o55o;cOMSK&`7h=C z8dv?Sv(?loyQ$)axsO`zQWr3WK#S-nZ#!`7XnkHYi}&tZ5RjddVphY9x53TkMxFG5 z$CuHFs}ild3W*CJYEv8PLvJ;BnUIUw%E-Qx*`-l4P;?;oR6uKFB%uks{CnB{lH!bB^!1`}0B zr7bGIxBYG*Z5TAHfo7rsSsioxy2$BxV(s2AX`!BrotV_UzS$PRqFbbMvNF93heAm-ZDSC@6)q0-y+??R7Od1jfze(43ia- z6@H4hfhIk%s=Iu<0TvzKbWakOiO#zu?Q93@`Vlhk_NI5e!VkR=oC%xHtt57> zS^LL0c0$Va?8Fiu)~!B-K0wzui&;B9pN7lZqM}|Tq`CJ)Hn=rye*Rs^h`9w-&%(x) z88qnZTj**)IJ)W~-hRc54I-Tsg{InkmpIK(@ep&Gl#t0y2|?Jd@TftLvNHW|5#~}y zM&4FUM8pwG>L|-2Z{1TfCmA@`B8XlUhg*s$7wln%H4shgl&kcLdF{r-7P0k|C3acj zuryY8UonK-)Z8(K;V_<`(*hpRU5Vni@W-@eun9$b#IiOmgfSpqB$ODt-)?17lSoF+ zc8;a!>|=x)6WSl~eVFp_uQcfmhn$dLLKezefm>2f?RzncSF_gaBuk7+6O4REP zDY7WnCPZ)e?cv|X&!G^nJCrg!be9ywl{4PI?!B$Qy4B#0KN5jWJL;;sN9wD+iP3FD zuEu9^JKAloQ80T_Of$M(GO_r_tSUl9zz%` zozzz0NGK%tNs8%cPc&Sq=bOaQa-Q*QYMD&D+G_A-&I8o>43A{LKvoITz zbm;vm9kV18iS5icI{e+SUDYR~W0_t)Em>NhZ)OgY$8DN!Jj$Hp*TjZ8yCZa-3b(kk zT8i-lcnB_p(L$?-u6&@fQrI|loP|aq>qMIk9|)dmj>=F?sK;7m%2I^iDwLx2m2rsP zgQtY|Mo7+F=wOANXj(zXjd7i^Q6HPd*lY>RytBQNe%(rSQV4z@EDz z)(mR}L$zf*#)#D#e7p#EsFB!n?rHIm0YdI5g6^HXT$D+T`Mu2WpRF^C_u|036WYod z(Y@8#I^VLDC?)$JoMSgezLHGqp-UoUv@5$}7@MLtSV| zCpk4E>ro^jzCliZ=|9(DNfL_j&oRq*kpCh-_(dXthvP*+np+gIkxLKBAgIEVDR>kr zo?H4Oh}yYZWVZCmQ@WQUdPS8^G`zp>N=+lI6$gKWTogqfQUB)=rtC8;Jy8iv)yZSF zAH*CZAGQhd=)(l}9gp#n5^o ze}r4WfB`L_F%75wK75H8caDP&pCCOJly`0$V}}An&;6Q$$y6_6FF33*Tm)L0@6+;y zj{obPW}R#x77m;?cj^L%Tc_zV3Koc^C*yH8XKDFsHXe1~A)0?DPXZEs>z3cfpB$ry zr5Yjkcp{;No*tO=2jpL|DvX)#g)U4?6W$1^C_QzMc5G(8WF^KAMct>-ajGG zWgh;qCd@{+n3+8^-vwxI*g;{;FIsBnJ=h#L!=o<7lb>=qc*+HjL%9lh5Gd@8*y@EA zzAXG*%~?(yPo?^yol6xFHZQ^(3h@mztF%@ZG01zf4G4SQL*JnPt{wQ1E1j3LXyS_Z_gpv!gj7EqsOudZ?s`lyJ?O0|P$N{Go^VtFZ>*i69g7Fwq_?7e~ZcJb`!6VzY-Fn)rD{^66USZoTBB!S%@ z7$-!!%#~d}2s+20H-FU)rCzFpf59C5(22eL085WS~+K=d2v4lNoGL~}#iXGJ!|cWfsG(deuiO8fcaIp56o^D2rw2CRk_ zE05h=n&#_E)thuBgj>9>JRCMH$$uNlm7D&Ow0Xt~bC@l>g$w{$kYr~EYWC(bMuZaZ z0N`DAO1_Qp!9TJKZ~XBrWK`VYy)h!>#u(^$*?n*JE$Y(WHE+*D+TI|7m_MuHqpghA zs0_oz3xs^VjVp;7Obi+UmM}_Fiv@LTX%KH`VuSl zL${U4_aMwzP-!J+dHJ5LpH?gAfDnhIU?D=?;dNPSx*$_=8&H>=W}-t`jHa*uyjaDo z1PftZjn}!jJoVt(xqDA}@E-AO-X5_-G@3Bv=H74Weov8nPyte7u-lq5<*S6}ZH^u4 zdon5*Y8mIp#)Fja%>C$dwVGw9##GIW2jiGjJ=u#I(FaY?7lrAv=c-iT3`zs##LKMh zhSdY;Og<;z7x}q5OnX;;<0XtK^Eeysn5FNHMVvlJ*v(j+iP(@d^{Nad0x*?q)b)|OzGKdS%arJ2n-hV-E7I;r>ouw zs2A}~p5WlX&6=O9RpFkh~G zYZT{PkU?vpCk!7yfmBqA_TyD9$iCU8=%iq7V89oUZ(hnOwljo6{u%zzHtGWU5W&r^ zNKSp4pj%Oipa1~YO$<^$t-?b%S9ic6lg z|I}I}9zyDbba7(}I)xHKZG@8?G)rWNhej4t_!&qaf2!RUWJF=&@7~K5^W{Q(RJw`< zw0=Xq136;j?IY?ssQ_H+>y%#?j7vVV%sHY_8v-3=Yw(;Uq_T1DF_w?Fd)RHfP$hbg&IlK4gVEQcy4Fwaup}{&!L0 zI6JOQhRM%w88{Y}Yg_DUezBAEm9|1GVvd2K7(BSpTalS_!W39wq+u4~nPO^X6?}?QYPB zmqkt@9@o*!5C`@-L9Sd+k1zf~WVj&&tQ%6uvj<<5pKNX`uxyZ*Ezx(?+LaPU((O=y zjKfsJ(u1q+c*_8~hU6yus*8EsJB@hs0F`h=xb-8m>S%|gaytsm?c~ZH5V+di)<6g% z?{O$r;Z(_V8JVuFwLMP-6QN{WM9Q2|@gl>X-6Rjtd0~*2XDjdTI%5cdbPF+_&ZMl* z8XzK;v_%U^AcW{Tdmq-z6ufB!`xD1QAI%XFzT}ygrU>I0(khn(E+)pOMnM+?ClCwK z0?F2Bfq84wXBNZyKU!<>%lWk|dO&1yekA8PBP2Z^q?i(L*qYyjy1 zmVv;Pi6fXcO|~g@069F`pti`5OT5}Nl;QCDv$F*Z)BqdKv=8T`dz@ku?H6CQM*)bQ zw&na9gUo;0dtfN#DIMM>9&Xj%s|!{6uL?}HiI;fHWh&KA3V{-vZ92v#C_>ZT%{VrY zM~Ar8>|yNu>7-^44-h{{M{zUUh>Wmre8aIiV_8H)zWUu@WTsYqetYL-W2IlD6Q5J*_ATg<)=5w!Z_5?Tjp1)7s^ zsOJbqjQ*F@JFY1j6y)dB4|l8TMM!s;l@l2sEw>D1&5=#ed)-E7olWj<;;$KPxZn}U z4RvOV$>Nbp`avSe8$F6iY=-FBW%v&_+A{T|s8WR^hGDJGAfxF=_u>7-8{&TMO6Y!K z*uRg_YxDGu1cMh5dSW~t#*Iu7+i`j?qDf}8O-I`UIxA+i2aXTqKrBW)TrX?*HvBDo z!lC~$x|%yjqK#?5EGYP1%7S9w06|-8)<@oI@aH&qk)r5wNGR5I%3JGob0)G7=qLTN z4J;BR^k|bIa&kw<@>Ufk0_I8i_zik|yPhwXgCg{R&%r{9L%Vcl|77jFZ3Wctjff;m zcEJ~Fsz0i@#;l|#6tBd5P~f(^=PF-N1z4A8BqVTWp*eL{d3oGChqaGPK zX9A#`{}xGF?JLE8>JsKY$$t}WTtse!^Ub4Xq+mu*dQ$jI={)`j-vG0y6{FeXUweFB zy;{{bD%xJ%uc9Y?w#!|+T83&wJ#iraDU=hh`mV3QLC7>oWCpA|qDzXpSoEemW2%J? zYZUvng%mgg3B4Z9crr_U;yK1zLEoC=hV7!=Y=8YLy+j=>nQ|oeBi2js(9`<_2b89kj;<$vlDMc;cAyCeCx*UbJ$z01xP{T_s zj8_E6JT;UHz)FBfpj5ccLD6|Eihjl^RGfiw9uxb94)PEebdcDk*b&%F6n>PKE8Sw|iggIGvR;NW4{XE#$_b^;`U4K>{RU-}*rBzsIS@FxY2Qh6-L?yS ze2WOUb}kl(#{iL{tI!>_>UFv>b?^fAT>h6oc=Zm|icO2XEes1N&R(%ux{;o1Hc?hl z6|M(V4JCo5{@lrrcq71ep8hSfi$Zq_A2*4 zF2TrW{h7x%$&R7M;aV?K(;4s$FzSE8GrZqb1tm_!LobNc0LjhLPwz_#IS9X)^in0b z*AO1EvF78u9z{*Ovjp~N0`~a0^btKF##n>PXx@L4vSaKmopcK(mk~T#>y8X!%_5_9 z-7z>tFam{RhwgTsH5CF8Qe;v-*2B-LY$7+8zbE8;m(Tn77cmtDHqGTdBtORibm?O8 z79sAv`zQY$it&Ccp|b_cy`wGXY%VVjcH8E-rcQg#kLm@xXuWi_c3fIAA(HorfnEfe?4s$ z>N3N!Hnc9EyUZCz0tw7DbUG_Kn343}yl80bUSn579x(WkHc z>aTg?bVflfAha7JGI^x<4>(eB{-DYSZBP=(Ykk7TkSfX zu#q!B8gBMPx5-p;JGPKa;En4Nx=xqxhwFxp0F`p$HUxRVOj5ZRKTgdN>x@4X5jyCQ zHJ?LG3P{}|C6rdzEP(jwxCJ{5uT@R)veUwpJx%GKl+fjob8sleD6}uT#N#ng&&BCehQ+2{Ko30 z+h~)L(C%z!wcDq`^Tc(Zc-YsHGyremiWVxnh=op6H>BFLrn@s}H>bzwHD$}EA91fJ zw;omN(aRDZI9Rm)Pel;gF&byFC>#}I%3flDR@o19bx+<_R;$fT^6&hNw<-CCZ82uI zSF@z9`T0h4yvLTxnrKRO#^0dN_;xU<(NFl4b~b5Vu8e zyoe|K-0Wh&HA*kE&36dj9j6^A23%x z*JVA*!y}#h8Df(@Zb6qs?ZZ)*43t+$3I<{WW4^S=Zs{9msVxx8+||k``-7Lz6x~AR zN$mK3T)tj?K8o-2m*3Q%{D-w?!cvu4I;ak78SP~Nii*+i>daFs9hgkfSu`hOJaOH_ zJ3VJVLl*(;T>$LEp-7jm#&_!}lOmdt4Oq)xKQB$VcTnh?M?=>dt;S8-C!V`UC&;}& zzX2So(PS=t-*Im*>1Mc2we@60)vjUde{v>7(3U3Op=(cCU0HkP#&;r_6I^jR5@kjk zDWVT?vfK}F=DD{brsPGSKUZ^6UoKCHl+g-oFq7Bt#lv|QIFQ?4W0?(Q<|T_26rnMLg6>2w&tu~x_D!d zX3Oi1g*`ktm=>|G_`tk~>IxL+gx9D`bH{zh{))uXoY2g&%X3_^n zYPeuJDl_vij{urplS#8Xb~(iFDgbHzN5cI87;wNv#Umyoj}LMNf8((Pql96OdrooE;t02q07U~w%Rbok8{8rX52XV5{5$Fj4&_-@J;VC3X ze^zA?=b$t1uO~wAdAqiMT_b|Pq-b1?ed|N?d_th0CstO`D2sX{oCaq+M3!m*NmnAF z<0(%e?_KocMsL#@zY^;X0U*sl37z1hRhLaJBj&>l0!%e2v{^iy7&r5c-yP%hPI!`=;)BydaN4v+}t~s&fvJa^`Iyo{7BZTrTE~IY#@-1!h>Wof{ri;ED zf0*`>rD%jmm3kyHw&zq}0>GfRXND>5tM+u_A;!aMAYd!0i*ZyNd@OiH1^;*EAd&D>x0K@Tn{=P6z`F`Yv3obW6MS~*ZEynR2vZHLkSVDLSkpU6>xhvUR;(wQ>NfDUC1DEsJL?AH$B8xZg z`k!RBdZ#3wD$Pc7Q!TpE|0J!e~F@P15tlN-##Sn=#qXeXBfDT`1w z!rYh|3vCo&!R2fHXj4XXovhB-(VL}nVjIR({@TLj=e>H<)AWV_sOg* z-Hmj@m}nujph9RnUXRPB0E|scD2HK7h$T}GXJLV7!g%iFNsM$>FXx-I#Kx1Y#gVR=A}~S4JM}0@xV>*!6tNZ%DagZj}ny8Q9^Rk)aI|5t+bo zi%*fABQpFMKRx0OrUQ3)VpUsF{R^{Zq39(2EuLxAFQ!U3E5QwEn&bot@`-#5hG{#8 zt-eDUvdaCiilL3oTFZ3)Bb#E6igv||i{&x;6X3;I$Kz4X%n=KYgC||)-3H-AML~zD zm2~(&e|dB#57!TKr6zT`W?p#f4{C|P=i6?XOTwe`u9FYJ<)`N+MYH!o$|`)50w#uI ztIqsXc&|pMOACGbgk?);ANYP+ICV;VhaA~!^!U5hz_^K6BH25d6V0wtdeO#d$cNyy z#G5AmxKK_E>cCs6vmhN83N6>o#_PZZajEfKNkor{!X3WjvI!%;-h*)Lh z*xbGY2cZ?i9jQ!SyK|{PvBMpSVK#2zF(Bn ziy$}6(LF=MU`XzXeS>!uFwua$^POeQ#JFXKg#Ffwm+1c~;B42^wN7_V!n5=C9DVP( zIT-R*{gd7W50Ibs@$5%bJD>tHXbODG1)e-!q5)yg=D?pU$DSOxzoF8kw<6TYz9$6C5e&ELL5V6R-1+KTz4iXWD z?b6^zHl~IYuq`QQ`7LgJ&eXtT97F5j_L)P8_V=#V7pGpp>j2t8qbwrIEnq2Iu^>AX zE0;$f-mivjgDusHQTu3i5f z=UvrSJiZ-EW$XZnNCUJOKws-`xu(^LHD^FtWqz%;L%iczNkd=g0HM8Bm(k%5046M@yS{0P&6%fl*Hj!h?RI?L&NJUrAawt z?~u>hf|EZOzPkS!IYg9LrLhg~B!px?R^4R(x5;?G*7w^d0e<`wOH}bq>Xz=gwUIa7 zW2)e1WTWFx3~lR_xM=2ZG&;Yt9&r9kMJ$gduQ}Xz_;NJ5S@B9^`Q^#H^z}yhCV>{u zDGwV>*6gRBHMed3s}<$W{|a#jW%bnA$NKoibJ9+|uFNAtk(qpZ1M}frypA3JQu{BV zH!mT(?SaADYVx|gPWQ*Q5wKU~T1`SVu2oc0OPDe?40QXjQtTPe$j^bd zOIsg6#i5X?t)A__{?j$xKIr!hp{)yXIQxvHqj7O&ja1O@%20B`aM*h%Wj3j-_idCz z!Ts=KA{HU+`~-)0lLJ&o5wMtFCz(Ret9%sqpdZP5SjqzfbAAXhy->h0E;nLbYhI4s z)84(JwHg;}q1CgcQa56q61ZgxzNL6Dyp<``+GkhQEMNhVG^$?CoRG}#sFCtkmbWoEjI)e zOhE|Y;KORkGGwn<&^DPy0bzi|?YJR?oaLvHrZ3dtx-?9|vaGCaBG2*ls}Y$U>Mpej z#?Y7`DeZc#M)ik~#Ujbfv8^sRn?mJd$;{>h4R|MA^3Ps^IBjU1+Z>WEwpdF2r0g?= z1Fxn<1j*#2-m`}^i$ygVU0Yu)*c=MZdJfl9{TzrbbgO=`^Ks$W-<4B48d>GWjC&bq zG@H~X+%d=ML)=~1*75jfC}LtByRl8z=4O#Lt&Tje1HS=x4z@nwC|A&6CBhk))Z=-^-k+sf}>gLk8 z>ube^2;aChRK=kyZBwnj`K8lCh!(BzVw?aJ@j#{Gvt1KI9FmBRuH@5hUDoggYCASva2))QPkg z*Pc-;Qp$_hS%XfCN*LE(j7ByTF}~{w(X(Xj+L~kvAec-$UJOMfaQjaUh%C}PE`#?$ z@!~3$-okVv)eH)V;vA(k>s1>ojB-Y4^T^2K9_h{Yc(qd6HrZ3nKQ~FUO)jX40K^2E2 z4A?B!n#}Okg<@d|jpzoDWR%sA!C!dlI`qBpQF=AyO58;Lg)dHt-@FKK-i48k1&No@ z(?ApxQVUys)jT6bSk?`*7!T9<8HdVD(Ik}Uvx9spQFDW^*uC)k;bb!Ys;q`neQ@l{ zj0=oe9d$VPMxM2-r+*>Vcc_GaCac?I_?AAeK<5QZ4v&087g4*xov#sTq`lm21tdGs@&XkLk|bk{Ly6S_!wG*1K`SKd zHH#0ed;EMdVy}ai;GGd=SSzjCnr$_&hSR@ae=^9DEh(t24cT@?3PKdiq|^%p9EXNE zvi@!bC>_n0UNOX?l0Naqe^?CRu3(Qa4#GA-41sEfWgG9f_!|1LV$KV5Z@ynygQU zcK!B49AtDZ4Ni)zWxedsfiY#-%evKAkpgfYtYaMIy-kbQ@W=@5S#`IVR)`q0-sE5t z7EB$v`-Jt0*K87WpbMX7nCYzxSQL788tce+`N4Uts{0xlahjos_FoKDxc1sxy9TU) zG0L@~VAMN=Ze?PTJf$Elk;4Y8dAhch35OG(T~-ICS)#F6t)!moDErz6oOTp)Ibs#i z$A$zM@y>5To&eB=X!Y;4?Ug5l*&YR~yBe{6<{8@4xR9wXQmKS1=*>pJXdzin%e6b-nY;myQ8eP$ z+exTo=(Nd?A*FJ*%syfDue)|2EX8czZ(z)F$O5wYu{`!tk^uQa8}T+dIm(SZ`+^X| zmT?%0+LH+Kqyvx*NyZ*b3^KJ3TRpFyKMO4D?a$THK0~(pfChzCVi;fMlijLV%k&rb z>Lx2Py&I50bBgsIrGTeqiWuYf0EF;8^`sZ|DS zV5#+1(If7}esL}GO35lApX=A3fP`PW3(W`ZN;axVDOFAc30Qmh1P1&~3^5?pYdCsY zp4ply)t;7TKb{u6TC-E!3XiGJi)m3)zcb+70 zx|#lMup)8*m~{M@p38>2F%`Ieft74KW>6}N-VY8BBj-%53y~P$1D{!5&DaVD>A4vy z;Lg%;KfPU}W*aXorao{#XVDSg^NHze*O&9gVL zX?x}yGzR}Fp2g89kwEJpjDKjt#5z9w(9*&beuidp+ou)H`U8KCg{Ul8$G*I3amu;4 z8m?mxx;@imy2d{0PrX}R<@g~bW7FbcStmQ+KsP=-$=a+ToNUUxI!vqmvDyP4){?di zlxWM=99NZQOcl>5yLd*q4{K8(cV|A9V^{&vTS=J`yUh#}Ybq)Wmr&MkA5EtT;V7?H zm2A!#tb*jVweTJMZ$wJEp=z#4y<}ZTPD?~NBbA;7tDI0tfo2@Rve6%SZxJQrbc^k` z^N%!LzK{E9a~+S>v`5W|e`*^Chp~ zH+epee7rwyd_Q9_4mxFNwFl$+YMC=$>Jbc?rRouN&RA=BhY2ev4R8ZAbzis$oI=>2 z68;AWk68p)Eg3v%Bp_+E_t!9;7>R&A#cos^9c`4wWZ`SVCQvV))^0lkG8Ek(Z(fc= zQqp;de1B4C<6i=a47hXa2)4DOwCUyf+PE_4#eCbTQ1WvMh&3EXmArQ7;!itS@^h;( z4um~h=5Cdc`$`JwV58hLgaF%WJZXDr1PRt+2~-E9i)GbIq5zgWJDV|HWTk(xR13?r zs|hrpgD)sIl9&)ETs$@i@~BKG7I%CKv9@@z6H`?wESM`Cf@Za@tp?*i@1fs|gRK)@ zM~PKE(iG%?3QTh<5YGE|_@%un767V=D6R|FexQ%{AqiSQ1PCE|fzRprJVB7C5LIZ< z(hXbiRlv$H`6#7Yz~$JJPZUEhvlpF_5M!oCPmQkGF{%sX`9`O@L3c=m2`v%wVF^7I zA2y7goKuB>B(O*d(7}&waNb)I(T$8FKq(9rKgkRD5SSCd7;ic7m~T^#E3W3p?R==%-sW z{R2nh(}luHGEXwQZ9aM^#2UEDmc3l~$GQmu zQziTvyjqS3x>~=o1h7@P0{XZR8MSghA$M$u7MZZHW{kvX8-b8*_v?2{&vjq zM27WCLPvHfZ)*7O7I<1GcxG|_D!q?0NB%tWWc+n`9GMn%2dz=@iT8iv)*AjJBqQPd zt|bCYfCKrKwwOBMGis1Xh{O8tmr)77;%v}`xRMIc&O!rSU$8CuG99y}97^%&JcORU zsfk2oQA}e&6lb!Rj9G_CX@4C|{&9K>zP)V^GBCc|JN5D_0FB~oT6w}d!P(yp_D%KD z#C!O)B;Rt1p;79nalAr(K2UdBA?p@6&_RJc#082Qt!g>;HGV$ZgtxyqbZDvLq?w_3 zj!OZ=mxVJR1hig$1EZehaUi~XqX)Tq`*H03NZNu!5#iqg!Tp1R9L6ZW^{nH?dr}qa zSxV$O90$jKq45rREs#dCtcke^DOvCr>5&H1LeNMW-|q#7rj8oVfe>GebBAIL)QAc0 z!WFCm)SqkZcN|};$aS$eK8%#5vNYMCQbk7AlSM{V8iH*BAl4{j5!(;ASe$~Gge!8{ zX;QQcIzSg=SElON8Jo@9Xul%gz%OkIXE(SURrJtrjN^Z?^svg9DKRSRF;gs{AU)%> zWK*Y{LrrlkumkWARB^k%F`=Xp>%LC}TP@|#n8Js>MeA7sK5Z{P#9No6E2xq+2&q<+ zbY<(&Q%eirf% zY!QA6MG3^_SASvvEYUjiMd+uxfs73nM7-aujmP{lsy;<5fyx%Cz$DyrD$`>FQg4O_89+=ri>PLZr-UB%)B3psKZ zZ|Ge*>A?If>Vo{~?CJYUWY5+9QCATe#)v?tu^L-cEu=Jc{`q&AbiX3JiRO3G^!~Y$ zSE5{9u#oZRT){4La0CrOUczLHH|b#HrzDHPU7a)Zejxxqlv4KFbF4yK-Tk>_?=nyk zT8nz`QbH^1jBo2MeR&$nGWp`x2aWr)NkAnlPaeUh>xlhW!QlMyqMdQRnJl6A76iiY zJ_k*9eK88hpUl+B=>DQZfYuj$@4fik!@h2$!?K^&@LMD~rsz!^3)nBoT2S&Mz4!$5 z?R*OpYW**M8G-3>(5=}ZV=slrwH7uFsP1^Nbrobk6<7x;zwDbZUESHUFXVnc+K6)Z znEs+e1`;I-R%Z5hD{W>t?jrjQx-rXFH|kc{r*|BA`Hwt>fZ*Ny4|t<2w?_Tkz^(|I zjx&M48>4sJN5s@fQCW&uX}Mg@2{5m0K$*v|<4DgOM*eQ3{{vIq$h2ALx?#Y;D?m1e zY*xZ?(IF*Fak&`@hi0rZ0}xy#I~2_cnDsxIpgdCm9h{W>R(vClLt*ML5msA?`9E>Z zjF74gHUnhBzqfWIGqjNr{BPI!){tFKJ&PVDLBi|A8vMTem&Ig1ZBQ){`lco$Yv1lQp4?MsnT78En?@*& zAh9wYfrG}3LH#J|Z^Dd4V$r$r*SSf&(!HdJOA8~6V;3jJg*gvUhgtkcixE(%v;e{v zFiV&6bo8r;wV(4YFd%L*S((h^BA_2%LQ=_xEIdZyC9Hcz-nBsd(53_qy1+mn(-+PTvlSD9`C20_qj-7 zk$7~NF!er*7i})Eq3#S+n)a6rGM}`<1kL$Gbc{qm;pHC7>_0Q@m;i-0j5s_&x%H1& z>LPZWVFR9bB~zdA3+Y$K`1aDiWtErrGC}P{vla|(icr$m3sOreWZ?gmp1k} zAeBS4cGToOHE*#N%=bdo0a*E-+Sf#GEP5im??H^V-Qb0fOv>!xg=x!kT!Ug$|FH_R zL~L(<{FOZbUXm(Q;PR(z#cQ*M8r1#M)as=oZ$1(0&R1q26@~4*=0kN099kS^LG1j# z8EH@0Q~bE3Pq_B_9B&OtStd95XESw(Z^)%<2kOw^w1EZWS&w63GT8O@siq1J@hZ@d4!3AzdiOX9ZbVd(EO5$*+KrgZmhn65`%Hm?}Krv$$ z#*3?ybDV$X^nMu0`%r}D;6QLMx@b_+Wu_? z2Je^m$}dn6N)3Yor6MIYz_Yq3+`N!y!w|oDRU`p02P_ieGUQaR1(a~Z$#e)s0;fIR z(f;Mg|58I03-+o#3!KK}cP}5i5L=+-5Z`x#HW$2_3z7x$9xC~&lyw@pU4%9;dV!sR zb{QxCaa9o;3Eub+zz(C$o0dw?c}Uh7wO*aLItE8r$Izb5YDM$hxD>RIZU0=GB3Iz# z+_)6eAEhmtx$_Uk@6ANeD0Djd4mdh%QGheDND56T1&a0=bqZ*pl7Tqg@}=*v z*L($}sG4-WjC$Dnff`N+Gz7+!{|{wv85UQwbb$tUclY4#9$XXL-Q9u{+zA#SK!Upm zcXxLW?(Xi~nR&l+?)l`$y?>aen>|~4*K}2_)xF9JyQ7STBOVwZ<_g%%ruUeGr+l(W zuOIA|8rsw%-=uYK6RFDqTz+PiMvTHTB&DS+$?_BEjZRr*ZYL^j*Qc&GpcB`8j5Avd zq=n4=xK$SmfQ8I}z}XZmjW@>@n;m2Go+;YN3>pf!SS z37En51`t_C{Wcf-6Hyz|HG>zclT~(vRNzPLMDKaysWk%^WOLRJ7njtGLO04NNckqiyOt@?#@XLr7utQv|GG*v8oM;kQu;{V2hQf1Yz6)<9`6L6SRciDTQP6LG7am*AUX9e;1 zm#MGi=(;6WBFFJg?O&@&WmB)BBO7R~E_c)GoeoPGN;W)mz>-UkMS`3z;#gcbYPm`8Y6mB;T#pa{6vVR zT2tmEDJkXb6BE9?f|>)Ehl9Xu3}Ah9AW!pEaawx$ zL9NdSP0i4xQd++%T!*oxlsMP+>Ie}nHgdYxEKZG47DG@89}AEU0MgEzQ-ZD(ejt2h zl2e;^6z8<=!(P4NU3ZZt9{-FiQmC~*cSjR#4SHKwpMHL1`W8np3G#cG8iMepEigs{ z*CF$Z9MQ}4Ho6Za;Vhr8N}B{8zyX8+96(N-ac4hyp@nfiC@u|>9BLf6D86NKHf6J()HKivZXuANM(eT2$4Q-$ zO!>#&wk9=w?Nf$)0C*QJ2b&DK&VB4I&IOm01ror;ob)NEP0|SLa=8~-{S*1ZrraI0 zhX30<$-NAC7E*imWCIZ*g9mO4wcNBE^jq+%VmKF4zJrEexs%kk&WNF-0AQNfS{R9f zva4;oOCCj#-?W7yPZH~jXcuD~G6Me;LxS+t%9E+9yEMT2HaU~<-1p_icS4r1;`-Gk zVUA+v{u#*ZT^CqzMSuVHt*@C#uiRIN)Hc^jUO&$KC$ACEfIPfC0pt1(ASg zFGv68pnwr|%n%scvkcb{yG@`PrHGhG_J@_SoLQUVnJ?zfrH(`)CK>{gV=)Rx=ahH= zWG@CtIaxl78>Q^7a#K)uc}m99>kbd<1RX5jf3f{yKtR_C;s%@78qV`j$G9oPpnx#u zPqr;dHyk`xgfCvTbYEj`uWTZ5hnU-1=izFN(Gm?nzLA<#1CUp6yPp!!b@Z=9dQ7L7 z+uqCm{qXJ&=^&0=x_zCEP#~)kFJ&<+^hEy+g$)$g9Ac`WlB(# za*eZ%D`S{o2U}_rHQMTkE!6=6Tj`M#h#evi;_}Yx``Q(~;5)O+7M8-TMN7{2I9sGg zm4sPS#cGp7<3D;>%K&&pFc2-+StfNRevK~$cnAQr%1Gb55+#40Cq*3lUGLNryHO@~ z7_Qe1b@%Vx3+F!%u>;2NwK@GSx2kV+6W(LdXn2*qvB8z)++S;G#jR#jB#m&EG1zth z(f5{uzDSGv8;=Qq47Z$k_5$&0063{?&XHhM?&lSb#pN}nUP+VEuHJ_0gi=`X2i?ng zC^!4b281)+2J%d;AQ%_D;Ri{u(R#YPvRCFq116CzW_L=!`pU^Zv#BDmZ>+_;3Z9=Ff4p zn$?*KL4n9jJkzdiR}1-D{k)sKL$ajd3Q*b#PuvtYR@pdJsn4=SsB3TcNBm=DqSQ^G=GepDIEUQr z0we$z3=n+Y6{W1@jcCCz8U=eI6v>=i1491dOPa23RUcj0A z{o|s$28(>TKKDClI}GK963i5+NiM4Z&ujQIFfP5YSVNXLm3my8t|PQdx*9h!?c7d& zd$ddL`_Hbrzys5uLF<0$gE2e`soe$4e1MA(w~A@mqm7LLydeNG=@ouXb=S0G*dJ~a zLvdGFjKG4;YQ&|$_Vu6pMuRB`gWXO&mjE2jDv}bJYWn-~PGh9;wBp0kSEq+IQvVG4 z(9y%OqgVN|`K6)w?E)?fy3A`G`>JV*3CnMDM+%GRS6~Lfmy7Q-0 zxvuj>d+w7BBtB!(klR|csO@g2mE;mBfm!~|uY4;y{H$4k8}bK13X^pXO+Yg613{!@ zvOVfV^J6VfNDZ_kFt4OX?0VXkkSn zZkhK|t_zG0sz7nAMkzCVolGG&Qo9}lwm-_wur@GzpO16wVG6jaBk0&heNT5MKd6$=Puq z&=-|c_GE#W4gi#u$&XWS1LV_au29jnk{_cfrJ$_cuEl*70EjFo*zQ)3;^q$apz8*( zp8}An#b*G&6X_4mY0}oE7aipk{%LGc8nE64fY^foxZ6*RFUi(`p@#*48Wuu!?f6dY z##%~%y!24XLFv^>T_#e?;m-~L9|c?i7~#=YP7l`S0>D#~NojEn3ff<1Oy6oZq;reO z&|oCtXBbLdYUA?&DHL?-RxN-*<{9%^7l>O2q|6_`j64AsoH!xuRU%ZAs=5+M>hKf`C<+goJAe%;V%Cxc?HAf ztwy4l7f3Yfv82QhjrMf#a$eL89IGZDXHbLV96^gD460f3ctGP^_zj?d*|1p!8N~w2`R@31dF&_T>)~~$R{Z;Yj=9Gm;&lJL@7;6yr6c@! z@xlpA_wmVAIrq>!CvQ4gM8u7f`EEXH*TL!y%Yy}f`mtfdd4 zuH%Z>a-f``dLgDHZ3Y9kHdR3z?ey(tuEFoEVIR&-ie!^(tl?_XzWzebD*CX0jD%sm zwYqT0UVn2xSn{*w?6+IKtF~i-j6e90S5~Y#$832Q*W~E0dng7zj|2PPB@}R&H&ncC z5BCR9sQ-2d3cYRX>OCC^ulsQaO+_isI3fDEJS4rH-oWU$uN-G42!9oPq@VD17R=gE zkuVeXdpaB7hH~heSfg9P5pH|-JuLl&Q^ zzV3&U`En8aBy{lpSdsbh*&yTclDp$o&&%QM^7g&;@p2}1W}+2US-UIGNn5LwC^c2s z57FRtMyx|yi1S*WM8D&CAl2P(WUc)=?s??6Av42}%*ny@O@{q`#h_X_PQusjn;fHV zIXDUNI`r)G`GCWChmh7IXG_w{8>;?MLJ6(f*U)hT@ab04GAJqdWOYQNMHJmcdHygCJ@mV2@{q`B=*NK6{7O?qPeu8zTGKC^ zd3GYd2%<5Weq#r2=IM!O(v>+5)lLtdFENJ{hAQMacDe9uu(d?*hT{OV9?&oORs#J< z{lxbw_HY&wug4C&L<^{s{jxWf^fXfzO{@#Q`-!&<>ran*9A%!fCT|t>ZxA zw+r^;KF&*#Asu1HX@)9`9pW@KmbVqEyXG~O1KqE<#Tu21LX~y+yKb$kA{B*vku^$y z-X_aF)Kwq7r&`x<4e)&pii+<+V}h^4IMg%3uQv~y%P0;2mDao0%#TsR)VFW1I4=tf z9azEtB>`#9o^=I~fTZSz+4z?Pr1d690#X^;;C1jK-R5H<%9eR;;&$Q6Sx6oF zQV?g@=dwBA^4vQc`-&Bq5v{iN6#jv{qK~m*@#=f>RFyFttnQ*VOvgi8huMzK5vl%C z1yQ+AA$vbBIrpEyA4@l))k2F>aT(~n>qfR+?5UcG-n~w~du~^Bc(3tpUnk>st+nxZ zSB@6N_MHDQHjY^+{Z$ZDDB;S`X}58~v@M;%YU5^#NT^n|Q@Wc#9kp|C`L>X7$LsvS zd>M~Jm<`VCDSBYNJZz^JO_u;WH}7CqDm9mN%u+#pT2lfOE|gzO?B5aY^`U$MZokgU ze*l*g&dTjIM}UgVYLPkw5}; zzd;&@q-;PP$$L0=864OspadV(Rp7VKJYv~dDE;P7x*R}lz8}~~-D)$X9@N_{5Q$(v z9$C#ke7n}_tt0wvQ`4}u^aU*y|>fOJjq>^g?D$_5omUyr zbUZO2R9P32%1-s|D)^~)vt%}BRN^^NxGMMi3yBF~l#g-(H>+bdzecD?;n{v5?=Q}s)iK|(D3al{-&S-3U zD)PKFIi!?3hE7R@N7vw9SxAMO%KqTnprk8Wu$dyVpIfXUCy>O-Zv;L~cbO@LZ*BQKC@Mf%S-p$@E#e4L4*f^S5$b=5IL>jP66yL-sreGuT+OWd4i8)P z**$dCK_q{3{H?44a|YCWdCPI3`gYF1q1ow7aZkXo0X?dC{+H5B_n0r^TD%zH9((A*-sW)fv(8v4VYMf`Rx_!I2gTE?G3Q;L>!ihmsJe%c^y*7HQc{DRNCgmzKGMA^+GEpzN`dY&WC+`mK;D%XpHQvTDi zP$7HsV4iA)88b%zD)i^ITIrhcR^sNtFo{ORrf*ZCaI(T7l=_<{#dy%MI-Os^%kbkp z2ik5^-aCq={!)_)om=%>n)bl*t^7NY1S2b271~eo%xNY z?G9oR9+Jho4XBwJNnK0^3c>k=3Uj}YKYNjH!oAmCE8@l1OrJNL4PijNUwv>yg{r`V zpn35B!5|7t_=X16!GGpquK7}emHvC9o==R%YqCsC@sXAT9}PkQ(R=ej|6I!^Ok*od z;AtW$AX7L$d9X9gkEpZroAHK%!$Zx`GeG{2_T)n3?y!mMW- za&C3jBZs52`7P!7Vj8G^+o%4Ju_x5}e;>lLvLb~j@Vnyo`e-LX$4f#^<{kS*zr#I9 z+E@ab7$j-hnG2LO#q;>Lq$xx!P=wQTV+ttFi722AxPiC;Dd0PScyGn?7n$Q+H?;-r{%>wJ;IPm2`k^&&U44w!``=s!VYT+L!< zdUH%MQD&GN)r(hd++JSkH;)L>v7iwW3FPbun|w0W7W28>+^C}CbtCvhsrz+9pT}MCb@xnQS))vfU?~4kSf-JkIKE6Co<<{)iHYB2DO~=+^*WIMn&16h+ClC z$NZSHJaZZ$?;#!a%|%Hk`+<6Wu|q$I=51o-g@do4_tJ(lY8wSG@iax$eFDx9)z~1F z*LiTzkZ5NCh6Lmqv=D&9f^K6JA_<6ka`WZNK$^G_b)A6mr~uE1Vn*q`On77^d|#`@ zRr!*eT-xGme4@JlX$U`E~GT(Duh_X2FD5K3ia{iEEz z-Hk)R;S9fgrrTp)`sXIxevZ-_+fk1&yO;0d_*J$+EQ-}-=X0Jy=wO*QfE}e8mlK{N zxm4rywIR;F!lBU^J$F|d4yZlmKZ`AL@DdK{-w*Q8der-B_dP0VMnw<_oR?O>7Q&4v zbamGL?lpMr7J2nd)v3HVn=LYBPRp4trePFDF5&kSj6IS|AuiX*91~5p#+S`v=P|LG z`CL9FeE6n6M6Lab> z-v1uYZO>cbvKp3@72Q`y==DsNZJ4J@7Us1Z33u-Kflkb&$6;)?dT%S_y>qr1q|M` z+PmrE;&cp3xYH6XS2)U|c1af1yJ^;yW7z6C#ilD0q}NVF8MXigstvCti?7ckA(waX zGTnC2yY6(SyGOmw#YOmw!Oi7pTcG|^Rp91bkkzbCqX4NA8iF0C#{ zbRrYlzmM1D{0vF(n>sKWw*DE__KJzfR(+**8c!XC()gEC=N21tbVCcH{m$1RsqC^d<)kRswy#|5=;-Y zDX*{)0=y~`)leKQRTub+q+pv;nXSm79t@-*XbPa%z-qj5mZB-LtjG7JM3@am( zuvxrO5U~x^;O?h`9zIxC1x~*N_#fZ=vB`GFv)FryXJ!=f0 zUaYXycpIw?l$eB*@D-U%$}_B59^sZ@Ej}UQY9h%wrV?;=GONzYaoAGKB4n;I@h&V1 z@hvhg9+3-?csGjQqv{!`eMQ*tj9BeeSPjJH zD^X@>f$B=iBI!fW#UdIgiE(W}6?9Q;a6IhMcSY!DJ7M#nw6{1X!`EVs>ADnPHRQN^ zYnb!zC5Jm9@b8wSx-t7qE@XPKLx3*^{cZUNm^#XCe44)~v{l`zu&H6e1h{@)rgKjpM;pdP7 z{|4>k!{*(%uUr@E&@gv6j%P2 z(^2+g*SaO988Wjh-=W_p`Y>?lLStBlg%W&>w%cz_A>;m={ZukvnxH20t^D*AD6KaC z7y!*a-C$!tff9}yCvzT2E*|?li{Kc`JBi8M65A=&cf9`*QZI9Xsm-2QZ5NXXvwaf3 zYh$twH5C>**lUtmylH(}3C&wOthK}705`5itg-VC%uAs9t)mFvUY7El=0J%7(3}k> zbtC%`9Xnr3rfp zOqz6~#`|H>QPrJH5H470ptvTJwg2})G>N7FM}Xc{qT)#N9$`uh<`yAEtBBO9$iR_z zY+z?GJ}(fAH_h|Q6|%Xe`?9NjzxGJ{Tmi3sGTiXX?A=s4KO?QPzhX?G};B8 z{FG-~HZ}?~X-TRdZe%u&V1CT4m{E%w)0=&6l+!b~zR^KC;kvsMLYw~p4Uy-mdywA) z(&=DYb$i+^NVH&mP7qvWO_WNyD#lf|i)f9)QXiM^+XxF;OY%_Y(ihdVAIBCCQSW1T z2$Rf3Pd5m7#?5yDo~miuuae1v9A8;x2%n*%+&YX4W zn6wT3C2oh8v4Y{`jYYxJbje*X`G)=V*7)Sb9-wnb=%omLg5}Z%E(1rkY*D1FXMOvu zTRmoOi%)BkYILgDn3cOXGuNoRMi%X2Y8;?5;P2R#NEcNuB*L@XX(s>}ntCl_-{$%s z<_pOC7({8h^#j;&ST`uHeB&SW1MpbObh_}ebOH?Ut?=We#hE(ysqm(DC)Jz=CAshh z;Bjva%RVq|$LSv(t2s1qOvjmoSnYGo4izB8eIld}R=~=`AtySvqSWhsrFeuoMc}8i z=Y1qUH$~9*I5~-wNRe&MhLr63AhZ_X15p_l_KY;cOk-T+n;UW&9mMUCVj4v!DY!$B z2IUikVf=2-<-23gb%^&U&s8e&5_qb@R?ur@_ zvU}0@MS0}_uYQ6Lf<3xEjrw_XDf~D*t~F{MN(tAlyR+TvJPj6E5XCbM29C3UzJReE*Z3-JHYC$=Nph5H1(0`1FJVHu2;U**>b&GyS;8Kx(*q zZf2}Jr)3}H24R+&-HbLdi>(LQ$Wm<*zv{<1DTL^;2L=NdPiF|rbLi*LCU-kV1yT!( zwNIiLtBS9Z6Kg_WoVOB!kr;0JxD986XsO_!km)u~-e{?&n*28F+eIR~k*jNs!@=6S zcv0;4jDC#?oIJF5`KNk@K0&8al%gK+ zg!u#hD3x6pH-r>C8f})=PvpHEK|c{0zI=C4oxf^CKOq@rvk4Y#D6w6otZ>TG?4cCy@o4czWZfY*| zpVr;@QB;tv1Cos4-i9m@i$C3?9v00~6Z!GSVk#>1Or8o*=a7A0!OH0D!%6CA@HakC zul)lJy&KQdf-J7>xtf@KnxIw;(fkyRT^)PiBgbfy!3+=w5Zu6Pbaxbrkm%!SfkKud zjxka##%BL~4cp1iLY@7Tr#avyBty8yj+>j^X>{V7(Hgdha%!bn)F37n3e}f&(dOc> z6aKNVTbGdZlgJ|B@K1$e_hBPuS(dOd8!t}MyhfGq+RqwqmuXresl25;Ad}N zKwIF&-=D3b_EO*`Hv-&~@%w;bK$ju*jmqFK*XnjXj+242jn!M;n{iJ=T7yq69jJ)^ z;1dYOhdZ0UokymO$~;1yCfr+?@Mh=1#Osc$aCtVf-0*DDq}o6CZ5mQzL9g75A(I4C zT|zFjAUB()tqvxf0*DaFQJ7zjnv*#@8`KwuqT*Es9m}d>a z5`si+OV|>(WZiXo3AWf5DK#yEw%a0HLZL}~SIS`eo8+wpqR$aviIlI$w{KJ5E;vl{D= zl%a6pY~|2-pm5%9@Fdu?^WTbFQNbJS%S2WPwZA_+oxTXB*({C=N45uFucNk)UMvVP zIa)oT3Xj7a-W$B_j$AAFd0cN^KF4?8Vhte_Zx>{rFnNc2R1H^CZ(})4kgy1N9TP#c0J9{hwb{<(&7PSQ_;XRy=ptzpXWT` zjGgHz?>TBj=rG>N4zVYqROnD>Tf&OEa@uFBK%se}kr5y*&!gOIO2~Znlo#0_t#%$#ATVm;W)K^5R`LlMlS7xR<1acq z{^pPOD{^@J(i=BSj(O5(9HmK*NDh60X)@iDPhF%jFY~fAmLZ?`GRfh!1AI~=X=oA; zKZz7lDYpg(s(3X+K&5XEI{XO^%&^)S&=>8pl5kuzOU%tuEM*xCVfNp+EnLOLRkaK4 z1cB`zKvkIhJ3x&K8$d}WP)V2%?VyfDV29p^Hc(>_0Pm#eL$$(G? zX6s0Jl-S!#BX7?5aX2J;6l^0@aVs4Z>bWrK|4W3zftJD6Hp@mZye71<6Fzi=0)yJ7xUYF50GiPEA{#Djc7dD*wivFd7>i)L(HWEdF?Tf^l3!gL>s z<6P2%_Ku=wZiTKb9TCNkn|nq-dzS*O%ExOMt|!OuP1?GZ1rKi|DyT(0gPrvZvYw<^ zU+=W6;v6|{W-X$S`jh#K*NMKm>#dzSa(|ExZZkJ7oRdtLkfh)TCw)!FJRJ6~9T^03 z!k0?$1&g%-weU`bg^kMga3*4;{oxnx~pXXo$( z{oqd!fTz-`+mgssUiv;@AWolAWkKwhkL7R`y}!<^eHrz=`E0L(%z8YsJ@7gx)0JXv z-!8+M7w02YzROcLwoCFfl$yolin44Nh?U6i0U~LM)0-i31xWQs<*NVmAFW@El7|4; z2|;cUJE6>1fLm`yJlVw0#qnqvOY47k&(?(xi|z&x`|euX9;z^K!8A)fY~tk_8lG z*Etbftn00O7-JcF+F|qKCD#XAV6UR|pU!3L1chNaE`r_8qKd!Vwjn`Iw~i!31Oydv z^|-b{3mN8#E_5DyJz^@^P2esBn-Vf)D@(3}E!1mtVU)MY>qeg2(Mzdu%eT$mHs2@9 z)#(Z4<4D&zQBBnJEC5b1F=S6U*Iyw_UJ^ZO(67#8V7wHC&w1vo?AkxdC{F;Q0jxlS z#A*i|3W)+FATSGLf;eiyK{cgnc~7J}$)`e-m@fZ8(vpyX?9F#~wswBh0m;1x6N(r+ zjsbJ~i7MViw`x|V{yig5Qn@h063yjJmI?iEgLdRTI(ZAh;17<0Ya@GrRZWghvv!>`4>l7P6&&!BT*>&a zrs79XEwV*#Q0XySAu*saDH#LD2{g0GB7z{u2oMBG!wQ5T(Wydl%2UcGoTz1^Mq~4P z8@KHrsU`k@)H`G#6#-ra`v(vi$3c*u1YoEGYRC@?y>-l$kF7KlEm@1qUW_g1ia zm2>Lz|1p2N+w85Nf`7xte6eC9Pu*{8{sOHJN3nfh_iJ!IP;I}hT%fVL6F^eMMdErG zZzK;S8`K|`5$T(#z9>WdkU)HqYmKn77xchDaQd|4f8QK|meH0Lxri$t$*g4~H79pd ziT2!dbq*x4uFjP&u1|hoz4=^4?w^2No=;E#Zv1-MvZIze+JWY1y_4p_Q)0hB@CXQR z_x)cVAOhz9`UmLf605ECp_h!8Z;juuTFq`|^c{GMmrnj?bvJ;Y`-f2k@shdIKql;e zSBL|iE0g#A%xz=++9@EGDnnoUe0%-un+Vro@dk z$w+G$YJXmf@zlI+L4}%717{rJA5?)o6>Slu?LU#exRy}{&xFPN#&DCRt9hs`hkU*9 z;VqR2t+hGaJbwP&9$v`Zdwj7Tc=*$=YM5fZXYn(~!2;ODEjKyI-f}GfjT^@0rI0 zd}cQ}0Y5@-Ro7)OdOyMqo)Nv?#VF-dClY&^6d?-wsm(0(XO>ly{tA6jec*>0j6}ue z>^MJq;~%+~K-l}>)RKnn*1)2PMb2`tEl1%QuSN=-8RG@np+ixU8Dlx|Er9R+bvLf54EA!2y-}2wmBA7pV(bW zV9Ywrpnv_;EJ1feh7QNA#X10gJw}il&wh%2VleL9Cys0H8bE^(P}m( zIJ6Y9dKqlWJFK|126MCeK2Sn+K|Nc;;DlPrH}5P101{o1hOhPTpi`mFR=#qS=bYWh zy^4v&&seVi@-h$14MOi^wd_A;Kk50iA8#wl5B;S$?}*!46WF}&iyeEFF__DtgueP}Ad7&xy*fE`gd@Tw;9>~M{t zPgZ{{{Jkt3O&N2F*)2grqoP)zT|gJbN?az%7S1*YKoa?&HvdkCAje{rnB{54qu|g# z2t-moD(bfweo#mg^9>Ngl1VwjyYP>+n^}BRpmb=ldjVhGG1uDW z;BQKH9^Ol&Z zIcsG?m?G7Tg~$X}%K)&NCbNXRuV{$nvlWm0jiD6DRIKNTI|!9>=hth@0m6Hk>24vD z0JgcS9J6IBHn z3sn_Xmik9Ew;q9U8vhPP8Kh4jHotvpD~nb-rwlhT4mdhQZw<;51g1#S zb-bo^oDN{h&0MYAvdVJc=%a9RGUR!|2f5Hx@L1$rz!j7_F0Mr{IB*hO_>;}$s`kryhR z)ID7HDXzu7Yb9k*eH%Qi>B*>#FQQI^T1mF}DLSJ?#aRD490PL*B4GvtP-pL~tN-Qd zNU7?XI?;5Ww2hzz2DusL&?fb?z*7d15+48)uOd!~Nx7mxQ{vmFtKzfDRAkiQYvbwo z#~Vf4Ytw`~u$+1aS|mUKct>Lsy}&WibV@mdm{yR5cAz_s}hoeL$fa_r!E7*Q&p;G{8m z0;nb|^yWsb7inJ+Po&3?ssz?*Fiz2j?9ampujuJGmcNF6et>I#gL6RwAEsno6T-`F zo%yBPqrGeWfEnf$TKg3%n`oG_uP0RlMb?FPjnVKxci8b*Ub)znK0g$zQ-t1O^eO_t zXA#!&3BfSpaS&|R|E3gdT_q*{)q7$$$6u=fkq$n&fxUKnCm)}cj(9usOQHW0H-r7} zC@}gkU2rK|Q`E11Yg#VYWoz?PJbQv1cw+N?&xDx zy9b?PU$T)D%op&6jaj=;t%p(M^^JmHe{7-HTZF$hG;@k8Ob1dZ5nawC+57Gl7z33>(B+885YQ;S_n!j3;j{1+Hn z#u;htw+v+o#7lqO&SJu80Wc^4%(n>OQyX#p zw$1KZKi4rOps)A@O|Zgr2`4TYg#zsMMyhnyw_?`Qd->MsyqJWmeC92`O@ zR+h`=`Fv3)t$eMQij!24jsGpv;E1H9{queVWuKH1Vi)U&nB>8_L;k{rM)$#+O&=})IF#ZCl|dnv-ay_ z86&=?K0&lzmx*iDIs=FgO7dAY7mA~X++{um$e~Z%&97`#wC4VGK|_?g6u7!_AN7>n z8cdZfMjemaxe5c=)Xa{MCJOyHaM?-mGq%HGj__qO=NT=G7z}!Fs@5_jf)?8QrZPaVeM=tZE6vT33=GJwze5U!l$AurK`YL3{ z#myXSYI1+G=wVMR$&bql%Mxwc)GStPZP9QYO#z(`^;5Fi_2@6I3Z1#$Tvdi-by#5r z7Bga2n_2PE7AIkvq&7!p_Q0y`EVC4+06!Ou!|_ev545jaSJJZ=y04<2+jG^nU}`uN zCz8QraBqKDu&jyGXxbZJc>4v4xG{ORv)W!ZUsFj{8q$$IrT;+=PxkN3kj`{~5Dvey1F)pK&ke?tCpbvzschZ~9rMjC=y+Y!hLWO|E2mdr=%b^It@pm?SjcV;n;pSE#{ zhrfZt565JG-d=>xep9VKGiBc_&vWN-YDEZ!`tjgNHP4aY z2IR7}|HcSo4V^9zyLBJrv8h#NQ;8PF$M*WGic!P`g0*f35C);5ZlBmaXK>+H>b**S zUOO0l-Ei!p^ce?eIqw?=qfdSzrmt-kb3Jm~CskFn%GVQO?dOK1R0+wRnzBMtv_MEK z)Zjs6Cf&hTCZaDV^Evoc9zbA3S0?&UzsQm3q+45wt8LnktgIKsz7b>I(pPC+)k7!`;TM5rfNIWA*F0P zeiT3zKO(z0>(bO}rW}k8-gLK-urlJeJ=}&`+)MaQ1M#)-&P*M3#(PO5 zLoby*%cF`ZUbnKrXW!`_Pdkz(PT_l{I`q5T4Xfi=;s^58%>0u#EgccEy&HYma=q8i zDj#y4Q{_4*P8mtxH3%ba)mUmm)f&(g!t{ort6hW=C0L&KO4r{WxeWwu1irmrp2w1K zb^Yx}0ymNB*d>7UlIoDb{-eh5P3#X-Qw(F_ zxrMgHZPC-6v4~oKv#`(YX=di(+#jmEjvlJhodlP*K^+zh(NXJBBH_lEPrHTM%vQ@3 zmlGIK-!SxatL2*Ri7=ax1Ta{?Tm6h|1|{%gAnyo5pn)BoJ=()tS8 z?qV{9`2TUciv`vA@~3J5P>cNu6_r=*vMx9#JG>wCv? zb)I*IC-iYYu?MyugXhN1yhE`U9+JFuzsF?e3E|hxv9JNa7NWb$!U1<6UvhbW-Vqt= z^MK<+Qv3e0ppbCMJhAbRy8g!0#tI;E{MKF`4hK{?Om_v4PUd1aQ<+{KdXC-SJ+99j zny8;H9=KXv-(Mfb9v2E5?iZ8>U&VM-HT3q~@BasrBD7rsU`{*=T{0UF0R)TvGk{=e z0rD#C3eEN}U%`03wRN=mJ|Dt4Bp|p!2FBaa!jld@2n6SnKvL^1i&TmLz6=NX`-diw zlZjqM4n&&nGeI*BLq^4vy!R*7Zl(DXbpj^BuA-d!36)uoYy2||EMN4?_y}n@HgiB0 z2H_(!bKu78Ml}&6L7rTH8@A`)EpW(PK6tO}BpO`f^WRQ~l3fYqr!=_1aSq~WP>CAS z3^dtUmt`u#F!lTw1EDTkld)im|ZF_UU>Mn(7=~!XI`+bV}%yhNkbF~FmM9E zt_YkDmB3}kU~j-J5;&cpHOQHc$u3|IpbkS&2iOg$K?-PCHxoaPM zlhdjlCeTXoGtk2S|JDFvzyGb{|7`$ZUt$vPOVyd%9R?@M#=@3|Ez_q$014y$yg9Z# zvEKeZ1G&229pU#5_0Bnx?&ERgPNMH8j1vo4EBNaBHftV#{WA0Zbn`%)>#*+o^tc_7 z^!jA*sQJUUnFy3H56T`*>Wx^`yqWr}@4d#lAm{Ef~CQ ziUtayU2OU>KHgs!TptECYUyoUA7?Mjz_kb{Gc~YrWhnU`4}TKoHngii#AL${SECz1 zla+Yii@W8iyWRN2i`@8j%bN`Sg&l&8a8;e~pm!yw&R3hU`QyIVqEKvk%rce~^T(xu z=#qYuR4;29O^~L1XOBgqGs2D17Bp&tbG_g~vc%oOLW>%-ia7UV6ffF4(!KpPF8bd{&uwZCpOG`*A|dKlBq6zOl|9 za(=#9I>hqhl4*lTNlwm}wc0Mj=M*&q(jK3qYd|=2zZP)Cx$!mwaNT{C6ZXG&k3}BP zD(n=TPP>Uo%vu*K?GHej#7Z6TVV!co$)<3FKV@^Dz3_1Q|8Vx!QB{P0+bAZ>vLT@KhAtsKN;iBqYtw;e>LtdtY9=NRVOQUyXdM}n4LyAD)`$_mw`I6drc&g zH`S_h^7aGM-z589hUx?Py4O6G^AQtDDuCZU=?(=e@Dpx&Fg`q^@gy*2^aesK*TXQ4;%iKo)F_*hayqv-{pp z`)8nJI2Hg(>dw~uW=AL9R&eP1Hqy;(X!v}jOF=!{MlfNa({r}+c(;WdZ_zGbX?qyB z*{IRRYi=-Ty7n%(n8(`D^w$%?qrgi1+##ZW?hsX3{HODg^xXNF2zeHu+loJ*ewDQX zDaqeEJN-mYduFc3?WkzVp>jSBcozVRxhAE|&wY)l#OJ<-o%|?p`N&uFPfy}`asM06 z^V1mQdwv@4NS~g@bagaA`8Onjcq4!MKc{`U8$SP?QC*rj}BXG z1y?}ZY#F7odr4d2QQ@Ovp_`wasxX@NaTTrXvx544uA97pH9|Pg{1JbUq-xFXU~_bI z=Z~FYEshBHas7zR!Rf83QlINYV=64CFPB>%=0I4KerqWOj1&)zbWdQ9pa~s8Po-Yv zlhU341={l$)eQFOX2`NW-??za|KKB4flssm9l`Uh?16j6M|4G>@DU7^P0zd)sPGGB z7JX8_Suu0Mw&chuA8Db?yg@0uH#y{^V;k8&GX7v%@3{WaAF)QRb$@#6G+LKg)xzsH z8P*I_}#5XHHi<=XE)E*&Dn(r(m6xn0kn$LXn zqjEWf6uqicI=Xp3O!8Hry{(v|B*K)J-0NE`c{-u@tcFk}?_TaxuC6QU)JOJCP|ec( z?BMG&fA&rTyQEljSJYm4MnbA&s#i4us8{g{$FtjAPdyPM@w-tqqaYXC6PrAC)6->s zchqU$8HVHzvbr0byu}S3X}HY@Q<2W8mvhP-&CzPOTe!m4A-nv$pW(Z|dYJnMwcj8_ zxsEax`cNARFw9oo`@?-U5}|G>jAElC-oR~G7ya51$k}OI22v^AExZa@3qw`bww39< zMaVf#t&<`Aoda$LcYsaY(oFi*dcaOeC>5$KRj*qQuW(ZL=H!jDe4_i!-%|ga=8Ktm z;sn--(~3rLQrWRt2TBh&h&8`kHMrDfVkC;1l~b_k_WEb-0bQ3SE9bCSf-YFsNR_>$ z3^BTfmyHxDD|*HJ0oI)AqAy+#aSSc>><$iZk>W3MFrA_Si{TT~IOCvD3}xzcjDO1D z;n)fzD&Zxku64h$20@P&aFSj;@7&05oU;mJF`i%evMiEo@WDMb|3qkoW=|>9<0Cs_={LIAm9hWY2(uX#RvO8GI8HT~~}r4&GJ; z6qa60%IBtL7`cQ$+_t--E`n(@eZtVkZp1P@$?;@z3C%)H5Fz?nULdd>-((Inc}mUW zEKh6}xDA(QAO@~ni7v)%>*nLKjXom7jMn{(OeHiEJ$doy*9gbAiZ?3hUBGId@yhIn z2vw|Wh=PL2Mxk!A$zu^2C+7M^fY|GP4iqg?3>vMgSgk=|Ik@x&P(A0|ARbb;b5emn zEhhYQFh+3?qR@D0!F}I}8%@Uvx;e>H&a?`Yk7woQ@>mt_Mk6hB#hrakf%}t;4?5i; zWYQ~1T^oD?2ahW^?%Vo-rs%VAgM+`VS9_9dvgN%$EEIS3x_T>MlA}j(Y6rN7z+W?A zT9CgN7qGQYGr{%_T8t@EWoy}_y;qid6le#-28q=W))OXoyWfkR;OUnz|WTFDlAgQe<>I7zElQ%^$IUo75gWcRij(DyC{IM zoOW{Bo;}9PxR6qTopp`;e4b^RV`$3*`tI+Q)Th6mU+%=Ar0=ekr1bWc^0K^F^y5NQ zv82$uKE4RnYH2dxX9y6Z6#U%tel|v1{&+$`t3aB*9-vZW=s&yPscR)Qbr|rV4_>O) zRFNY=;w(y?bcvk(bPZAwA`{Z2v9?;xQb2#X=pwP}*j=Uzv1 z5!)w+BMXu$iI0KK=01Ifkq^)(*7-VwCYB`cgi#Iu?2ZLVck>UtRs7(j$j|h)^!7Rd zRrkXhhL7X}6hePFN`#($J7XXorl60SMd6A$CSj9zyt$Y)5MNCeI{EL{J)bZN{H%>$L&(8IVENn~NSv65t%wc%6^=jtf+j$Qz8DelD@SX?h zj`Imt)^5=40hC;G0aD;~2fzoGV2R)_516011d+EFrZ{vfl2Fe}M=~?ixmgcin%G&3 z<|eKJE24z#Qd~mUke>zubrOxvS(LxrzVB(THIkB;JR&qBX6AGjU$FB_)1DV2Zzf*H z3IfYL)RbHJ&+9p`1s#Y2coLk%Uy}AW@FY!c&m6pw%q6MaGR$6;0PZ2oMVZ9_tkI+a z?%T>#Xr-(#Rnl(>ru{5Tvjx+zx?OU|aF283Ci)$`+nvxgSe`0@D3(6*gF`c4Ln|d& zLWtOcR}(*nRFP6j2wj^UQv{RV;|#^$f25w`QzrG5*cM!?<5|hICU*0POi!h4> z4E;~dNaVC2NkzF+ktW^ANx5ex?fp9i`%_Ay%O(|6{>bQvHEq zAkc{R<6M3RTONrZjZGHC(re@!$ZPXPz^g2>T@OsUC}}S z$-yQd`}l&>Yzn5?)pD$R;a)pF~#=fAz^5kv)f?>kE29Z{A7ymUeH*E8O^VyfDNiE0@h+tz zABU^W2Hu06#+U3M={w-tCLflH8q@{4=9pLH&zw7V!D4MQ@7koI&!r!Mu!@&}y{!MFrnV`5 z{gA=Op6Nm;ae0y^JBOl0-h_ueg37q-lqM=ZG6b6zWTg$S+-VY{A>r_nJ`QAJZJdMs zAqTUyp!uEj;!if9f#@hkS{WVf?Ggq}Pmk|XN`V|xTJONYORsxdDWLfRZBQCWcWghJ zDGaZnuXQf?Yzo}j$A^!HK3xTWngp-8k=X|M#SgO%cUJ5zPLUG`^s&DAaVO)M-==ar ze-+|jjStc?ryc##(kEr>p&06TNwM#w#G!GWfKz~l(WF|Dv|vF0GPFFB5OlPnyQJ## zCrrCI10|G6js7_NQe@rL`G*n5#pc>kkc}I zp7kd952#QkC>x38O_<=UvHS!$#Yh`^Ncw8Kip>gAXxd^WTWIx`-5Q30M(N16#%^?- z1fY^!*-9vdgErTI{YKbu( zZKpd7ygUM+H@5Jt7Fe)t8_BnWUnXbHAx_2LSG;jXIds9S>Rt;*A!!ZrTBm4G4&E(d z1fO9o39GrE98|RuuoA+vu54fE&L3cdPh~F9(iFg)>Rz8q0y=&5^!q9SC=Kj9LIsA| zDu{8ik6aRe?GD$=T!vdU;za|r7|s4DFgZLKdr-XWEDjt3A(89 zZ$}0`%aqHE8GCrJGrgtH$r4@3b7yD5LIjx`%n~?dku~!w<=#NF%+EB??sN2K8R}sW z(e(K|Ib>f-7yD=Bg)>N)LuQ_o5JWF!Kj~nz8b*0~qV10c|43)EkgTRhnYKXx&^lSAN(%-q40m)0ldf_y8FPO6C3^x$6EMn05xlymAnGlgb=2NA z9Z^bjDMK*hm;fZ#20Od9pJtCqM8{@b@*!t%Mb^VP+?J0FRT~R}5!HOGBEc{#rsDydK*(pRIetjYH zTDFrTP~t4aiCt!FozQlvaHw7ag5`22p25-`IEv|w2w!fzJhecb(i3Z#WeZmf&*qS( zqr+jH|E{UAbRA#7@;lGlhh0SG&l%F zFjOTWt0vzoR2^wUxC?VdZ{f6b_IkDb zA~7l|DFG!l+P9F>p`_gcr#->>z#zcFX@SzY^*{&zw$^GiLV`ND%*l?28V6pcSYNRp z981ZKf_yR}aZ5-r4(Tli8agm8BWYcpME2vc97REXLG2TyWj7%gY#C{O;!w0#HgY)p z+Y2B2kwOa@P;CDmg#-1pj*O;~r8^f$|bPX;pwu!EB?3!3-{q?rbt=xWpxVZ0^5>y?a zDO}_EHP2^V%VCxTaiaKCoeujJu}t`@lCGee~Wgo zSC$rw9a|xm^f&1u@mm0`{6Z1hjU~;kE|Hqv*NZ0H{{1A24N0Nz%$~vic1EJ2eVL!R`bN27ZD%op^+LX|u&!5GThg>o{8b@<&$tABWpn zFjSQyT!y}oMFwK$8^%3J%g;e%gJxh^eXhK~qxhv9oKoEFh2-fsvJ6`559>oaF{ozR z;If4AfG6l>_w4^<<#=8AE@QVL(;^72iu~v+WxJiy`N+=iqbn zPJWme>K253%$SwYaJoH2)%=&5ttD)kd2nH-Op-bVz?7dn_*_+IPBo}=9pl%0LsfXL zUr;=@pY^CJH>WKLQah`xzW)lj9fxeTh59|I5xX8qbKiKckT>USR-BlZ7lCw|^K7)w zAsL{9iZ52i8>5~16d^8(&G@*uxEl)sA?tp}e*5y4brv2;>`U<^u6Z?cqxz>Rjf8(}E_Lu4%;k`{F6ryo1u}`H z2MDRt2_C3eE-2}A`j|1uK-gqJo#OBuz`m(34PDVCE`^%iJG(VITr{Ad`@g})VsOTt z`csGa*6qw4kz3(XyDaIXIzi)-;qHY zZP5zMc|4Ym*-qWgjKqT_KSBH74fP`BiOcEO+Cm--@kTzPdGT5{~K5gQ&7hs4u*3K3#NDQV!x zSrDBJWgYTaDb+ZQ*4CB*L|+8;s2W&8ZtQ*GOMvJjO%%2FO1ksmT3}0_&`WeT*wxz; z9~M?Pfw3dXJzoyB3e)Rej(LkAQF$-pz_>=&%U2JLDw&oQjwK1>dR3`K-hE!zj3Vgr zi>_Fa&YTb^u3-^kSrosJYz$Tc%v=)ph8mIBF5>w~T1)I0NAS{I~d9 zH||I;z6bGwkL*}oy|cOR+$d70Mx-0>8mdVPoDibg2pz<8p?;)`szCKEyLUI*We(#O z{pUKaUFv!&PM(CJ2T1W|KUwhRH!(K!QsY9~hR_9Zz!iY4lI97dG`z9)j3OAjRK-{* z()Rh&H!>5GYC$TCGx|z&y~e8UwK1n8_k@a~z8S+;3So@3f6HfRB#JajXhatrvS#VD zT285wpa&xTjtY9zr96V_}l8dmM%Qi{d>5hj<1uLh?Qulj?eAR z!2vWs#pljAQoUhsjbxK$9LuWC76S6njW14AEnJ>Ellk_IaCb{8|C-h+tlT5uc|d7I zVD9)LtXy?JV>v0uGSBzHA|*JqC~0;QM_;T9P54#5s1{|rs8*!WTw1bSRWFB!GT%d# zgO!0Q$Sh4^RMo*&K(&lrOf6USQJiqlIIlNBQnuiq+}IUu_9u1&j%I`qaF;tAJBc{5 zV)*nbl%VQg1WAF4oLjc9Lh04=#ciNS)B?II|w9^--&zCQn!OwGv*hzHKJG#lQcy*TBY2#fHM3j>=uW>2xnbzhh!HAtDpd zo@~As{`M76HhpVGgfO(k-HERfV9#dhew34h6wkL0>b-Y7ZYeWZ&W$Z#OVMr|<=T%x&RnV~G**HV@97!C7lAT0=m>bx?Ai4N;WUjTp)xG4VuP9xc zZ4<=!YmMwvRQfVL9w&My1q?(Hwhge+6mNBrR*(KY+@Hwz9T6KVW|!zNuenTwgbB`Pr=EwAAIC zarEb74dfLsma>s9JbEMUxcKHv#^ZCa*E>DlM*-xEYi5pB5;ET24+6<7_I+1?+GE)f z<8#spsAuq4Kc#lx7WF(b45a#40vC}SSA%f!r$XmA{at^mX>**ii<%#kI*ss}hZx^F z6D51YIK3ca3E>BKM>o`~##Y4j#|YZ0wjj%KkWY7U{rpblSt!l7YfXYb`>7QHw_8A7 zbt&w@eP8?Afawsc!qWciPVpEc^*f7Y*s|7u%lCYS=*8B=87K1#D~HYgkuZUgEq}P-SPb z;5+5pG>gZJtE*U^hW7sgZ2R)YDvLY4UjfzGL8sJ%!$b!Nl0#@9_jWckL_S)F<@6N-jbYV8EuYRTMzl=&-TiavwDL$Zt?H2Fd9X{Y-Z~2e;_ey&{&boeiDc>YrEZ?p+Jj8(r9u%ZxyJ-(>yjjrtk}-^!X^7;*4n1~S z&%Gre0pSJ|7;v3bhw)3^um}(}c?V>%7l(2H9SajAHcJzoEUK!@9@mRTB_K!pwjKus z=>*V?44bR?q9QP!w9L{YcWQjGc!3Dv=lPhPnR%Mr`^OP4&D}v-?C?vkJYf}8+Abzu zVhb8#nK(rW4amn5P=>SwBvoR%^m6*5bgleiTrS88PtY+bum?lX;%Dz(aI$|P+4E^;c2>(ZF z5TE)Ygi3;nS|o3ns3~Qo8dBlz`v{T+kB($ugzc~QITd7?&OPDwuPO-`Nju1j!I{lq3iQS zA4%;4x|EpRcdgWhjc#vjzRNKFVuwzTV8KGd%l%pU~*azIJo#Urbwm3;$ zY<0gT16&JRz_FyAkvII6kI#;1m5Whf7Bnh&(@D-taK9ELj>2L zTKDj>jwi;D3lNO}6t-)@)uHJA+Z4*~ha)BP7e75LB9xXg%!Ga$MD^Npoqv64NU~W9 zv{PF$LQrL1>Jzl%)D$Tw9}(88bZrOtL<_ycAoRGFOjJps03E5P=`rRzVC7>7?bT+) zszpqu)H&O}@)?L{$afIHCfE#!$dV8zaC>VCg&_hR?BZq>1T#(s)Wq^~IdEXXg>gCf zNC!^l*p4cuGUOi+PNRMaJ^~wilegPQIa#{5_SkH6t_BX=G4m2F_=s+xA<|moynDDM zi#T1IwooL~G z17lHmcJ^wINVcqng65k9w*Oep=x3I*@E^Pg=4Ov`8YKFcAZtDbIjUa#)cy;V17 zeG-I_m;>nH1QCC-C8Rd4hww2(2|_13++Zjh^xQ+I>wNB^Cx8K*GZ{^;5|BMS@rQ#D zI$hvW^iKcK`DdTVr=9X|`5u3*hJU7#&l;WOsf`oz*D;GT{x*KHeq0z}f2p&z_S1&& zAi%FBCH&*T9)B9Xyo`aAG)IO3`*hUbLzE-5LFo##fHdd2wrF0hsssdONert#rX1V~JiAnmLcoGW!KT`#zm}%Cc(*BA9%dh9n(D(={?5Smq+gybu0RRr zM^pl@Pxo%GtK|u$`TOw)Uf{LcyoM{KI7RIG84NR^#Xe@V;>2&V%y$cy*G;}_$qHV; zcP5o8fMzSS#Z6IceJ$zFOojqr(^q6f6l|qnZ4YgCm8}>t_=%8~Hh+{} z+RP7Q39Jm^Y;zzR`o8z(`711zZT1_V4(pAO38%kI#D;IAV44^K4vNbxSeT&L{5~P8 zV!yCQHHHlCtth$&qOB#l_LB2l2JuUd06HD)t>ZN+t*fkZ@UslK2`GTnPi$lcOttmKr{X%&ISIQmGezfS`WZ{Qu8{e2%&I{I6zCp zgg;u5Eu#i8hf}aYu0kzD}IT`MzQ2j)GZa9y@t?M=l^a>uB>9$Ti;WJN@9L64$`p zIhx474dN9gHi&hq;)d=7e{W>zw?kj;@3?tk(fMq(M??ZczOR8^1SJ;_J|N#Wv~z$s z4>EeVAPZ+qwYwt^3v`08N{e;fkbg;xjF2mu;GB88Z1(qye##2ZcB5gW@hrff;O7H} zy^G*o$JT8Sezq_WKV_meifZBmR_fMIVhY%tX$;yy1DV`%s(>v{$CNPKXia^J0$3^kto2cGfJ=z8 z$E{yX4HH_Q!fCZ1YA|+8fSWpGL}0SWPGO~IP)-K=LW+b!5CUqju*VJD3j(R`aBX*} zUx-)wcrd9|YD*2OYkVu-*-2dl~@uP`q#fhPZZz>nj8Vf37W3a+6U?Sj?Lwmd0pf?x}G^MT_Jm3#1wN=#USSx6F7~u(=9&p4V&(2|S{Hhq^m5HyX}H2Fwb% z0KNvH(HOY=&(#!zBj1y^xZ?b?1IpKi8LIzsHsExrc~ae)u382I<9Juk?R-Fy zu4T~^bG^2C1mXPTqXn*=yt4{=7Qy|mny2S1m>*JA3GGv<=^eb4iTxp} zJ)8C<1~PFkN6+V5cCRYZxVq6_Xd3OkP$3*&v?2Tf-Jj#kZQ%Ox0jwXq2$uth2A|B|m?^a{7amPn-9LMZWN0a{Do*_0%$<{-hrn6i-M#<4v?~-t z`PRv%foOo3C_@nbmvCJ?#uoZND~a@vTQpsDUxINKtMBtM=N?YXU@g9$dVXW?&HGz- zh)!pP;#FLDt`syPJu;{368JqnS6k^QWHi*5`X$Q*RPIL8LwWJLYG*+l!&nymUqj} z(@3rh3i(&Y*6UvLtsZ{K&A+{LAvZT=F)B(c9c9YS?=I&sZEs~-amrgI4TY9QcVAcv z6!ZN48)&!ToySp+v2)7Y+QUb;F|0+o=ODXfPf(2^XZS(VNEVrUB&B8EW%b+ldYM&shge@9?!pMMPY-v-V$uACHkH&C8x zJJJDe7(KWn(^d%X+4=n#>#3#;33x6r_n&m0&lc<}6i>QPROoY6^fm0sYegLRQcSv& z9~CME4#hg)E-P<$E#W;jESVq~kz?X=ELgQ{bDsdy!bs(RJA6+&LV346k>&pm2tS`a z{I?@|GWLG0gA-B$pEvcp=*dwF z3Me~vNp{WgLoBJBF4nhbRIC07ps%8$D>)^bv+_v_U-~joWNYsRVLQY$W*VVqZ(N{- zO&)hFru87&+vkKb$dlFPn;F0*`d<*KckEQ6`Ou$LdE`$Jh4EffBw3y$Bfo#L~KF6O6@r65W#XfrSC7Llu75c*h{_}``-%QHmM5v zu*ONJ7b@@;t5GSb`LNGU&;|cDn`iIr|2mNZDvIm+>HbE$sbAw+E2;;ob)Gzbx%dkx z@Ogm(U*haY$?Qd^hegour3kPPy{z(qPEb*lsd=wy*vknVu4e;1(U;QR82M(o5iM6` z8eDDP%|5HMeh9apKN|Syo@-{z+sV+-e+Wa0%9pp-daLLiZ2Wd3989WQnNBchQm~4` zCxGie_0)UG{LZ;dGU~jo>%M;}{xYba&y+_9@1u;zrS4O=U-|bAIh>Pd1E0?n<7WZuxZP8!{muu2<-f*% z#R}6WbCqO=>lkQ|*UoA++mojDc{9oT$Ic>S8&H@ZidV?hk+u>jX<2B;OnW}N511(Y zRQxjR;yJ6oZaFbYx+s88tM|6mQdnac-pGDjV*K$#PHj}|2RX$#DIU*7tILlLzbrW{ z%97dAN??koi?_BXUQF`UediC7U>;hAa?nUiMESsAlx>_Sn5=UOdwPmWWX~C6B4K9F zb$uAQ?EI1Qax{Dg@EKpw?)tIiHu^&t&6>aq97g|BG zxZP)}f12VdZ-#0!)gLcS{PFPl`CTddC~))Ohape?e2-z)oJwQ3_r?;zz&{7= z#((xT{7-WloQ?mU^2|Sfxf4^Xk(dp+qm$Ux9G?dP;I{xD?L@#wdmx&r{#MoT?t+%q zNc)-6})-xo~jdYJ;;>O_4$1ZFNbTGcf6t{VW5LI zQTNAJmT4{@jS|VpU#(7u!e$O?{ITbSYof!AR>cxs=Jg;96wMc2;=uhKZtm-iEx?4! z7EentY#bS^hE#%JmKYR~gcN+Wuf$>aWKd~|s> z38Ghio>cbyGYAp@CY6)CY;JUCzOH8ChEY-<)c-$O)%@=sH9&+45PktX1z*zduTeQX zs8$@f)#HizJnYu=9^v(`Un%J*?)d)7S}|XFBxpXk0Ae#m_41P16}9=KAI(UZn!82y zxWJkF6R0bA9UWmg5#&sy>AYsk$lO{5=@C4lsJ{ha2jtOk)`x%Xy7zoR*7;O?&dh7? zpNYgyQ=Ami5H=ML84t}LswmiZ%O5gXd}XgMhiu%PI2Ql7uty^8b>Vtw8r*-mo$$j) zlRR?Kd;ZnFo1JVc;PD>g_uZCY!P}#szJpvjbYWNC-M9L*pz=`I87fql52B&cXg@~5ione7F>L= z_>XU5dxo9<=gw2brtt58K-;sx{(o$MTf2VC!??78x!x>W-PRRRRnq<}jjXU~GNVy) z`|uu>MUIAXKj8kT`n%s&%;bz`dBO+YlOtbNck<6y7*+wwd*vBZjeS_kck-)%a=QJt zP$SW0YL1!2854o|Cw{pGufKDd2Cp{=!$`6N7uH0eEeif?UW3?Ytpt>-WN&=8F$sj0 z(I69vl7(`!5G#%?8a|ZqY=fy?HGGV(7b!T$;|1=R9wt<P&< zSk{XIYMFR+ZaFAda!(L}DhLBub3jWJf2uIy@7#;AOOJ@w%<^$tdu)S-qeZgxg))CBcsbCNrv71y7X+UB>#&y>r2#ar$RUl{)5Iw zeW+!?q=rb$?EzhjW{}V`Mt};YSGsZ|3?F7nFG!C85hi(qZ5j!RIcoy6`s76f03?^$ z^cQ>}n@HIn$@b1(wf4HX^i*~C-@i30Z$iSC3obc5aFviG+do<_b16X4oZe`Ec;io@ zoL<2(s68;6X*P7u*XoWU}U@r0Tn!;QXRYmUKo z(eH20y-y%Plq=f&4gKSn%pc+THF)zuCtWv!GGBLf-~Thw*>~MBdXF~#cXEfo_9&Aq ze17G!_fL$qn6kFT;VY!%^GdP^(kP&W3ABKt?pa{!fQXT|w0v zfnDjRSQ{aq;c*I@C7WhZT)|z&cVA$cK0ZW!P5|(!boQ(0h%TRqOKW2{*BdsqzXU~k zj%GQ{qKp#UydBF^?207aOWsyRK1V&%O`1$zsFwlF#;F9F4_bSveClx#+cB+13&mC{~hZR3|7gF(z=RfqW5fVddnorD0tr7Q7- zbQ7OUp*3iAI+b|$WxHlA&5^vw!7}xA;D63DeJC z>lgDx+vvFd(DZMGk0M;M`YX@z3V5u#mE*Eu&5I9yJbQ}N3sY=Y*pu*%A1C=T6-w~C z*Z`JcLHvAFZ6WAlP-QRFG3%J(ChPJ#2v#V;Nu;q#_iZwnl96cn3eHw}+fis%@)u(h z9e`iJ{fzW=UUvU7+#P-Yn?x5A^1QeJtVmGDD%Xpy{Sj8qzZ(PyZfZol!eKqpIB1!J zI1h-6RKwK7rXYxXiQvnjAm+}8_<3<|-*;H`aqYq8IS`1?D{g&k1`L7*f@xmf?H8=@ z4VI?vZj~Km-XXT<_0y=FLE5?tsH1}~i7?%Mi#Oe~#I0Y%Bh*{ryI21yCSz*w$#h=5 z&Jyi3g+uhb6YbnKf=5hN^If6BQaWaxh^1%aqebbWop_DNmL?tR-q2RG{=#;tKxsT$ z9aEVi9r`4*Dx!6Okv5elZ@kx#(I*p-j+;gOWOZQ{ekoTH#NvVp*i)H@JW1veHqq_| z*~}3#jBDt5%$XsGxUAD<7@58q;QGugIJ$#TQml#!mPW*5(r6nRb|fb(p=U9>I3n}1 zYUk`dTHH_-T3kzDf(()@h2BnQwPvs!MamjuY;Ev>dY@qEIXh*{@$|Y{`6Bv(NpqPF zClt|s<(=p69Uw1Q77AapFH1Jx*-KLq_y^FpZDdw!vemZ2=Z&U_HZhNrJ>b3`z^{OD z=M&PPAqN`iEnO!=~HbbA*Q3Fbp z07M+61!Gv2_z7*6A+DU7l~g08!gEW^U)#vW93&8F8<5h1PTv<~2JwfciIi-hz+~%rPb#ty4Ny+DsUCW5t{0SC zO(Tpw&1if&*buzniXefew^nf>!j%T9+PVINr25lj^$JjMY*L$rc6@CvvOOtq4R22htruNy+3=k7QyoO|z{f?Y?8vsVIl~;R6VDU>C z<-1=CrFvb+`Gu5Xje#8^X`FQfVfR^?ndYV={P7Z;2rnY$i|BxmrjLJ!%`y#t1{%}L zB4kR&;C`pjGXl~;djg&m71yJs`_A^b_USdxh&U zm`mkU;<+#akdrj=I|4yI_a3Nt`_EV=ar)RKrl4|P$b8UcoI&N;M_XsKTAi$8NN=~I z7Qr1?un70GcaaJM#pq2IUIIQBdQun@N!17Z1Oay^o@DgQRD2Fe_;@=`?HaffKnlF@ z>M6+MRS4-ZW_i{P!|aQc$D8IDcJaR0cS=lF8#33(zlkb68Cy<&KZvRN+n%MoZu{U& z^oY&oI9E1)-CtV+az^?>TO|0iqjlU_1XY5oa7_bUnH@D{1rL6ivRC=ncnU1@#%P*) zCrm&J{5A#q(CGlF6OmoH)@08co%TPtnvDJtv*;9&=g&TP=oCcjWDM0SKED?RUl+Ei zdGj0p&d-BEqD|R$V=K2jQxty%7=A$~?MqP}!dx*~)303iTE=XC=p~jBO2};BqnNpw z!_7t)Yhnbz%Citp&QIlA>l#Ca*|ia~3##q){-UxV`vsk#(`C}PcboAoHrlV$0o0t~ zs5Kop@=Oy`8%(SHUU-XbcqqlcG1QM2hqA@oyXeC{PKKXMZRJ{85MpscuAP(pTCHQ& z1ti1l7oGNTz_#kaf60ukoe`uPKnq#^Sv7Olb+nvmCxHg49c{|QW{8z@#5ENfZ-V{P zg_A6AkUGMB}eIA2s4sFdRjtt+%F^t&Retg5l z_Bt1>Gf)*v0Eq=dHPyL$90=R@@wHi*|$3% zFdB*6>%}fIV#!0Rcmo%=5u1@JUA1asXEu3N@gJnuknmjwDfo3?uM?yN&Jl++=CgR9 z5yzw{jICgW+G>+eT$B<^a`gzly!FB~`ApZa@IGz#l$^}>`^e=&sg!r5^uIcL<$_^P|G8wmyFtekLiGiQq&#-xYTqs<2tV z2ls)$hFM$f>_TPr?czLQ-j@o7J{cAwnR?`>pVjdl{$Z08m~8}u_vw!46QTy`jBpV4 z+@S9ONnwN^{8zaMJ|Sc*VUbu~Cl6_+Ft|{!*^OCY5BduZ8626#0ot-L(GKSn?Uq$S z>?Qm-YC{`p99(+W6qUUcVhoCFS zEa5=9!Iif=1}wLkvrP|72{LVHp^)iDA*XMcoZMZ1cU{3DrL_sEYE*8@I%KAVqlPG4 zi0O;_UTm1sr1q3AVF?;$4Z zO6O6kJ93Y@6Op8qx-%^aqP&k52{_Q}Vs*Serm|gKPmI;$Gp$A>qwdiKVaTOwOj;A$ zDGQa()nSd&4Skag!8tk!h$MvnKxsshM^k0pERss_)h9*zUv@76f%Pef1d}M__>V$6 z;m=WieQ2*nefp0$onwfk+CF~m6vP6Mx(@_dX^7v`dLw3$CCZaDSFwFSUQitxwlcGO z?S3^T{)X~L8Y9{3KraGy-sB=%k|O;#(Ru7YJ8#o#0<3i3EaRsIdU9EBTvCRZ#wZGx z9B+x8#cTR6^LfT}!fU;NuU`WkBJmrJ2eTIql8Ok%gw1ufS;vL9LAZ%lvbTf{+(5ooxLQ&4`RdwF9`@#>>)n zpFF3&9wOe8H4$F5L5gjz-;O)l)1HdoPh+5?p~rU?UuK1g>>*+of%rU-qZbr&T;!&s+}m5!EX* z?9hew=4j7DlLiO5g!gc#nAsG(if)J0q101VhvYpg!AMy{<@9m@w;4rVmmY>;>0JCW zR{pNiht8aOYNihOxLj~dw2%BOXDUbLug_DM!``KC$9AfDJ2o#oL;PyKHoQ~TD${?2 zbE{K znOO^S1#fLWqy5M=a%wBXyVH2BA6hyDg6yF?8fu-2wnKXUpvpZmF6TU2?}2h zr7hG$Om_&StI-)P1yE&LNc2x@C*C^BCTw&#QmiPtIg!$S%krT@lSn{{)%YDs2OZIP zk5b=pRJqpZhS(kb*NYn&ulO4d#2E-QQ((y)t^8VV=FmD} z!tyiB*f~;+xLS%_BfIirvHgP> zStFgCTb!?Uo-Wm)$+EHdAMw(hv1324Lp|s2C!k1%j%~3s>!2SEdL=lf^!nvqXd?o$ zvdroMyx%w4VYMh3+*76ws5Fa z1PvP$hEt)`1m`E=h>rIOp^fj(x=ypBhu5;3hsX48{OGcbMq?>gVSOOi5R(rJ4mCQQ zyl^=IG-v1u)%9dG(W97Gh<({KIX}4chnGxkGejy)I73|uCug*IWVH>neV!~(3&B|5 z#c(iLHChks=`6!62^%P<#x~JM$MXGdskE|wF|z_ICfG8isY_F>FePF0 zL0};CrT6+?W^+fR-gyp{Xsl2?Rof?$AKEBuXp74CgSf{q7Z_)oAjhrZxSR|W+ewb5 z+qPE8n2H1>t$Cg~H>OGs#*^j{=Mj68z0zN76S3yQZZ7VvTnSp(@fsc00P4wwLy^Qd z^uXrO#1rN|mp8oP2Ac*~Ti|fxOkHrSeo>3s>Q-6_PQAXO$+FPSAQg;sDxx$oL~DYi ztQ3{HC%tf9orlYd+eLPDQ4sr(h2cxc?g(U7IiqRI zWxYYG^~wsMkqV%Jw2HMj@3{6Bh6zi|mp3OlC%Qfg(s{23ik)21>h0*m3}~>m6thjK z?dS^#H+IxawWT7XD~xpX(s&9MYF%|kl5;~bM~p-RKKg?B64bL7I8<9yA4U9kw}~_M z#np#JA_zGFpE|0JU$Wo4YaWLYe_{eDQ5_#QFa%P~b&lSB?oueUp`KDB{_1&U%%^61 z(vYX4x?>=K4;L$WkQczmNA^oC*EAh_oggG6>BMVph@DB=F^kL4IXIP|+8I_Giy=W+ z81L8oNB1O|LraZC{G{oE>+~^#x_&N2!HznJY@ZJeGcc!!p@unY?S@n>Smt?I?Ca8f zM?i=5kts9gO!yd1x73oaq|Re--%+^V;T%iR9(j)b@&cU*nrSu4F5da$mBX!0(J5_g zdnLC)yB>@^V6bOk+JTYr)s^o3ohw}t2uPtLoh1k-5+bNOv+>i2xbHqC<}M zlj44-<*?GcVEWO@a5&xM&sY55K6^q|bO#5P3HoSTFFtix)V$+f{*exBUY-h|;v5W3 zsO-${W97qRFzN<^?l|pL!Q*GOWEPK=Sx&KAW$D3k@vWRcWbrfKzQ@d~+XhC&wRp`J zcUYTbJpi)-ooGSOTbTCs2STvcf364Lkaer=JjVcXmF}t8?1%|DxJ0aSegl!i* zsdBOT2UqFEiz6o4YUWTvbs0f?L6H-Myot6tb5@h!R_~s&kR)tl&I<_XIxz-(4kL#- z6~)l~(kcR98Ars1H^hm?259%F*ukO9HTq>wVuZ)*eyk$PW)Hj1>0&*I(R|tN+;r&H z|Fp`yZ}Iw7Yn-y7CY))o#{X3)abytz6yaL*j0BdU2 zxS=@&qez1|c^WV5+$z3})xdhcMl|+BQov~$z7TIck?@#wkBGDUQ&Q1=XRE7*O1<7G zB9r0^afS*;t<5`}-DGo)a%AnTD!@&Xc8xo~AOyl%s}Vx8+2gDSfhxqxO)bm~`NN_C zb6dN`=}};x#sR@q3{fosArE8G;nuy7e-`*ILGo3Wi^j$=`XOW~poVZ1~s9QSJV_cDraKIVhQGXfjm3^GK zMU!8}YZ>Ct3i0HO0e^!2{JZ`Szm%f>68){{Lic1}Q+Mt4%~d#XUOqv?N+E4Eu65gL zxZH_uxca0!DKC}Hhqwcw{;>giLT1Ft20~#hde}2(x=q&C_H0h}j9ptj0zNwdo)3i> zz^b37Qa^t$7&P&4DwDdrB90U8hmec`-VeFw>Z`VV@Xr$cC9s?LBvrZI@>v6}bD#~Y zhA&=zh9skh_*nZM>E)r7(!wEpgl3-U_^C?HdhN~A8yQ$1Z*=B>CB6X7`K-w1iZM-PQ zP|q@D031H3x;I|%wVNA~_~E`qckdL=fkpE`jcoZ(uw>b^OUr7q682JG5*-|HPXX8c zYf$YeSJs4qo=As@|F# z1PV|5Bm4cIuan5Td|TE@hvbeVbes6hasKx_`7K21IhP6`&H&vcXLtI9stS_cv2@ zG}u-c$>6lsCUiJ!F&%Qh_TPCCj2_u!_qVT42ofUT9>KC#QpSIb8UOH}l7nRa0Kgn( z3ioPYmGRg#HfcK)XUIMt^|W=L#%}m|3e9x9)TsD-uY{gb7(PhU?%N-CPSPWn??+*U z@r+@i2X+TW@t|6!PlI%nRJHqs0bjZb;?aO^0__vo{I4JQ-Qs5H!VOv%hru1H5ooN` zBO@5MHD&p#Q7$iRvde%*8F4k4khN5CH+B|fuLJ+CKs7wgaPtpMd>zN9<(`~;AxBPg z9#1k)y$q`^i6UYpVOP>l$ZtpH8r>orwy`HT4Q5k|c8A9yGW;DUX!U5v|4x>FXjpFi z+G`nH+2e+mD{X3Y4Xz*amA}>1dja|Y3kg6+%bYf1DXy67i6@^qVYaj)n33mnMnUW8o#+RySK$oO*_c59ciX6d?;4p6tfoh zRP*|!X!%l}k}P|(=OlrPSxg~G$#@sj+GxLy5nJTgT3&}sc!=wo2=9`tR4vs&LR%Da)XvRx*(od^`iZV_lpjTKVy7x&{{A`9LcwOnfvu2 zBsa$ht>-D^qK$_p9;wo(#4p$CWC+Uh$SX+C81?fXjq09aSJ)ngU`RBU=+ziC-Wy#K zJM0F}UWkmq%R&^8pKuBk5-bKL(pj5yOb)S6hIL%2ARlCl4`j<^e~RXf$WF05c!oK% z9(y>4)suHCi?IL5PkF&1T-kF6&nJ$=jfNG3oc;AEU~9yC#XWh589+^b*k>QYYuNKq z0!XqaWn2DkWE>}Rk!Gd}1*01NPLVX}KQDq;j^Z?#$P2z&Y)n{pmwlcfE5+reI)C{f zoO$ig?>ti4{}ADpz20RYdDSPj80JA}OCwpqhzpRfGoF#?A3@L--ax59-8iMMm{jr@ zC%+M;cnNbpbMlhch2o+5yidIvfnPWT4mL>dVzEO3K zKs>za`skNRl^JX_VT2TsWbqF386E1YFY|*N89x}gJ{0WXIR0gb9eAWqqeFHq*ecP( z?>+}h8~;=nZV5akvLf<#uly78XJEa{*UNrt!Ih@6MPQ+cjXI4%xZ9p_Ej(oGx$wi# z&gb!kbtGJYst&rO)2P5dh$jKeEy?z)Y@cabu5KBtQ>9KC*4J$DPv8DIBGTOa-Th7R zz9A=$?;^_*uRU+}N$B*(SbbfktJp%RaecOF|FejaApQC>eRCU8%sai?FwJSK>O$+IW4i+^Xi~Ioqx7i+L{*x=#O4 zueSPdbRJjj_r#yG@0MbVCQzUJA8s~0zgL|bs*Jy(7Z#!2cTNGavtbWT!ZO`|uUL$L zaQyys)VK}5m(v}kxN~3k*R!4T4L@Ac=L?o6;m>~`tFx(hjIw$!dAeT>Jnh~tZ~t~Y zUe3l9`w!_ato+66I_!)44-uO{{yW#&{Xbl5o?d>$y>CFSHG{928*4@o+Z`Y_wF400 z8mJ!VJI`9?ze(0kftP{wYR}Qn!a(|^QRvN++qbKy(_)$qg|kI?xT97XP8iX%)k5l8 zTqvbB?x4-2 zg^S9nbAdZP6#kXG9=PZ4O$70zdEva|jR9U$nG%svY5h7{@}tXiGS+fTu&0Ii`aGLn%NUaPXkR zA(-S2S7R#5mFvJnytXXh;%U|%iF5J15Gnw)D-Qktm(8tesLNUE{x-ciPGs{c$S~aT zHkIBXLv!AG2w|xYbUb8e+{TYsPu%RCAyikFF4kJeI*sy9~b`HFV>zy3Ri<_CJW!12YR=%-nOH@9cKG_zFeG! z@x+PvJ-;box@I%byuISSEHK{RZH;XBxWB&Y12CrUZk`|2(sd@TmHr;j?i5FN`#c}+ z$I%$x9sBXUa=5?jT{OjxVglF`6vo-#V>LC2){B&6^@^;K) z9bqHYoZdUVhM}fD74l(;pc>~B;mkb zJuTQH57Yj;BS3yfauyS5_>pxV$4^$m*IDWLKXX!zkQRK#`!CK_80rWDYUPI12bpCl z@Sz+wQwn|y6!&$yg7&phdbb~3u_9UHIYqweCb-()0B zkxx2a?E2QTG4FqQ#tEIQKYJQR<1IlQkTcf5ae>A{m)bmg1$-|-`eEu;gkU`G|mmefA8wtcNXLxj%9U4vpbXdq(!*kc59XX0P7?E7q*4p7XV zp~I6;2Rz7shCrXinVdb>N&^IP2xP`vV21Ne0zP^H2L4@+_rXQ&uQphf9En9f*i1PUT84vk&&{0fA(x?dCla*gs;K{{*j!6K!Uj4UU-p^?KbD_|h;{9+&BRc4X zrOR+Yv=XvQk)aQj1|f0%^6+Z$^5+w@b!gb%0wD`}&`!##?46{r7Ek3dww(BNN%tBf z4_tX$KzQC|Ye;QMD3ABVNmaJS4Y|#=Sb21;R6i*ZM5Wdgy}(vu`ISj?06=tXAu6E9 zg!83?+4{C5&Ev~wNQlVEAE85=%T~u~f|i;HBCez>wp-wpU9mQ*1c-*G4aUzgKe%3h ze-E}{!CWZv((CN`!e4e^VOnHp{P;!Zb+q^fL&v8uTZ*Tu?rHzyhs59e3#91fN_(Hi z!6W53L%J9nkBaC5%Q!6UbLrmUN1V{_!Yz7-XG~*(1pAKiO?xi~%PUdv$)CuujaLE+ zR7Voj0P!sa!ap{?&26L)k;k466Qy?w4Y}$z5LB1e6SYqfq=b2}gUXZ_m^7d(&z7qS zwh6BCBbzZ!&n!rkk*9wEn>p$Y@yhi4z|&CG-Bgiw z0#&)VL=OF2+CIb!>W<%)87kTPPNZvf@)qG1=26O93Tm|CjzZJd_N8ipOm`_VwI05T zELF<3-y8&a<}n8E^fA0(InSAcYA_-dCOT9^2B~Z1;pDLXA-XdK;j@@k{$a6;d`G{5 ziXBVT{)2$yzG2TK9|ZE;{Z|j5ps;{Qx5yaUre|f_GiE#6pQ{aFkeaBJrVS}%BYQvA zSb9&MWV!Jbd>JiFTO!^D=LVblS(gU)0%9WYijUy;9My^2=9kb1n-C$|R>;XpNB{u* z6toJq=J``C%gu1pcVHVf(=L&A6{$vC*49C|=`*xvYK6&yN!@D6#GcN0L8!4bVqvvx z_%f>wy4zkg$~T29u^@EhZ-pjx@?;3Om7W5XI14{fq0oG)`px%&glk_+BjLZ2BfXk-Awp{(D-?Qp+l&Rca4dy!o~jql)RwwoSO#oWD%i^W3TBuJyLn5U7g7 zhlrKdK2^OuSrm}lH`&PFfK{4Bt*k^UaEeO&Ad5*ssKFIloeCfd>+1h%Ksf(@-5@XC zs3*l?G1$HFy6fLAjLPtx^j$3HylqYNIC7?nJY~j8Hx-CU?4(X9oHOG|eqR^{$>PCM zn{?o`-?q@tD^)*!em1WiN*?-ao<`B$Po%>~jhGKzQ}XfIhLABSBqEl;$z~_5*L=X2 zjYX0M0Pi5`ue!eN+1nSSVl$=7qJ4!$&?KyUmg?CnXBP?J^2c$eh6>Uqwf5MV3`P~_ zF674nVj{N64}!@1Gs42%DJrVJlUEA1Ku8{{RVqu^l|o65{Fa%tGdd1ikgBFfi4<;b zngiK2I>t}7F@0&POCEaUD%z7&POd;6zzI$nTb3s)mJqH|;7L3fgmTc~jDS9CkVD9+ zd9Qs=TLDrZkWqvF|GHgv=$VhKqHXTW7&7=ubZbY%YH$kkkLp@~{PyPkhqF z(Vv~+!dq?^o`^rF?-zOYCu2_QX}X9c%B#OB-fa?CWeNU9kdR2jM5E%NFz&YEwL@sv zq~N#6CbSavlVgeJT=;mn%(y@DBNGuOoYRlF^a(Nz_brg)(&ZPh5Ph*4jK-(2Rrbtx zkR<2-)qw2d|9Jy+5#&brerzOjHixqOrL?z{AgFl+xY!Sq55LWa~FpEqIFK@s&DA(VYrTm<2*1i ztJX>_FO+$oyBsM>iV(Be8A6frS`O>sZz3N=`qc%^0EQ zSrskj^}_zWWjZ(gQa)gMF8Yy$Vv_0P&rPT4-Utuo;8==>G~w-=;QVXXvd+leb00Pb zDK1BMz_;u~_daGyqK%1D}nN>wcEL@hhCWum&Rp5-^dt!t;Lr}&E&qls;t9& zAjz%wTwCAXOHFH3-=6Bt-nsjYOKfh`$*%6Apr8z-%9A<+nt{nRV))FcN+)hR{2i3u z9fWV4$@K-(+6#X%QaYjRT-u5}^P_8@NKE}3bjr{L{hwnXlV}wes@fWxvwS6=pX0(z zp|11WW>ukl3DVU@m(uwRkrtan)&JK0Xa@MDx^Gi8 zt*Rk7!&_g82EWg$r!Uo+fSy+=gUg#|Q%1rjJr)CVG_1vJ$`xvtA<(;gH(-e7o!nemfB zenDc!nl^b~TxNV7q`I?RXv{4ceNTwTQfV&U=31J%8E@b=j*o^fD z-)(42LDJ&{_`l;uT`dBo%Z&4<7S@f)fbo|Z>X~kvgX6-1sS1e~P4j2AscXweF36jO(2#4#LSBD$lv#v?(~@>_vWUM{*OKp9;K9 zou^fTA=r*E+uY#fZijZ(=7Q!&JbAnJo^Q9D-H}78j+_KH2r7dk^$($?S#!9iaWe#h z3gI?b>8*|xw+00nVz=^k?n46Ae_8v#JbQkuM&jQ}u0tF)xW*JgJvxd=k_F&3G>%>4 z%?>W#Y~t=4$SG*r-j6#hW_O=SqIw59*Vvyf!g@z|k7=;9Jw;G)Afa9&!`#()8KSpB zmoTynhix^wPMWo2>>?`tpoNVpF*nSGD96I#52T!BB;gml zk_PKYQa_mF%U#?4v3Fs_FN<=0czzELxxQJdn@GrQVAaI!Uls=L_k3*hI?Cer%Ex9K z77VrjCOsEv2uG$90bqd6QvNLsLBc#WTmQ`e(B13P4Nf5)?| zf9Q(dk1Wgla;#i5*(h)7iy#7S?TA!?9L-1q{p+i*4Arp~6BO)8hl++4TWdUh3Y*7j zIhnCIk80_|cja|ObS{Rn^>XhQ&#LwEj^$YO6%IbDf<$J0_6r4@DVdmzNE zDTe?a?|S6-3K2pq0U9ljBE>c#^sCpZ(6eR2L%CtItYiLFzNNat*I#(4O7yTl6B{7# zY^(Fj#;O4&)lZl4t$Kd_R9Y5T9ds*14e42lx|!bWX?Kg-A(?u23#|}F(Q61yV}c_G zp>q{Q!+KIL*@+;y63>4I>R2&lit&f#Mcl*D180BziCoChg40SrNw;snNuu@HjiV*nW%?N@mJ@Y72QJ4e-2H7TeaR1ct{uNkk~-Z zkBVA}Eox%Qf5u@^xu+ah>Rp}OD!=&b=c)ni8(IeN0chmtXhK`_Hx^gYPf;?P|GZH` zHEQJ@rtJM=3g}@_VqMM58(C8q9i5GXUUZSL~&Ol_Witj2-Ba&$t&v<7M;}akDh?(|DZOeZts(CG>sq z<-BlLfQUU#bQ!reynOCEF1J*9{++U*=lUA{R{HB&xU10l=lSK^ z^Y4dOFNQVwto!<42t!8?m$rBTvbBu!HBt+>cmI_X@a~uXucoWlAYVQ(=-om>!Gb&* zGd#+>iNqka6h8@8CviWD%FJVQHhQj%-0R2b9mr<@Q|JN?Z!n z!*!EQck%S}^nb?QT;`H@(y~ra50jIILUKO(;gM(vA)tEk`3QJ-6{GY8-5|oP;RU;) zR-YY}mo2`27{?CozEBFFnvHLmCH42yf_oDL<7I@ZK6>E!K&iLgum8cFx{R+@h6&=& zkn3xfmb;mf(GXX4f^E-n<$%n<0j>T@m5tZdGM|%OTO;~Ce|dDwgb?b&ucJnq*!{_k z8M>Xv35?4ZH>N;_tiL*Wa4any-|Da?7f{>ly9083lw>yM0P#W~i9m|JyCh_&C6E}o z(aitjv;Mgwf9XBS^p)}F^-@0Y$M;wQ)Ev!}yN6w_ZAI=IjmAanlM?r}_1RPt8PpV55n!5!IBgW07z1CBvR87@vTi!=jE^>jT&Ie3(>X-L_m!(ns) zlgs_xXPbb^o=;lF$Dm+K;1oIq^?|KfM3G!?I<8$@Z&u%JaTR058gH%Hov!USYn#KR zs&~1G%f@ygoL2uq*Pwv(Ju-`AD}mHKlVPMYSM-@)2iD;~-RFVbyWBfT_9I;lM&n{! z71mG?ks?h~=fzX41^oVT`%3RpUmXF3YY}|^3E&3O-X{R>xAzdi?7i&&f{@ZTt@tDidH?t`5Lx*_4;sLo3hODVQ*XQ?G3ZN>g12k zNat^E6X>4gv+2rO-`DrKT|X>)Fq;JdSq#d57m=vaS}Y~rozdvJJ-vLNz{}!!%TjE@ z$hwDnTR(k7nDoQ>`xlWp4h-7jAf?WWFofWc!TJH_rLhR znt$vC#w6$&Z2zm#H+Y&!c-u{I?{1IfRQ%$^=GX?${WeGAE1Fs-{YZl7gz9sE0Re<8#ytwjWP!pN#IWCpX`xq>%fHq3AqHRE}h zlw-qnZZkU50|W)o-0&eS0&rODvql<_f|3@ZD+| zT8|j-X%zJFh`}1{FU90!i^w^l!8UN+%_bESL^HlB`>GV&lJj3kCxF{iL$Z)<7L1VM z4M6jYw;XUm+cbWK8Sw2VC1650M~X2*{D{i5rZ7z$c}~XyMy9?VWf1#wf8if&)iu* zZBjUIZ8hVB-bA`!*e6JKXog;u1b_7XsWZa=6;Vf^k12`i;%8Q79dx`kGj*Y-c{`~r zm^BADZ)NA|G>dvOv@$}b_{$EomZJD!z;C*F+ko9%r6Gt}l89-;-2GO_%3GP@7zGpl z)ehGTgDbvJN^qq`@b%BG3$gpt9Wpsj%#k7DGPTIkmQpR@#A4A)bPRw*O|anREstX1 zXs9MPZN#5#P*~i4rf7(%NUM4G>hKj(uo-n(a3}?86*YjTsbaH$6>8{m*94qcM>q^HGFRq8#Hso)6?80NHi>5TJYSDo}sYydOD6YLLCi!-A_gXi_{~d7Jt`XAoO=GP!3~UI?y$tXM?zje!gIfSo_^NrA zr{EA?2>=UmhgNG@_nvyQzc1`{pB(+!1SG-9Oa~a$z}gNPNF~871)#M*BoF)f!9nxd zIEVYS4R$8JE3y>$B}aA~L0e;VKfn3>!#5I{a z59Y~1Fq^vOASnC#tN7;Dy8_JC!o(~wdkZA>*T*>6Y}GtyG(X`&3K34i$@{ZeIPxa~ z9PP6s?Xxeb{2!4LFAmhenCU$91`2<~J1ZdfxH3+E_{#kNLuxk5Pk|^7?r?HS=6_*3-izQ7}m0Fu1xW5>Kmp*Rb5Ar{SUh?|fMyE}l z06kr(Jf_OqIzCBpL+f>@GVDm-A)IT}u6a5JN$+gB8Pk>}k%V(r^>zX~qmu6%li`FJ zvrIfKj|$Hbr5++(g_va?UWoT3u@+qCgdE(Z9?~LFcbjkMzB6CeF)Fkk125y+CHae6 z&xUIy3+qA7lPYr}blN8>C5fL9IKj90k=-8b`7#YXbshPfmdqL_(gx>YXpFim6k08oWz>0J$`_n1 zSKn?D^RZ8Tg~bQtjByK>-oq%l`gh-vWclge(NrF%z`IMba(xfCW=`G{2B7Zc@?>Q20z(MIf+cofT#=YN1qv@FrhWS5e~N)( z3&h>`8xTHCps)gv!<2s(0v;qerk|KLk^07F#nu8x zjW-x4{VRYY#y!J62IsZzZketcdwyqY!bTm(v@tY6#mNGV{4is0_`&oAK7Z$$nQ61J z*ieHh5;COLj0*AUn(59EQ1@IiHwy96cighV)s2ye{gem#rAf^ft7Y};t_&WLHJ+mB~OTG^*730tBc#J)R zqk`b0!<}Xzy~JIoDytwup-=C&l~1oqX2najMv_38^vbF^Op`pIT*Xu@z3QFXxMC_75Su-DAUJCN-s z6uc1zsGU9gLwYDvW2G#tGK-9?ffa5dD!O!47d|d>EK+3Wv1;F>fTr+>I6RkX- z3NLE>{Wx@|t4|JV#1m!?k%ygkrRb>lL5k+aH1L;i-^?RTA%PrpJQON#y_!B*{`UX| zmPjbUifDV^*}k?FXy*P*f=-o3O!TDuFB4G`mwuGEGK9>!!eL&_xFDG!?imn#!r!%% zS(}RTvI@v$nh}@^DhlFDZSwr3GT=Xw3#1Cq3!?8-`f07N!L({Ty4lr3=;a!T!kMV? zdszyfTKWf!b^AJ7UXoN=+tiQmTcz5+{TlLRtXE73K+i`3K3@vHnT+Wkdn%s@;mC(yR5*}-bQ00fV=mRVD z!9l!m<9$T?6wVL8XHH>=mEMmz$=``Xdre>Her$Hh?;r)K^J5s>0x{vpXbR_JG6ciQ zK^tZv)ZlWO6OVh^0d4$+xJ~uzY$m}kK3#o;UY?N%Q$NGCyfs+QhCH=9jN4UmC`%j% zogOirI>^XEP!P{0plKU>bEF zn5WQt7a(y?s6S8Q=B*r1Qi90V>?AE$&14 zFW|#}B$T!2^a+x*O#c0um9Yy>SUinogvoGw$sMpx!Xt&W$AGpr1FRI&bDAs;^^=#A zZ?4{hjMz_%B-PCd4|SKu!Tg-weM-M`Nf21?vDNJ;KbXDEQ$h z@zs){wA_F|j{uD(Xj=@5hlaNZ6uk0(E#6(}&>UnNWOd|@&Ogm0f{nB(W&E5Eet z@1s!xbr$-XIc5hUI}BhkozrAO;;ASE844CU?CBt{ zKNAccwZ$~Zkd*S{89n`;e}qQ~G^7t}f2h_UqmV<2nbq{UkwYdKrf1FUxV*rR7_S$a zm@D)8*jGfAICcyz!;D621n^GrlC4T8(%Khi9V8h>1J=n*BU7v^oG$)RQq*mt+{W@C zwJ=;fxY|rqmxuHYo?BCneJ<^oRI@j-uX&}Tcdo*q5g-MQKs0K!ZqMzFJu(k!Vtj|fe!QZv_TCfaf=w7?V8;N0Sri@uT%W5)LIzl{ zV1nl25eLHl&35n+lO(mU6jspS(b;u(Ee*}9WGK2!EQT`bHSX6reBNp-`n>HiE8Ma2 zPK5a0{0r#ePuHUSZ_(Ztz$tJN^k;-ayz}|7m&BSVRT6fyM|>w7?D(fB?^e~Ch6^+* z0-^3Kb1g+uMLqQTV!fI=08vM&wQvZRbjS6#YQs@5=Yz9Z7RaZYDWSQB73SwxJtHHNdu1K6<{UU42V|DJh@Wj4g?=?&OTMa27t zsc&2uK3XML9fAG^xhi4mVvJLMXKigwbUOcqG8lpq`LR-w~a?=#AZR3 zd*0#r7_yjUV%@MXvk){)|6);M!b6kbg4*x(?Hesn`?*2dPq_QZp(Iq_#8;@Pdpr>l zNzW&J7eYE1k(x(^0yo&Nvn8h~8(QdAh<>u~J^T|%_+T?R)xKi~xwJ(CVbx!eS0x=_ zoRs3qGp-sWw@7rcbPIF%z){!x%#3Unlr`I|kTHBF&`G*SRi|7nBz`VwgI(9rA#wZ4 zS-L#uk~c&Mb8bjJv0GpOaARlHB&n7PSP{U_X%dd+0h*ivfSy^$a`~-AOOo6gg;pBS z;9vq3a-KO{iDYAALhe;RJOW}d*HT(oMDH-Q7s`42;0WE}9)0WgNY0z^ufcIBs&*bp zaq{VY=FN{vOVIO!ALg*z0VErv06P{gumH8Ppz-YRQ-}y|eFdM`hoBP(#%I2p!MR$PHyDAg;Sv5#cf_k&wzRd1Q~= z?EHzya}U5@*cziZXhyuaEW12~a63$%zvpnK{n9T#CP?mm1@`I7FYyB85lBrxfMV(4 z1sc5iaztC0jFrvQ=-RJw$D_BoUdY1-%&VRb`F?j}kE;$m!o_j@M>7vHjU~)L(tR?I zo^dDt-;iftgOeCZSX2B;nzCDDIM7~tFuK#n>%+i>_y!?Og6K#b`G1WIR*czt!A#Gk zVzScD(2ru^pBAhzSVqsRRAmkQ=OsC49H9vxm3fz+fmNV1@=|-O^6Gl<7b^6TwO~|k zeL5cL^N}S@VA?~?2cR>SWSJe+CwJ%+9lhV;*X=QbpEX)eFe}Fcg}CV4Pl~k4To|L* zNN8k+{_(m$p@~1m;m#ZQbCvZ*N-w>*6}jcY=t=y9&K|V6^2j{QuUpC;(U%1Z!Rw3$#oyAJfc0w$P^<%QkeDV&64x4%LcDt zhEKsMGe5OQ9v09G7P-#ScQ&3K!4nh%?K>3EzH=abwXy>}ko%GaWXo(Zb)lU4CFloG>~nHD@&9mTxzeKs@kZ}j!O z$li{?UWMki!c{6zUHHj6vzl7=$aBmJ%(r6+FuZ}%sLLBt$|dIB+T|&9c7qFD-(Z$}59%DiQug zO^R^*`eZTd3KdvLIVTZ>py&@6Wyb6TG3I`@#Rmv?6K3hBnZkccI*~q}Y}P4RZbD^Jhh} zl%f)MBcU#N=KnZTMfUJ^By)T|92m~b(rSM*Aa~-!){-B%BpgQgmmtS|n+durExq)o zP)q3HplkVAh~4$g01al>lq5sU{meHomAum?aJb_nQLze_r}h_nKFOKS3mUe-V2Ogb z<9vp7W%D!Bbnf?FFZp!sL>W{YN1W6gW))4Ie-aDIe>Cnyg^62?f=|k*(p~xSZ#7}1ZV55T|3)*Yo$_Yt$;R70 z$CL`TTZ3UvQ45@L6wE7x%lh1+cGB{O-MMVL76blSncIQ9sLq4#H&P^%&}HJ^;gDt2 z+5N~P@MvSi&NZDaqt%HufpH{!E6_=+=!2DZr3dG#$4?3yLUXwab3t`mk(05@cm7a= zAHdr1kFk1fZD7tXt{3gEn=AivFnyIdONmjj6J2+GUxkbbz;jX z^`H+|2kJL#eGijHc5xc;)N{tJdt9Z>aW85bI#Q|LT7@E(;^81%BXK9TnABzVMPe*f z@IA=IeX}QTTPWmsgG@T&{|G6PIX66_s$PcJOvv(LcqBnvTtvC@2ztCX=<&kC6IsM7 z4=t>Jrxt5)6*3p?ka4nQ8y53}XlyN^G9gR5JckX|~=wmpsS(CITO!TJ^Bm+`f>{EbTolyp3PQ z;rQ_Vq^N-wz~tlRcL)(VWGnY0qt( z!AO=q ztxII&g42VqmTT&gYJVVu*4^X1s8Yy(X%FPDN&lDRZ?;7oG*-m5y|>By`?og`$KO#2gAC>9ou4xw-iCb><1u&pbbeMxERM#we zBQ=c&V2a?YyLUdv($1)V~c2Zxo<#BKe}dlMuP2woT#@!W&*N5e-Sjeou%q3}I7_ zj_KqxqeD3^O1I-P5diKH6eY^elJyR3my;urFdH#Vy9}$ySdhv2)2G4YsNi>M>6UV? zuf!U{{NpZC_~CP1pH>%tCh0(f{eA1I^fgKS(ug5Z1HfC;wZ~CB7=%fFWyHcmQeZH^ z!E-52CJ-$|&s7;}=0lYB=D?IOp;^l1$CL@;!Cg}PxQXWJ5;3zyvm4+M7m5$lnsM-u zm2`RI+ahHUNno2FWEtT310M$Cvn9A+NOXDiDAORHBHTB7$0>Kitr#JnWOxAB5*XhL zjlGw%_M&^cf`3V4^a?C)|IaPed{=b_4b3PL_(5-*|+tLr_H6+iM zeu32lA%k)?z#IxP2g=EnJ+bXZ?&IGO=!ZN}M&aRUGi|8HTzX)BqtZ^1agd>~q%D`a zQ^eo6n26O!GQq0nM+P}${^wfNhRM9*{sB3GG<Vi#+_e(Ud?fHbCgZC-N+&~JT(MFnL`V1XuO2(qT7o+7d~O>}9RWQBuE^N@sxMxCEM!wl%g8thdU9@CzZ(FgE(T54{UL6!_q@r08YyIG?tF>u2S)=~KmS zyAP0GTq99QGGpjjBIihZV2RnWa@Ht|wf{g8+i&pKY#DkR_nVw0;FTALlMy=Mc$ZOmyL|eR{D+>e z5($~k{22vp-7-s#-`5P`i_pNGG?=^#r;$%m+h z%xB`Or5=M?^X-@s2Q=GKEc=n(XV8u`U3S9vU>CZ;lp&s0c*nH_9m*KgS=9{V3NA0% z`H<@I5fp>S!3T~neb3CN))YNp9x8nZ_=hQNYyi~|>IY2}S%fxSH`Yw&1zYmP@<|0? zR3UT)Bb{vKfiQ7`^+CHpHM%bfYN$9Z@>Ft!-@mk3f@as4IN-7k>`5F>RJQmd><5s( zpaCgmXb~Tk&SjMvzXwDYe`hrqa_hLweNNeT;z9 zUuY1v_pI=trw=xXg!Z6B9Fw54(ni?Uba~6hq2O(75_i+3a@bd;VW?nVY;&}nk2!BLNORABz!Kj?k>^r zUYofaot~E4&Ch-KL4n#J#lcyC@kb?mxMfwHV#~ak9b&1Kax_l7h7$O&gej7lSP?T~ zDV&-hG~lF{fdhul6v2NE(~pQkAzZusNtTbvhER`}xZy_Ns_e;A>84wkEL=1`%o+VC z#8f8WX=1tRc+Ur-XT$|KNVdrkXMZYa9!i45fGh7D&z3owLF-s!3I_9>={sjdpSh$l z5B5m;nFe=wcr74*hzTRj#KfAH-6KR5~u+ zJopfqM*5y)^p6hKJ`0LDH%@tZNIvZ$+HnDkk4mh8;!3W)PRm3asN?pZph`oo&~7~# z=<#!rHmFYq;V^>xVs(RoE|a|vxK_r?RhAdj)X8&f8`{ISK$H=wU>%>US1eYu*W3BE z6>k`<+*$B2$N+C|tNrK$vfl*(fIE39n%GHx+wmrosR% zThJE~-}TW^HM?TS5*HgMzMoUEl7^q^IjoCfLy5BbD(EP7y$S)nRen^_-Da3Oa~xvg zq##9RAVm1t6Wn!Uv6m$u3lpV1#zGV^b-!_WESDk z0C;_>A?j(la;mSkZbGI|?=elsnz##BPic2-$ns#XtEg_u$4NyMAY*Oine_=7=+!zmdC4){7$iO~rPzbXC>iqcb(Seq>Q2 zG1GxkL~ab*K4>+8slRFVXRI-a0~_0lco5oB0N1}6rGolbLZjxd^Suucb2(n@^0SvT zyMZ1lQck_&gNfe>dWNU$P2SKEmNib>Otv6Iq$lSmTWpbu!zerxz&*TV4h-U$r4$eR zIfN_-;ACNt8qi6VJ)73#gJ~IcMU%7m@R{S}_ zB)KWN1^|?Dq3fnBZGwqb+w(wYkEk*M>pJ4w6B3*;v@de-(TDJIH857-?bGUUM-i=d>2){z9R+-ThXj0%2%$ELQd$C^)l3G$4hY3rv`Xo`7C-g7d zvq*tO(a(4+PtkAyfLC=RW!>TJtZi-nqz5Z<;AN}mf!*&)4yJ34d+?S&8m*Mt*-!~Bu>9O`xrHDE@5X^lM)0ynH4HqRGXRU}^!$G0H; zk}0+BXBiud3qIK>u~i01w8a9IgjicrBd_!m>={KRkjBMPdC{Ls+YPrgUAJVB$=e^S zu9!mi-~Qp)*y9P%vv>idm|4uF(e3CFa!J^HR&-&K=G>Mb#80A-{#SlQuyYc1{OQbE`55$l3V_U63P~Lr%yV14aa!(nChFoiK6DEgO`c(MdCBdZFJ;0S2P&( zaDx8UEcJlA?_JXE%*1|j^ew<#Sv{Fe(n%Qg>+UBnM;-B4;`}EM+sZY~w~E$^y+yaEzsA&-Wd@K3GLrtf^=4Mqg8FgET|};K1_>h%F5u}I ziHI~1-)t)W)PF5sczl7;Yu<~QCpm*3rg=(byzlb@o>odm{QuZ`%c!cpaC;ay^r5@E zySqD;mhNs4kdp5121#k@Zjf%I8&L!V=?4ApqrZFa`{DiO7&sQ2wfA~r&iTw0AVrsT zXUl<2V}t�!9?OS@wB0umbZyK7Ht3pst+h3$f%H^>jGK3aB;<;x zfGQk5N}r|FAR2@s7b>s)nq!zTATEG^zWDsz4Qvwrzif|69ky^p{ByYt+|3t?M4$WN zeRkjxkQDnNP?ndMJIi1G{`@^vJ}Qq&3^|k zs_xG>9~{ynR0v(so?#|^p~v+*&JiG0hE@xc@&t8EKh+13uqS60k8-#qy>6nRl1D4f40?h|f z6k-8I;lIbOay!*%&v37}NmgYa(f%M<1twD3Ibnf$-yrEE{#Mb$%M&>k<=`(CWLRi1 z#%w~o3$mZ}k90Gwxy1UZD9fZf3jb~&2JsE6W{_V1t2^-6hQMQ6bWrr_6MI!B*Pf!u z(cz{akS{%r_?D`G)TTQU$jd4%vU{Jk%#mc9zL5$_aU{dhN|G~4~Ll(BwI8|P!s zec_`VTYhZF&RoFf|rVM&A;JvsAvge-*Ok^_m2N}$RTJg?}v@mnGZlISNZD+a2oqg|l7nwK11+n>y$ zyV__N#J9I2^X6wsIbl@~{m8?&QW#zRtFW~st92R8EtvHN zZ$M@=*B-q8$3#4g6IYR?sfz1V%&-X;4D=lPpkNbFB`XN7O~D!)si?Ie`yVUkQ88Og zhx}{5+2#sT44m-7tT}*in>F2d3H0 zC2QMvH$)^=0lb~}4vOxVncDJ#^WEfHk=a!pCItbgv>yJ+sH zHLa>0A>6fbast?X9rhs2582L`kF5id*dc4q#0s(N2wjy*h^A@;4F=)Cd5_+n?J@s# zD5y=SmrV%N;3U5pzC;u}RNt_#)vl?Qriec&@-A*a!Do8T-4E6e*YG7n0|ASaK((cP zzHs-vJveEjNoj~zp5gT8ck2;CXo@?kP$-WP4_yQZ#uVz4c}o*ov}EeT-_p=o7XA=9 zj5-Kiogyi)PcTGVckw-4Uw>=PkSu%)u0*o@SCPR)R=x+WVB_tox0}CvDLUaSSLezz7F<$t= z;v(Zb&o$KJT7txFXN)T(1hi^r+Oa5m=*JqRkh6mc=^ZFGvFa_3G?+&|zTn^y=zq;9 zi6mTt`%*;LQnS&bJd7&}$OaU{2z^SU-jEqiQN~|P-DS)9J1!0) zOCW&8^N#7yh<9usqzA7yJje7TRsjy%^r#bbbaH`( zE9cmTl9!_|vNZg&-~MbU)Xxo?2uaisvOGkV69;w|EK2r*W@(Bkb*GLl8h-OEoTmiAGg&P%{t_R?OVd+*_1AsR05CTg%Wp2qw331VyXc_@_+j zPG-6!Po_$`b;j@15eJVQ9tD;Fwky{$U+$;hAcPnH%Pt=lx<1O`MQ+i>V(FG`y!A3| zn=b1JTBrfPDbX3C$eWD?SGklJbjM5W?hbO}zlFB(o#1f|(WreY+`YIBg0?*xd@_26 z4HwV^b7~9?!dNFt9unQ)`M-2ztw%tI+E*>Y?48GIig4F8ej$wlCL)k;k4bAhO<8_3 z2SF6Ox*LY!?Z{Pp-D6NivXx59nnx#>D^DGRkku%b;c7h#L9F#b_ZF-D7Fj)^s1W1^ z2#`W+rj^8)iZ8mtg$spzeH(g1S`9zSEBckHXt~<0E2%uyfs$@Z!&+u+z6$3IqEIS7 z?5|q&H1^f*Zu$yhN*3l)2Bk&hn&qPnwrGWDcky;1XZ?#7R}p3c(rtMJZHa_r}1rZ7(shOf~{6`$jguqhghUXChmyw zNy4rXo=_FXK{N*41A*K-T3w48K+q z8~Ewyo7(PthY5&l!pNgVMmx{;_-qaPo)f00B-u|rcfj8B@`&}T%B#j;%C&wXHvwYA zrk+((jWQ(sz+&u=e9PIS4znKj2CYqywj6?(n>o5Fx5ip@V?_`nsPjUh-QZd^Rhwm( zC15lb%)E1$qac}$g&)i0r1TPOuO0XYBOUFc8`nX2Ezf1N~u|LGC;)sl{&-nV4WtiKF&cg3*_$JFI4=5j{=M>Ob|V&(!oNWj_*j02#Q`Ph zr#OzqR88}+hWMq=?$E=tX0(Ko8tXWxwWx!PJp@IaC6mr8C&I{Iv~d)d-RSC?2DF?p zl)2MPuw4=}8M!vrpu#zYbSRh_gO_$&wwRMp^w&QfH~mK=m^mihWs`%K)&|H7A|3Hm zD6ZREZO9HZLYAnv^UR4L^n}yd{w{_4AnXgJ@a0FdBin_N>NIS)1F(h zTTO->te`e-$WCVin*m^JCpO6z#G7d(g8P0|!>hq18Zq5jth-B4jU1f%moQ_QA1Zo3 z=z*NLb<@ou92Nj#uMgM1`J{*c)tl4dB#J~}aox0sw(-{8{;VnU17aCY6?GYqg#MD_ z0+STt@109Gd02?GLVked9rFUyoOa8~^r^tM8BHUx=*NH}SX;LJJ-U97g(>oJWEgC@% zsU8YJlD9iGcC2i6F^|I8HnixkxbpwXc=Qn!M$qySI|oVd6t|5%pwRN=ixE|WM@(r? z@DwvNstUaLfnWjwZubc&#oyc4B-oh+Cu^f-A)AJUa#oQyW+o`HT%NL`eIjapd~*L% z5EDy9>&}jWI_Q}gV>xnyL&!jEu^p2X9r#g%3|CB|FCS$vZALF$95Ys5xwiRDm#56~ z%vgfNrTH`AIM5x9}9%e~-Ny1IF8vsl0exTZ`BNuqxVh1swtzsu`((gGX34iyTI#lB}7$ zh2*fKGOBP-aJ5ZNYmeE~>otTV9vJ36mIacXTRG=M+G+zT>cyXPaaRGCCTE|KY+G7M zg>ZpF{TqN(3JDhZ{$}Z#MAZB$B);cWfvJJjK{kr4g+fVk4e%d6aA`k2ocMyIgU?5POvMO;#nxB=YWbOfjmsRg|iDO2f3_BevStKp2Ps)DJvc* z8+(ySCZ)2bf^{j;PjtaFxgU1wf-PZ~S`?Hv9Ld|6ebu(+QbbPC+&?CF^8!o-e6uIi zHYvI2+=i&!-ppAkJ*?`09zSUa5F%49&+GzIfewin5cU9SL8A5ChL-X0eN+n&PB`4^ zgE9^E)Jn;X_a%vfW5ugp5xHnO*&6=hIr@C4apByd13grPEIj zfg+<{EGF!wxlSdc0k)is8~hn%18Ma3(z`IB(_!` zd&&Khq4k-4(_(?(2x-_Ok_V=nK9rv9>yma2!fL~m6S6Cm_gU^KPXF&hbo&qZQ*)zB3!ENt_1z}v#%qovz3<2 zh@%u4zbrpMmAMv*9%PAE5$14{1{4EL3{Q7?R~LjkLWZHi790q<$_AMhQ|HY zLJUgC8Kaak(6VC#ExR@K=7PI<57mIP{8%jY&(Gw*Jwmo{`h21$rlB;T!gY(P4tDcU z|E+X0{jJw+W5^ix9-{7fnxC`t`OwEW8u=c*kA&9D$iiNe%rL4aI;OV?^Nn8bRiD}= zWCH~?Fe$>5fh_YGmgDz90g02a4vVK?-@!^-vY~0EJ*dQp8Gme&qoOY{d-buv3GHy4 zk*+?02bsfJ&_7Fkj?(-%u32aNIa+shJ<>&cY%5E`0AIV$$rT4OU;$|vEkZ)sPc1ax zxOm#Qjj3N+EaG4a$VUSdkP7T$4AJ6FW|@+p(k*zrG^BJA(sxORS!RNqPy!^-n@=q{ zXj2skTbdYeC8Ue;F#LR?Tqkai)tZfg0SH3KfD63mhhB0uWC;&F;;Q#YWJe5y71SJl&6hXEi!Jp{{=Vh!m<<(&viLq&|AYL1`7(q9Gmm3>*yfkLpJpJn zW~#e=HCG1e^_MrFvjMWjwbU_R%FkaTd3$@`^ioQ}?k60=?wp;O*A*_~spD%6bs(Y} zLLxewZb5egD_Wk>=4Lz@vsun`;!vKf@yy-tB}A@feu*w$5*$8{wS1Osb$WSS_MfuQc1-LT=9ZJuFnm&R2255c8*@f*!|llxmVA>#qdQH~R2)j7Yq26G{k2WdK~{ z(ep%TUkCRKU$)>e_PTjc1^5slb5YiW-yT0Kplio7AmJqPh;dzGY@>6bsUgzDfojQ$ zSDx*^G50h-j7H2evI}jGM(e#5^S)kxIIJz3FGjs zs6dEgs&N+3VhEk*q%mT>B4ty>{C#+;4kC;ou8>bEYuNKTzAFX+M#Ld@kbacoY+itC zY{0`3-u7r)jLYBfoGDI&QHl~(-HXof?fv~NpjNV-XCuuu>6~Lbonc$@xL;Lx-)&SV z6G<0-Q}I3@5d#Ct5c&jR4Q@}Um#=^%`n=1I6u!fHv(7sD3_>rDIN&@P$_Zt*uLH0!a;`5EC4R2Bw+BfWC z1se2poEiDv{lyUlV{Tl}VJtuoZE+ghtlmKO;ZL8gX@!)R^ZYrsjw02$%12^Hcs=#`+j48wLoe!vvPgz z_H7iZFG1YG!|)N#?<{=oU!vFz(u_%}{35p-ur>2dt6_`!6`(VvCEk2zC(cK(fzYwbL{yY)WYmvxg&R!W zNfai%ou}jxt-ko~I$}9lE#D?KmZyGzMQb!GuNj5Lj&L<`7iOMP&pr*0sq-2?oMABc zBu`CnI}x)OQN+n_ouYhyb_@DtG#WmTQmw=aD}U zXQ}alT9kpU|9Urk#ok-VPDJ=5M~$hQ{D4ZT^pQY>*y8dk>QmUldc=gmp5JvZVyhS) zxBl)2p8o7^C6KAdCQ8kHPLOo?Xf7=2y2oB;STJTZTb9GS4B}{Fmf{I$ame?oIAk!SlySD-L`<@97v*#=So+8 z0=+lp#mAR%BFGYH>t`iN@UuBt1}*v4*t^YOm@_0(^wbf;riB3N8F@HmGa=F5t*PRU zONHFRcBqX98Fy%0`g&jZkO-3E>UIF<%674>S5JD35Q>MXgibx^D_i-Tb&f_d1J`bp z!Pfn>mn!e>XLW-Ft+T#mKBA+zLM;nX4@02KWfq0Bkhm@A8VmFUFAHXZOSt<$=wW#M zrL}%t)|{rO^OTR(zbcHbt5bCnScO{Ms+9rdjMJ{gD)S1BexL}sg#6Nl7B3~$s17ji z7rHj9h|~X^T-p{55&!%r9}aKg$Em$~Fmp^<(0E`Y2{m)GsEvY$ABnbJvu}3jb0b1kNpAx8D_!2wsa?pqJk;YJX>Ip?tQQT4yY*_dn7;wX z!eP4cXqscdUOJsL)^*Rp3Z{;YuA(>52hhn|!(em&gDx{rD8cmNg}KwWhEzW=T4*}g zD0E%D*y%XnJ>+egfK{`BRmIx$`r#dxVUpT$J|-=5u+7QX1$#4e^SchDD{1Y_hA>Q+ zvgs;edA@~)IillwMf&MA{ghWVR5%ykvz0cN+Fl}PwKiBXOTQiM?@TBu1bjxHTA)K8 zIaxJt@(*r6lI5FLMA%i{P4+h(PYQO!G;j1J%bnz_`1PaK%Cq@HLOuq7=l&HxBC#pR z%;^|olV{SAXS`Ze_hM;TkirO&DV+E2F|e;h(lK~F3z@3yfS_E4NtBFzHs>E>y9lba zJM+jQ*WmD?^toNMni3J~QuZGM*m83kzBjb;@Rrb*JHotH#$;?Bl}Gl7RH?i&{#-me zJlAlwJmNVX>LU@JIEO9t3P^8&kaax~!IT008xMEuJIUn~)Dg@9!*?CNgSb5Sq(7B2 zKlK)cn?k>n=Z6Q69Wipb7NFqSS|kD&_a%r>9nC|{?{VdcacbwC%D@{FY+>@hiu{m_ zynSaYpK5xN-;;DMxO^~%A7r*E47R z5&;K(-fXhxEz>$Yc!cqT16L9IDzirYq7SLf*_b==_QK+%D{q(RT7BJ!$0D%FhT-Lh zh3S2Co=n{~2*+gDt->@X#OKOFsd2S|A@B-FZ;(0b5xBc{XOE_lf!ZTwaNMkxl1%zY zHauB0TjpL`+cG{NC`g7@gjakYOtN{tN|!AIBxETJv6&hcgXF`+M54lzC^7@H=F0N) zuZvJ~M?EstWp1zTcsKyuJQ)K3TVG3^?-Q?ojxIBfuMuOL5}y-LlY!M^woRm%{43Qg zDnjL7Y;Gt*6%-vnFiRYt(g?rx=0xrh&1LWxm(!!XqBplurwBz#zWoY14LjNUpUG7t z)Ucx77p%Rw^pY{{H}bd?lk#3$$BZjgqAC7mI7|EZ2;zk_Mp5QI!c=@PCDd%+mPRGsg19yVcXZCJG4Mp@sWEV>unGG|l7)B>aUVh8HX1h~ z1@O)p@F*vjgHNg84w68qwcKU@S=d>m*UPc-v{ER`8HoloKbD0uaX!q)t&iT55{|DK zVNjtFU@0|!%mp`g`NVX$&V5;+`=>Rt%d#@XC(KDj+@ACE^J)ZUcC6sWNB4QsD9B1iaPFvndTz8=GHG!H;J}6Tb zRnJ;+{Z5l{1&BEr(+qt~_Zl5omZh?x80xF&%ZD3@5((_Jvb`jJ<;7^`4;7DcomS;k zyEajrK+yTi%$Y$oXChkYg$-RQx7~H7jev3Q&RHBXkkgV7ThW$T;10*xRe{5U4ji6T z^vyY6P!~o@?(>Q8taw#!uzxd+p6E9X{yV{@H$#(y5hDSz`M)Y$rR`@gArua_KgKqU zR1%E3;Jfzs6sCR&jE@}U%brgXk&)dOUEi~OZ3^Y{Aj%{NK*T~$Blb=>4Gno+#y65= zIsU6W4W+4et+UW~u?Q8N1V%$aKs>m$oserqYPxtF$IWj=q#)fH0~{p|RH2C^dQ>4= zMAaBmY(k!{7NOLS%H_`gO*tiK?MBcC%88<+6YAL~O`|yugHvEDKJlfRn>J~?%Az1| z=*{w(5?2{E?Y@uZzkM$K5T$N!dUN3we0KQOI$nEbxha|sj(`!n#Mbdt?;cy|S>``# zdn842QCsP-!8ngaY?>WqgRhTd6yYuQ>J7DX4tVx?>3dCQ5#01!FJFUD-7RkW{;UE^ zP~o@Z=7me2D)iKPFVgfeWAEypA0^wfkYkdIb~z3=#AGyj8grD$~*S98z|uhU>EEb z6iep=gMvH}uo1;QRRLHiI_O5rsm&mf+b%8?jSd2UU_(aY=v>7_{iIN|df2CcdO$6oA z^ERMw!eX&F==ynLADHE{gM@G%HRPXY=~-;79IYV(Y*AAbxnl}k^)kkljm15e_e^{a zTXPLM1OKEOLUSRG{wReB?EG&xO5ik*&L@^?Lrn3cb>Fi8NlIH#%%m0*_sIJXv=OML z052=HDt*&wCac!ZnXZeZ#SErrnvsY;Sg?3E42Gm(@?_Gj4R~{MObf3V)b0|?=s95l zvE(tA*EC$q$g*h!W9MuoZ~W5p;1aY+rEq|n3ON6G@m3)O3bcLl1mKFMhsccF^y{|M zdRyquP9qF7fmS*2;6ry-;W3T=Z8;Q%1@-7ZPad}VWYZyEjU#Ms^d=M`qI|fJw^T}) zOew+gW}uc|GVlRRyKArDAywM|P_?c0vk-^-{BFtlPCM~~d;%WsBx&gSD2=9DhCz#) zxV`#|o~QhlM!go4fkBuZ0Zm955O4X^;$GP(krc?&Xv;?jw+&dQ-2CDrF#ZuPv9nqq zj9E1bnTq?5cd}$$xwvphx;5h*T)?L7AtKHi!{6mUq+|?cl31RlUrj4y0|8u%vtHno zOUQho-XkIjhc*2)=q~4dl2c?ziyIkEktTu=3*EFcK(o33@O^~T%nFESUCqpt{ZQCN zy@R`}k_A}22^a^NJqxHEf?lTVs6US*@MNtG( zEBcg9Zl1wom_zg`;ZR)^7wIiP&lVO2Q9@A4$b&Lh97e_bi4EpZcqNorJtDs2R>~W# z9aFBHrrTtSso%tRSC0?3uuq`2qW>E_CPGxW2+oiqB@@T+^?Qb*%<45gh7Y;4O~(gD zzU|IeF(v*#Oh~y)O~8=`Kg_$O?tZg*5Nq{<%-)?wD;`PMm5-a`&(ko6S}s*kTnB)v z4!}hf%vGgM49Ty;lb*8IqP0oIfJ@zUI!Ie;(ob#eAu@1?jxx_FX$7{vOZh@XPW5AF#aSD1xFFcuJmy_!fi57pe7QVPv?XQNvVt({es)t<*_t@*(-GMX}v#8zR zYn{Vp7#CaT?SLOo0xDj1;oiG_>hbYsX;qMqCUL9>+fsWBNcACFt-_kr zp`0B!@a)1#sRs|tr5@wZ)>dVbR8;;(dJf2dmUun= zKf|OzVP#S?O83%kg-=v8I$?>Aw+sH2>>-gxs#2?9(f=(QnBW0q{gcS9p;s#JWf1b# z7=<_zxKMa}-}>t$A2A5kPYCW+(DAP$*HcA4OR741+|POmLADyG4V&pnQ8F==d=DuS zwAfjvW?kwOQzb=*dGao-xe(>AsXz-EDvdC(aW~l6rN_WtOCl`jYvfB6iEauZTF-e2 zxlT;ne>Euu(aYZ3)DgII^pf7%E|>i;6s*JxZSnG0`6-X%WT?l<*DH`E1r-HT7EOMW z-!EngmlpuA>PvHT^*sL)!*&}2Mqr3E- z888L-r?n?x;pPbs>8TA(o~amo-nUOdO;NXWtmVhZ^z8Sc#-7IHU48RUO&Nk7;v?bJ z$q6;8oL!Z$8VseTq|Q6O?IV*3cz4J;nAMC?{^yqSZ(dq<6D^KpeZ8N3Panl8$}s1H zstpaPZSgaUuF&)hm1pDFz>Bhf;A|j%f!|KqQ)1uK-cFxC`HRP&R)Nr1>gFIeZJ9nXO4a8)W@mj_iKK)iclejFuU6Ei z-d|!6?`QjdZ}s_WpokQ2t(d;d(0;l>@@6W+#i~Nlbn?@x_9gh_l=-&T^)#)WkCwQ~ zK4zCD#Z;g6cxX1Z{dO|&{)YYcMHAJ=6K}$OM(^WAok?$B{7W0V$?q#;zymeV|L%#< z@Avb5+s<|Xp7Hp3(Gc6qms5}!-`#^-Mxw~h?{u4hCx7|z@|e_@o6(&=N4arNA9L4F zvGCL$N5%h8+Vu9v-(CFq{rzX(Go9)8GjdE4X#VBr)|J;m< zhpqj26c5bP`7=yP8Te4$r+l^>`1tt8|Eiq)kH?dvarJ}Mz2@ri9eL!3ne9(!eZQ{) z1Fn7q8enbQne?KT{|XFzIDJ`Z>-*tSvb#q!ggx&)Grxxxq zVq?%2beLVD982tg>eBv`*hqiVeq6-9mq#dTWNN-_xQDO4oph7&%%t!4?WkTqV@K^M zB15T9kP&R~!Tf$5G0c#%Rd%2WGzdgad6muvaGpQ2*sYM@)Ey3@8Hx*Genm4B6>=p! z^Z@rQAuy~bgse}`EUPlu##sj3e)U0qGmVDRJRr1_5b{dlz{FTa(uU+Jow2A8-DJ}X z2ev5bVGYb6@clgpe{ERE7fGw$Q#c%0TGp$)Of^?`eRjASZXB5ow9u$Q9Md^bl#j*) z9)p}F6EMg^vs)`)Rp-19dhuLB(EKEM++v0Sc?C(at}&` zSc(=yPeBYW=bx=0CT5n;mk4h%jp4eZ70I?Y>-rvUC&_(2Ns@nRvK{J~Yd2NZJtY$$ zDB7RBDmTPrT@lOImEL!F2+f!k8GVO;bA_5+@Vv@?_;B2NVDt24B@p-m&jRmW_)UI4 zcAfq9d89gPnEdmb|I%gn@-UqK&#$MtKOFMmL)v?{PY+|Wdl65s|EwgM1Q^0<^uAow ziEqNt^gYzIiA#@&_XdiOyZqko+F>V06IUXCUi&X%9(a@Z z=d`WwVOU(G81t;C{@lff=7enZLvPfJM!=;P+@^nHgynpN5wzfw#f&>%HgARw3Cw%L z(;)v@WxS2IvTpYBzmCg5r1KYtuhgRwHhh(p?c&~sBP?6%MyJrvc3bC(%u;%1poXFP z>TV6$kEI~L;?#mJ1_bANd_l1}wtJflJ(L#YFR?2L{7hb416Lje%&ONxRnT5nT~6Ad z>uVZpnTQLehXTRF9vNfgw=2^oXJCaVE%T=T$WNG@d^QoY}D;9OszmQ{) zxo!-B`WEN!bG2@>L>p2zF!Nknk&oAq+4my(0Q{~hl^`aB;VB#}UmUi0f(GKYh zBZf}5faw`kD#6|PQ1=^Kmnfo*w{HFV`_N+CQ%j5Q+65WzOpxpt4qKs-s`(#AL%%rJ z6?!hX^453}_$V!)(kbnB1Dj^WQ>n<1&z1z`}^n+mu!~d=o zN^{+y=k@aSZ@zDBfhJ(9-?uA$kC{L`)0AGkrDISD(}$(BJ0o)I(t>BO0TEGGwN0iY zyrE!vUxi!Tx>ARQvE;V%>=TggOBnr!DomTg8A*L|$A{*DuhDHN87rG@ zyQjF#-ZEm8()V)T2p{8uxbF~ml|Cd(SigIz=$;+(qwi>*j$(O7*bx|(c*=D-Z|*B# zvrH@cszChoms^Nq`0q~;Gs`d|VtBwa8Nua27uh%9 zCgokb`6cbg_pIUyD#Vxvu)(EwVQnQjXDXUb)+dnPnd$rb9OC?ZEII&5c9nOq|C{Fi z`-!RJen(pqmc^AkK0z1cPlW4rCNMPe#l=4B@hY(N8NciM=r4Z4wiEc}RTTEwP@}X% z{oipO@dK70LIiR4PO-o2l8C5Ua(l2@i%dSq7l#C;dhst=vxbJBmVEv#wL4CXEBma> zgo@Ua9A;@CG}#)LkN&!Isy9X!{&%y{@?iVsWNx>6UY7tLsLx|!t)nO`f!*&;S(_B?axOeWU#ES6HD_+(z$cB19NX;$eS1F~%`@h7<4bQ4|z> zxP1LZN_*kKYk7^a&b89#l7=`#20iCRjEj~S%P*K)R2IC8_)>756uECF>9u(D>os`U z0V-E=3kK{!tIw~G0o|aWPw}%^48*FHvf@>b?@oHqU@kWKkbH`3%do2IW=X<5RZF{^ z*k%6XUi9_PVU*^9Brl4u8K!LnDCpL}XgFsm+ z17j)rQEobvga&FI`gE~eL*IsQQfRpS2Q9GDLr)~=Jc zwMy~0*Rwy!s}VByqeykcpPo7~UF zAx!XIcy{*8U&39=$iQ0Y#6Ow2J!k4Fyf~GV_hfM&zUXH8kwmT^Rw`(;2>efVBcnJT1d_zitwH7WZbUl`_ zEO<;C^-m-#`HM&`KDSo3Sx3pjQv+s%4+NTp<+d&Lolq&~bH^aJ=5)mO5G`(HoiZM4 zRNPbRB)P%1R-2LTIkgm;{9y&3-E`I_1GgHwYUU2sO_#?yf|ink$m1t@b(0Z85TxJmm9n| z6lMMIO()DM!FU~JrE`2$chhq2J-pox<4t%J%t8FeZ}Yo*_XB#OtdREn{6=4W$F2Dgmx+VwE~FVptN>Y3p85e+Ew?!2-~nMc%i)N(t?c!lq>#=*CC%m8QFt*}=tJs%GI2Pj5{@M7#o)T++gt32R$I9 zX@t&Bir%uA1uddmMAD@md}*>m)$ORV8*Y*jozkh-ZNZ%rtjnp!-;H5j+{p<4n1PYt zV8|snF#3vS=y-*?3s$U9^oM5clv2Mw22K(Yp2_g`a){!K0>y|vcOAfN-BK5%&|#(qO=dvuX>4ui0i4%hKPAG#HJ&dCrXe^geo-K7zUx*qyj+ znenyl;8G>kMF~om3Q`nefH=?%#DUvrVQhLg8ZYSy6ir0u*)`tWWd(C@Tf>?H&o3${ zuG!$0(=>QI7iyS|e#_++Nfj_Kw5va7oO+}%5h!6q?9qC#ruQ1RrAS0kUwsL@ETHO> z@>Nfam6fG8W@s8tbn}VarXRWM?ApLNQlzl6hFAX-<_#0`R|7qEOjOxhzI0XDl7kzOmSUhE{YsO+xsX&WI#ni zvDsc)a>4`(%8KqY5NkDb{c*d%)U1}T)KQ&!4712bR#<3e*rzirKKe*vb5#*JH9d!S zuU`NV9lGw*z)|}6l3$co-PRX4%wWJL5~-v zHuXL~K7j)+RWO<)jheKD*cv?S;PLyWQqGlUgNB5 zyqF%r58j8hh*ubw_>(?jjQlvg32?$wxqJzv6g?#a7_3X{&(^lZsRl{8GOaY+p+v{M zrAJfzD{Q;pm8*I=Xp^0seb?lYo!ot6$jqTnTf(fv(8ttIYm8P`a$!?{DqVn(hn!-T?zXasD zKxLstE%r7MlhzC;wmmZ`wTWE~U$YB7G!x-0O}$kT73fWZUSe(vMg7p6Rg}(U z-(;X1)cQFRL(N)FzV5x3p^;+9(nGeqxuzM9Kd9$YXV#9>dW0H31jsbTXTHL!T(<}| zX_oFDh1k??VRxd%3iWx(lc5@8dP#YQ)MJ0lVb~kIPA8Z_`5Y^-ZmS# zEk;%-*UZ@R1?hyFL3@Awo+qrMu3jdQW8ve3je+zROQTx5__^fIn78@4W@1;fsSi@? zg1$i$0f$8T7usXp{BNlXQn#w5%^0^z0{hRpI;&nV54__@){_JA`B!ieORaX4)d{5= zdM@ZfCRB*wu^bt%!U+FnFQx^Xazoo-nLjr zmr+j*8U%zP^|IM^a)rK^wyPL=?O4gjPaVxBqEDWd8d6^+&8J?u!5}gT^Adbl7fv72 zNH!J{);1^^$18}aFj8Ykr?&tfz_x(P9{WTc@VVF*Ouu(K{vJS9sZ3}K3U-5&Ut|p% z+9r9EgwL#Xca_ndXNf@}>=!Q&{>RE01BBvf4XeL|>V^Yz)%tly^euRgF$BN4DMd*= z{(sWREA^g^M`~0gr_lu|z85vS(vT-xttYq_HCA2mpJMQ@3Z4!ZG|P4}!{@&4c81Hq zghKiT4fP2h4L#3puwu8z_YgGlHMLbT}OfjeYsLh@*`T4d^hi$R62Wz;ir zg$a#b{Fwpen5q-U-HxV8iqLd9ZB~WiH@R4QNhzK9m#{tSK(_vs z93F?A#LCmc}0Qy|ADu zpM}8ve)y__>N(12m^%Ub?7mj9dWwbtm4M(EaAQ|%|{Z~HW$w*{V z^7m;ZN6LLHiCYn8MBRlH#vwjhRJDU5j|RKy>|~Q-4YEI}R?Lh`GT1Vpf6vllA62Uz z)N*Q9!iS?7g3_khEioqo{`X47`I_%B2E(;I&PCh7G|x*)itN>&qUmh~39SZdg76ry zE1#M}mT$WlQT4g9bn)C3d_ywSmMdUtXbW4s4{s!xbrKBVSTQDj+~=^?z0qv5EADJD zn@s6@#rar8YPt z@1_&ioQ0FyE_zz7I!OFw0O~C z!QI{6iWDdm3r_K3L4#X?qCtX7ky45`xEGnw|NG9&n)xzoK5$o(6>h@0`|Pt1_qU(t zFaj8Vak?n2>cs2@m~>U=XmK??VNBTgPt?@>oxm12JAy7rL7cLAm2^2w&8$2{Q8MPLPLit#eL z<6!)?Qw6Cz0s&d_b_MF2F%U;FP%uMc%c8>O@M`mY#fql;bqJUKfh@@J7CHZ`OrPE# zPlG5fLvpJSc(e66rM`vg-rf?pf8okfbVc^P0)?47K$1&dQ=MWA^gLkZ5CvuDLszcC zV)8v16e=5oxHtUD5Ld3x%h%1w&J%^;s{(OndjsCy&*aX0+(u&R?TA0v7-+bSfCN~$ z%+}$tfXyAU026&4qdtNkiGUx`W&02yDgT3TBLtZFG49=M6q*WD3sVvG?TapEG$FWDz_03WM2i3GdK0MIXLuF zx;ZtnZ%J1JkSfwc4M=bcb@do~rK^7~~dl+L3FI zfd|xmodMX=QP?{B^ig)fYz;k%qpEq26?jE4fX-=f|>esKL zMLAI0o-b~5I_ir2oEAqimSY6rC4pt*Qc{I3DM}VUowMBb?!ByO#UT9F&{s26l!ZUt zjzuizw9?-v=dkQ=U%Qx-f_V#SNx@g+iR5||X1q_uzl^|R0Kv;Q1|36R_zA3pVlpN!W_168C1tR2nTAO$i+c2+D*{5H z@!zT=XW3Og;lE1Z`0Z6zdAcvhg8|5f>BlqJ_SDJBBonI(;K&~zpuvYGw9w|TQ0Sv+ zbJRix)f56q@U*DaEb&_UYnCI-e;Kg(+P#V(f8%I=g^$lPcB_WzN3;9vi~<$=p!Y2l zI}&L4OA8O6{Apn29fXy9w+x=R;uH8)3vH^ArU#@UglLxLHMVB=D%I)OV9&aRk^(y- zwmm$Cm77AcRsrMT*Y!df3|MH+qiuUNq%FSViTdg?oFh6_yO{b3+Ek8!sWf;Zs^1Qu zTx?|Im}8@edt-dP|2?)csTt4c)Ka{k41NVJ^D>tZjPz|Kc56vRw`49B)s0Gz0V=29 z=1AD^iDPuvvAx=f>&J4a_!D0(lN)-LHPxx`ajg<{T5sgOS^Up|_Sf6{;i4Uyy zz{Qst!xgt&!&>L_0a!_#?%ef$A_MXScXT_Q?%0(qs8=#K&GZ$+U(P;F(u3n3^L`S%IS^7^l?_sqpnx=^f4<&59kWQSkjPKx;8L>qWs zU_J(zs%f|7`NqY!ISnSjFaSQpTFB;|c84m-yIFC{V`Ny%FBC4cKeHKy=j<8H6lm8$D$6ef{Sa;Ch!HA8cw^-utoao-En@BrRXV_D5GmKyOJ5h9mZ)z-YH=qWU50_wm>8+)9%aCT5 z4273#MZmFNq0taBTle>I3~0q;-$13Kv0xEpUhP^a7=28FD z?SEumUN~=l-G7?KZJsXO!EL)jDED2mX-hJaHG^`9YFsi>TLGD zqmy%&i$N5)!hw1p1lFljZwfyO>dYPyK@N>e(06m!9SJ_VOvCs@{ty076lzkOhc+DgVMHv0Zu1}S(iD!!ASbPb@XG`5BaJ}AZq{Q868fB=H0M`S>Piu~;E zVouj-JZB-^-w6Rit?!@s^enJpwxqVJd?Kch(P@1l!moHtF=L;eb6I`HH8MxQ7ESiu z&v{S^8*)oxOu9MF3_k|}uefK;VcrGZoY$z{h&6TjHDGBLwdYmIJeHwD$nL!gmXcCyW zFcDbp|8iJB6T+McswZEIj$8Pg3EI~lj^v{BW3MiKlQj_?*JkG_5$@gi`HmJ7Inb;t zsjNTzc{w&RY*%NSdOsUbP6ld9ymN@~dBJB0&kbkt58#=qHHEI!Eq#l_7=zFm6qF*k zaPMNs4^Pvyc6^Gj2D(P=VhVv>V2_$w_)Jzy7MXdWb@orBM#xPF{9*Oqm*i%WthuG$2N#x3OD_3%bJdn9TW?InDS!5|N3(4uY zy`{69l$9ok|u7FZ)@OT4TIc>A@m3VEzt85ODG+M|> z)$}@8R-p@&glZ=~&I0%)j8@pvGK7E~4(m!Q}yk@aZfd4SBnA z%1B}Ja5)RcYWkc<&lp%u%4DI+x`YSlbR5QJQ{6Yv>Ice}C`W+cOC~axI*0yNqgsu& zhI~YJu=!?r#`{20H^2b_l#$BxTSwwHnY6GTxxd|WliW|A_5Pon3 z&LxeWrcoj7vD+8HO-9Bf)Ska7KMtP`2j?2Gmgu(V!`F=B0|FSLB%!4C=;>1EthTZ_Up=eDL^R&>}S9WM{WOXD{Fwuc)=ipmOE~@sbqj=g+9u;A4 z{YOqq9gV49)PVOJejEeDUkz&v?ky~sL)34T%jV+11kcsBIm9lz2o+h&uBb0Et8#uF zS}zoBdx&QFud1Ysdp#USWqzoVWcO`BHlp3g44a}V{No$#W@U|Zq8?ZJ4%R>GSOECy zUNXNau#l<-!f0jH7JklV5x||*W>1$}*_9{(R*J&1!h);M&s76qRXDvhJf&CuOX)(U zm#t(8*d_WxbOxgUon%^T<7Q1)jRo!JQ=)y;oZ#~@P0&7Xcn=Z$w==K_@xiLFp+x*{ ze$dHDVK~Fz47Zn%WDbA5m{r@+obIT;psnvHf!yp`c`D!-ZFWHwNBfxpi&ZqlSa(VH zxuP%VTu913-JAl?=zBG%BYw7~gqPNkA~F>PI{g7W?dW&f)}6WdLLH~ND4Va;5{LJZ7vjGjFS4pc z1e`vf)4;$sn(PSRVQhI>OIlzl)JY$B!JGh#1NdMQb`wqv&K{^F?iT9h)XdY!dV9yx zmSs23&UnejjVbn&z>6_HE>j4mYgNyU`DbkYV~f-0QredF%?M{v(Py&ba|Ecs?hGKRxiKGyuM* zfG<7533sIDzZ|ChQJnalIdth2Dup-#PSN9rg5*Z#3JGD$fL%yJZ&3J!cq%l*xaVg<@8x_L=9e>HMC1;-%Z>Z6V}U2B$xFOnIDC^A zJ{5VZrzr2oPVknibgA;!U%q(^MrM6<*dw5FAH(k|N!^m9v*Q7*Xwl{5N{F*sEgLFG0`^RCYb-is* z7s(sJPuuuX_XiNOp8MNmv!~nG-lwDM(;&LXQmGI7|C5oW`Nzn*P5f_+?Dv(4g5A1D zl9rB>t^dWxdQLy{vBUq($8P;?)gtecifKI%U3@rsB)4q)^J1;_^~I+nBkI3!pWE6y z_+eeN!>w%}{(2ki`EdArIDTPbcJG1QHoM*%`T$El-pey?U?Kn?7=9p%C|-o`t?Loinc`g-@lm%y3A6hUUI zYcq+Pdy@9~VObQ6-oC4Gd=UHjENyMVKc0WX!h@rfhytrh#wvdo$&e6@^jt90LNjQ`j;FtHdc3s}Cra-J9>s;a?>F<;ljC*Wuc@r@ zaRL!ueaJM2miW@}?ve7EWi#S5_7&npB|+YxQh<@#OU)u>IB57|$<3iEZdoXrA7rGz zV!*}g(4uBK`b~K699kuv+lPy+@y?%_9{!lyXL2>W|Ji#t6UX8iYW?T-PTBYG6*cpJ zm@69ZGxGEDJX5Nz!+Y%}KcvGf>>QRUbw>RmLW+%c~H!PmIjzv)y~46%!Ic`Oc~jcfK8nTP|;dn$R9ZVImj7jcQ zFr@ao9aVg)?RVLgN$j*Vsx2?QQu_4B66E?|chvZquYjVxvEZIU)^3{Hy9Z3VKnq;W z3t|Y|AzamAr%6I?r-|>1JG>?JmU{2cQN@VBmv?!;CETKdA6Zu-^_7Ai_irxE+;2sD z-%i zSt9-C?Z{MU`k6h9e-_Ua6_&P3$l?Fjyi}TpO)6r;X*bBfwXsZNu{f;T*U8 zZv$#eEwX0=>cLdbe+Ja3hW`wxAq`7#1L``KpO1gA^1@=iwTscayUu)T@A$ZlgOjJ| zlJq)FYR&uY{wp@lX>{_$RAHi)B~kaaPUB|m;qR($=PW}vAzDrX`<*tXlOvhv z{f<^n9^REg>M$CgM>U%QmwszHS{n&!H0e&l+ef9Mb~vm?7&o2d*<%eExTlPHt?HYw-w)kJ;RFNJL^ll;`ND~@0PUeQ`SlXN3qn#hgeCvZrsZjp7o)>Y z3Ab#$kbr+VO!IhEp*%{o119dQY)ebpKgZYI*zDcbPHUhHc41MJZcM{V@V{ve@Eep* zzPKgEVL+(vP<3PqCQ<)}1qW>0R5#Z#etoR!$3uk$-=+A<9^L*53ydf3s`!B!==f@V z43k6biJV2=*X%deg_7_q2Va_rQOP1rS$KT?P zt;84G;#IvVubPQVQQ>S1m{`r4#od;rTQRjlB&SR9stpMi`xXn1fJLNWo1T&2G|+1C z&NhR-)BL+fFU?+TUSm++<9j0OicHd){UA#ytiLOa3tjSKwni$B3^mq z&sLNm+Jyu%N1jB(w&ZVz4ScA?oyo78(vFyD{7{VpwC>SSNL#+r(foXqx<{jVR5I85 zz7Q2gu_G%8*)5BXtaeeno`!v9mO<^)o3c2SZ9(vtnlm5q_+CGot7OSLU+{LbOVv67 zLn8U`Esid>dwWrQh>itXF0myX6(K|h$g;m&!sZ*IeT!4fQ(Nx^rX)qm!TAFa_$(;D zu%aP`Ji6N$K=Zwj>^gv_y~mi=+3viA_CRq=^LssvCY~(3kvz|f&%M!J#<(c@4m)j# ztaRGSuur}Z6>c=!{Bsoio{)IhqHibDmocJ&%<~

Hadoop Job <%=job.getJobId() %>

-User : <%=HtmlQuoting.quoteHtmlChars(job.getUsername()) %>
-JobName : <%=HtmlQuoting.quoteHtmlChars(job.getJobname()) %>
-JobConf : <%=job.getJobConfPath() %>
-Submitted At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getSubmitTime(), 0 ) %>
-Launched At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(), job.getSubmitTime()) %>
-Finished At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getFinishTime(), job.getLaunchTime()) %>
-Status : <%= ((job.getJobStatus() == null)?"Incomplete" :job.getJobStatus()) %>
-
-
-<% - if (!JobStatus.getJobRunState(JobStatus.SUCCEEDED).equals(job.getJobStatus())) { - out.print("

No Analysis available as job did not finish

"); - return; - } - - HistoryViewer.AnalyzedJob avg = new HistoryViewer.AnalyzedJob(job); - JobHistoryParser.TaskAttemptInfo [] mapTasks = avg.getMapTasks(); - JobHistoryParser.TaskAttemptInfo [] reduceTasks = avg.getReduceTasks(); - - Comparator cMap = - new Comparator() { - public int compare(JobHistoryParser.TaskAttemptInfo t1, - JobHistoryParser.TaskAttemptInfo t2) { - long l1 = t1.getFinishTime() - t1.getStartTime(); - long l2 = t2.getFinishTime() - t2.getStartTime(); - return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1)); - } - }; - - Comparator cShuffle = - new Comparator() { - public int compare(JobHistoryParser.TaskAttemptInfo t1, - JobHistoryParser.TaskAttemptInfo t2) { - long l1 = t1.getShuffleFinishTime() - t1.getStartTime(); - long l2 = t2.getShuffleFinishTime() - t2.getStartTime(); - return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1)); - } - }; - - Comparator cFinishShuffle = - new Comparator() { - public int compare(JobHistoryParser.TaskAttemptInfo t1, - JobHistoryParser.TaskAttemptInfo t2) { - long l1 = t1.getShuffleFinishTime(); - long l2 = t2.getShuffleFinishTime(); - return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1)); - } - }; - - Comparator cFinishMapRed = - new Comparator() { - public int compare(JobHistoryParser.TaskAttemptInfo t1, - JobHistoryParser.TaskAttemptInfo t2) { - long l1 = t1.getFinishTime(); - long l2 = t2.getFinishTime(); - return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1)); - } - }; - - Comparator cReduce = - new Comparator() { - public int compare(JobHistoryParser.TaskAttemptInfo t1, - JobHistoryParser.TaskAttemptInfo t2) { - long l1 = t1.getFinishTime() - - t1.getShuffleFinishTime(); - long l2 = t2.getFinishTime() - - t2.getShuffleFinishTime(); - return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1)); - } - }; - - if (mapTasks == null || mapTasks.length <= 0) return; - Arrays.sort(mapTasks, cMap); - JobHistoryParser.TaskAttemptInfo minMap = mapTasks[mapTasks.length-1] ; -%> - -

Time taken by best performing Map task - -<%=minMap.getAttemptId().getTaskID() %> : <%=StringUtils.formatTimeDiff(minMap.getFinishTime(), minMap.getStartTime() ) %>

-

Average time taken by Map tasks: -<%=StringUtils.formatTimeDiff(avg.getAvgMapTime(), 0) %>

-

Worse performing map tasks

- - -<% - for (int i=0;i - - - - -<% - } -%> -
Task IdTime taken
- <%=mapTasks[i].getAttemptId().getTaskID() %><%=StringUtils.formatTimeDiff(mapTasks[i].getFinishTime(), mapTasks[i].getStartTime()) %>
-<% - Arrays.sort(mapTasks, cFinishMapRed); - JobHistoryParser.TaskAttemptInfo lastMap = mapTasks[0] ; -%> - -

The last Map task -<%=lastMap.getAttemptId().getTaskID() %> -finished at (relative to the Job launch time): -<%=StringUtils.getFormattedTimeWithDiff(dateFormat, - lastMap.getFinishTime(), - job.getLaunchTime()) %>

-
- -<% - if (reduceTasks.length <= 0) return; - Arrays.sort(reduceTasks, cShuffle); - JobHistoryParser.TaskAttemptInfo minShuffle = reduceTasks[reduceTasks.length-1] ; -%> -

Time taken by best performing shuffle -<%=minShuffle.getAttemptId().getTaskID()%> : -<%=StringUtils.formatTimeDiff(minShuffle.getShuffleFinishTime(), - minShuffle.getStartTime() ) %>

-

Average time taken by Shuffle: -<%=StringUtils.formatTimeDiff(avg.getAvgShuffleTime(), 0) %>

-

Worse performing Shuffle(s)

- - -<% - for (int i=0;i - - - - -<% - } -%> -
Task IdTime taken
-<%=reduceTasks[i].getAttemptId().getTaskID() %><%= - StringUtils.formatTimeDiff( - reduceTasks[i].getShuffleFinishTime(), - reduceTasks[i].getStartTime()) %> -
-<% - Arrays.sort(reduceTasks, cFinishShuffle); - JobHistoryParser.TaskAttemptInfo lastShuffle = reduceTasks[0] ; -%> - -

The last Shuffle -<%=lastShuffle.getAttemptId().getTaskID()%> - finished at (relative to the Job launch time): -<%=StringUtils.getFormattedTimeWithDiff(dateFormat, - lastShuffle.getShuffleFinishTime(), - job.getLaunchTime() ) %>

- -<% - Arrays.sort(reduceTasks, cReduce); - JobHistoryParser.TaskAttemptInfo minReduce = reduceTasks[reduceTasks.length-1] ; -%> -
-

Time taken by best performing Reduce task : - -<%=minReduce.getAttemptId().getTaskID() %> : -<%=StringUtils.formatTimeDiff(minReduce.getFinishTime(), - minReduce.getShuffleFinishTime() ) %>

- -

Average time taken by Reduce tasks: -<%=StringUtils.formatTimeDiff(avg.getAvgReduceTime(), 0) %>

-

Worse performing reduce tasks

- - -<% - for (int i=0;i - - - - -<% - } -%> -
Task IdTime taken
- <%=reduceTasks[i].getAttemptId().getTaskID() %><%=StringUtils.formatTimeDiff( - reduceTasks[i].getFinishTime(), - reduceTasks[i].getShuffleFinishTime()) %>
-<% - Arrays.sort(reduceTasks, cFinishMapRed); - JobHistoryParser.TaskAttemptInfo lastReduce = reduceTasks[0] ; -%> - -

The last Reduce task -<%=lastReduce.getAttemptId().getTaskID()%> - finished at (relative to the Job launch time): -<%=StringUtils.getFormattedTimeWithDiff(dateFormat, - lastReduce.getFinishTime(), - job.getLaunchTime() ) %>

-
- diff --git a/hadoop-mapreduce-project/src/webapps/job/index.html b/hadoop-mapreduce-project/src/webapps/job/index.html deleted file mode 100644 index a20a7e4f696..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/index.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - -Hadoop Administration - - - - -

Hadoop Administration

- - - - - - diff --git a/hadoop-mapreduce-project/src/webapps/job/job_authorization_error.jsp b/hadoop-mapreduce-project/src/webapps/job/job_authorization_error.jsp deleted file mode 100644 index 9e163103114..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/job_authorization_error.jsp +++ /dev/null @@ -1,53 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.net.URL" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> - - - - -Error: User cannot access this Job - - -

Error: User cannot do this operation on this Job


- -<% - String errorMsg = (String) request.getAttribute("error.msg"); -%> - - -<% - out.println(errorMsg); -%> - - -
- -<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobblacklistedtrackers.jsp b/hadoop-mapreduce-project/src/webapps/job/jobblacklistedtrackers.jsp deleted file mode 100644 index 6b47ded6073..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobblacklistedtrackers.jsp +++ /dev/null @@ -1,92 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> - -<% - JobTracker tracker = (JobTracker) application.getAttribute( - "job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); -%> -<%! - private void printBlackListedTrackers(JspWriter out, - JobInProgress job) throws IOException { - Map trackerErrors = job.getTaskTrackerErrors(); - out.print(""); - out.print("\n"); - int maxErrorsPerTracker = job.getJobConf().getMaxTaskFailuresPerTracker(); - for (Map.Entry e : trackerErrors.entrySet()) { - if (e.getValue().intValue() >= maxErrorsPerTracker) { - out.print("\n"); - } - } - out.print("
TaskTrackerNo. of Failures
" + HtmlQuoting.quoteHtmlChars(e.getKey()) + - "" + e.getValue() + "
\n"); - } -%> - -<% - String jobId = request.getParameter("jobid"); - if (jobId == null) { - out.println("

Missing 'jobid' for fetching black-listed tasktrackers!

"); - return; - } - - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, - JobID.forName(jobId), request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - if (job == null) { - out.print("Job " + jobId + " not found.
\n"); - return; - } -%> - - - -Hadoop <%=jobId%>'s black-listed tasktrackers - -

Hadoop <%=jobId%> - -Black-listed task-trackers

- -<% - printBlackListedTrackers(out, job); -%> - -
-Go back to <%=jobId%>
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobconf.jsp b/hadoop-mapreduce-project/src/webapps/job/jobconf.jsp deleted file mode 100644 index 8c157006f4d..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobconf.jsp +++ /dev/null @@ -1,95 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.net.URL" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> - - -<% - final JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - final String jobId = request.getParameter("jobid"); - if (jobId == null) { - out.println("

Missing 'jobid' for fetching job configuration!

"); - return; - } -%> - - - -Job Configuration: JobId - <%= jobId %> - - -

Job Configuration: JobId - <%= jobId %>


- -<% - - final JobID jobIdObj = JobID.forName(jobId); - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, - jobIdObj, request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - // redirect to history page if it cannot be found in memory - if (job == null) { - String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj); - if (historyFile == null) { - out.println("

Job " + jobId + " not known!

"); - return; - } - String historyUrl = "/jobconf_history.jsp?logFile=" + historyFile; - response.sendRedirect(response.encodeRedirectURL(historyUrl)); - return; - } - - String jobFilePath = tracker.getLocalJobFilePath(JobID.forName(jobId)); - FileInputStream jobFile = null; - try { - jobFile = new FileInputStream(jobFilePath); - JobConf jobConf = new JobConf(jobFilePath); - XMLUtils.transform( - jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"), - jobFile, out); - } catch (Exception e) { - out.println("Failed to retreive job configuration for job '" + jobId + "!"); - out.println(e); - } finally { - if (jobFile != null) { - try { - jobFile.close(); - } catch (IOException e) {} - } - } -%> - -
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobconf_history.jsp b/hadoop-mapreduce-project/src/webapps/job/jobconf_history.jsp deleted file mode 100644 index a7879961941..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobconf_history.jsp +++ /dev/null @@ -1,93 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.util.*" - import="org.apache.hadoop.mapreduce.jobhistory.*" - import="org.apache.hadoop.mapreduce.JobACL" - import="org.apache.hadoop.security.UserGroupInformation" - import="org.apache.hadoop.security.authorize.AccessControlList" - import="org.apache.hadoop.security.AccessControlException" -%> - -<%! private static final long serialVersionUID = 1L; -%> - -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - - String logFileString = request.getParameter("logFile"); - if (logFileString == null) { - out.println("

Missing 'logFile' for fetching job configuration!

"); - return; - } - - Path logFile = new Path(logFileString); - String jobId = JobHistory.getJobIDFromHistoryFilePath(logFile).toString(); - -%> - - - - -Job Configuration: JobId - <%= jobId %> - - -

Job Configuration: JobId - <%= jobId %>


- -<% - Path jobFilePath = JSPUtil.getJobConfFilePath(logFile); - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - FSDataInputStream jobFile = null; - try { - jobFile = fs.open(jobFilePath); - JobConf jobConf = new JobConf(jobFilePath); - JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - - JobHistoryParser.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, - response, jobTracker, fs, logFile); - if (job == null) { - return; - } - - XMLUtils.transform( - jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"), - jobFile, out); - } catch (Exception e) { - out.println("Failed to retreive job configuration for job '" + jobId + "!"); - out.println(e); - } finally { - if (jobFile != null) { - try { - jobFile.close(); - } catch (IOException e) {} - } - } -%> - -
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobdetails.jsp b/hadoop-mapreduce-project/src/webapps/job/jobdetails.jsp deleted file mode 100644 index f39e9e96978..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobdetails.jsp +++ /dev/null @@ -1,493 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.text.*" - import="java.util.*" - import="java.text.DecimalFormat" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.mapreduce.TaskType" - import="org.apache.hadoop.util.*" - import="org.apache.hadoop.fs.Path" - import="org.apache.hadoop.mapreduce.jobhistory.JobHistory" - import="org.apache.hadoop.mapreduce.JobACL" - import="org.apache.hadoop.security.UserGroupInformation" - import="java.security.PrivilegedExceptionAction" - import="org.apache.hadoop.security.AccessControlException" - import="org.apache.hadoop.security.authorize.AccessControlList" -%> - -<%! private static final long serialVersionUID = 1L; -%> - -<% - final JobTracker tracker = (JobTracker) application.getAttribute( - "job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); -%> -<%! - private void printTaskSummary(JspWriter out, - String jobId, - String kind, - double completePercent, - TaskInProgress[] tasks - ) throws IOException { - int totalTasks = tasks.length; - int runningTasks = 0; - int finishedTasks = 0; - int killedTasks = 0; - int failedTaskAttempts = 0; - int killedTaskAttempts = 0; - for(int i=0; i < totalTasks; ++i) { - TaskInProgress task = tasks[i]; - if (task.isComplete()) { - finishedTasks += 1; - } else if (task.isRunning()) { - runningTasks += 1; - } else if (task.wasKilled()) { - killedTasks += 1; - } - failedTaskAttempts += task.numTaskFailures(); - killedTaskAttempts += task.numKilledTasks(); - } - int pendingTasks = totalTasks - runningTasks - killedTasks - finishedTasks; - out.print("" + kind + - "" + - StringUtils.formatPercent(completePercent, 2) + - ServletUtil.percentageGraph((int)(completePercent * 100), 80) + - "" + - totalTasks + - "" + - ((pendingTasks > 0) - ? "" + pendingTasks + "" - : "0") + - "" + - ((runningTasks > 0) - ? "" + runningTasks + "" - : "0") + - "" + - ((finishedTasks > 0) - ?"" + finishedTasks + "" - : "0") + - "" + - ((killedTasks > 0) - ?"" + killedTasks + "" - : "0") + - "" + - ((failedTaskAttempts > 0) ? - ("" + failedTaskAttempts + - "") : - "0" - ) + - " / " + - ((killedTaskAttempts > 0) ? - ("" + killedTaskAttempts + - "") : - "0" - ) + - "\n"); - } - - private void printJobLevelTaskSummary(JspWriter out, - String jobId, - String kind, - TaskInProgress[] tasks - ) throws IOException { - int totalTasks = tasks.length; - int runningTasks = 0; - int finishedTasks = 0; - int killedTasks = 0; - for(int i=0; i < totalTasks; ++i) { - TaskInProgress task = tasks[i]; - if (task.isComplete()) { - finishedTasks += 1; - } else if (task.isRunning()) { - runningTasks += 1; - } else if (task.isFailed()) { - killedTasks += 1; - } - } - int pendingTasks = totalTasks - runningTasks - killedTasks - finishedTasks; - out.print(((runningTasks > 0) - ? "" + " Running" + - "" - : ((pendingTasks > 0) ? " Pending" : - ((finishedTasks > 0) - ?"" + " Successful" - + "" - : ((killedTasks > 0) - ?"" + " Failed" - + "" : "None"))))); - } - - private void printConfirm(JspWriter out, String jobId) throws IOException{ - String url = "jobdetails.jsp?jobid=" + jobId; - out.print("" - + "

Are you sure you want to kill " + jobId - + " ?


" - + "
" - + "" - + "" - + "
" - + "
"); - } - -%> -<% - String jobId = request.getParameter("jobid"); - String refreshParam = request.getParameter("refresh"); - if (jobId == null) { - out.println("

Missing 'jobid'!

"); - return; - } - - int refresh = 60; // refresh every 60 seconds by default - if (refreshParam != null) { - try { - refresh = Integer.parseInt(refreshParam); - } - catch (NumberFormatException ignored) { - } - } - - final JobID jobIdObj = JobID.forName(jobId); - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, jobIdObj, - request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - - final String newPriority = request.getParameter("prio"); - String user = request.getRemoteUser(); - UserGroupInformation ugi = null; - if (user != null) { - ugi = UserGroupInformation.createRemoteUser(user); - } - String action = request.getParameter("action"); - if(JSPUtil.privateActionsAllowed(tracker.conf) && - "changeprio".equalsIgnoreCase(action) - && request.getMethod().equalsIgnoreCase("POST")) { - if (ugi != null) { - try { - ugi.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException{ - - // checks job modify permission - tracker.setJobPriority(jobIdObj, - JobPriority.valueOf(newPriority)); - return null; - } - }); - } catch(AccessControlException e) { - String errMsg = "User " + user + " failed to modify priority of " + - jobIdObj + "!

" + e.getMessage() + - "
Go back to Job
"; - JSPUtil.setErrorAndForward(errMsg, request, response); - return; - } - } - else {// no authorization needed - tracker.setJobPriority(jobIdObj, - JobPriority.valueOf(newPriority));; - } - } - - if(JSPUtil.privateActionsAllowed(tracker.conf)) { - action = request.getParameter("action"); - if(action!=null && action.equalsIgnoreCase("confirm")) { - printConfirm(out, jobId); - return; - } - else if(action != null && action.equalsIgnoreCase("kill") && - request.getMethod().equalsIgnoreCase("POST")) { - if (ugi != null) { - try { - ugi.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException{ - - // checks job modify permission - tracker.killJob(jobIdObj);// checks job modify permission - return null; - } - }); - } catch(AccessControlException e) { - String errMsg = "User " + user + " failed to kill " + jobIdObj + - "!

" + e.getMessage() + - "
Go back to Job
"; - JSPUtil.setErrorAndForward(errMsg, request, response); - return; - } - } - else {// no authorization needed - tracker.killJob(jobIdObj); - } - } - } - -%> - -<%@page import="org.apache.hadoop.mapred.TaskGraphServlet"%> - - - - <% - if (refresh != 0) { - %> - - <% - } - %> -Hadoop <%=jobId%> on <%=trackerName%> - - - -

Hadoop <%=jobId%> on <%=trackerName%>

- -<% - if (job == null) { - String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj); - if (historyFile == null) { - out.println("

Job " + jobId + " not known!

"); - return; - } - String historyUrl = "/jobdetailshistory.jsp?jobid=" + - jobId + "&logFile=" + historyFile; - response.sendRedirect(response.encodeRedirectURL(historyUrl)); - return; - } - JobProfile profile = job.getProfile(); - JobStatus status = job.getStatus(); - int runState = status.getRunState(); - int flakyTaskTrackers = job.getNoOfBlackListedTrackers(); - out.print("User: " + - HtmlQuoting.quoteHtmlChars(profile.getUser()) + "
\n"); - out.print("Job Name: " + - HtmlQuoting.quoteHtmlChars(profile.getJobName()) + "
\n"); - out.print("Job File: " + - profile.getJobFile() + "
\n"); - out.print("Submit Host: " + - HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostName()) + "
\n"); - out.print("Submit Host Address: " + - HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostAddress()) + "
\n"); - - Map jobAcls = status.getJobACLs(); - JSPUtil.printJobACLs(tracker, jobAcls, out); - - out.print("Job Setup:"); - printJobLevelTaskSummary(out, jobId, "setup", - job.getTasks(TaskType.JOB_SETUP)); - out.print("
\n"); - if (runState == JobStatus.RUNNING) { - out.print("Status: Running
\n"); - out.print("Started at: " + new Date(job.getStartTime()) + "
\n"); - out.print("Running for: " + StringUtils.formatTimeDiff( - System.currentTimeMillis(), job.getStartTime()) + "
\n"); - } else { - if (runState == JobStatus.SUCCEEDED) { - out.print("Status: Succeeded
\n"); - out.print("Started at: " + new Date(job.getStartTime()) + "
\n"); - out.print("Finished at: " + new Date(job.getFinishTime()) + - "
\n"); - out.print("Finished in: " + StringUtils.formatTimeDiff( - job.getFinishTime(), job.getStartTime()) + "
\n"); - } else if (runState == JobStatus.FAILED) { - out.print("Status: Failed
\n"); - out.print("Started at: " + new Date(job.getStartTime()) + "
\n"); - out.print("Failed at: " + new Date(job.getFinishTime()) + - "
\n"); - out.print("Failed in: " + StringUtils.formatTimeDiff( - job.getFinishTime(), job.getStartTime()) + "
\n"); - } else if (runState == JobStatus.KILLED) { - out.print("Status: Killed
\n"); - out.print("Started at: " + new Date(job.getStartTime()) + "
\n"); - out.print("Killed at: " + new Date(job.getFinishTime()) + - "
\n"); - out.print("Killed in: " + StringUtils.formatTimeDiff( - job.getFinishTime(), job.getStartTime()) + "
\n"); - } - } - out.print("Job Cleanup:"); - printJobLevelTaskSummary(out, jobId, "cleanup", - job.getTasks(TaskType.JOB_CLEANUP)); - out.print("
\n"); - if (flakyTaskTrackers > 0) { - out.print("Black-listed TaskTrackers: " + - "" + - flakyTaskTrackers + "
\n"); - } - if (job.getSchedulingInfo() != null) { - out.print("Job Scheduling information: " + - job.getSchedulingInfo().toString() +"\n"); - } - out.print("
\n"); - out.print(""); - out.print("" + - "" + - "" + - "\n"); - printTaskSummary(out, jobId, "map", status.mapProgress(), - job.getTasks(TaskType.MAP)); - printTaskSummary(out, jobId, "reduce", status.reduceProgress(), - job.getTasks(TaskType.REDUCE)); - out.print("
Kind% CompleteNum TasksPendingRunningCompleteKilledFailed/Killed
Task Attempts
\n"); - - %> -

- - - - - - - - - <% - Counters mapCounters = job.getMapCounters(); - Counters reduceCounters = job.getReduceCounters(); - Counters totalCounters = job.getCounters(); - - for (String groupName : totalCounters.getGroupNames()) { - Counters.Group totalGroup = totalCounters.getGroup(groupName); - Counters.Group mapGroup = mapCounters.getGroup(groupName); - Counters.Group reduceGroup = reduceCounters.getGroup(groupName); - - Format decimal = new DecimalFormat(); - - boolean isFirst = true; - for (Counters.Counter counter : totalGroup) { - String name = counter.getDisplayName(); - String mapValue = decimal.format(mapGroup.getCounter(name)); - String reduceValue = decimal.format(reduceGroup.getCounter(name)); - String totalValue = decimal.format(counter.getCounter()); - %> - - <% - if (isFirst) { - isFirst = false; - %> - - <% - } - %> - - - - - - <% - } - } - %> -

CounterMapReduceTotal
- <%=HtmlQuoting.quoteHtmlChars(totalGroup.getDisplayName())%><%=HtmlQuoting.quoteHtmlChars(name)%><%=mapValue%><%=reduceValue%><%=totalValue%>
- -

Map Completion Graph - -<% -if("off".equals(request.getParameter("map.graph"))) { - session.setAttribute("map.graph", "off"); -} else if("on".equals(request.getParameter("map.graph"))){ - session.setAttribute("map.graph", "on"); -} -if("off".equals(request.getParameter("reduce.graph"))) { - session.setAttribute("reduce.graph", "off"); -} else if("on".equals(request.getParameter("reduce.graph"))){ - session.setAttribute("reduce.graph", "on"); -} - -if("off".equals(session.getAttribute("map.graph"))) { %> - open -<%} else { %> - close -
-<%}%> - -<%if(job.getTasks(TaskType.REDUCE).length > 0) { %> -
Reduce Completion Graph - -<%if("off".equals(session.getAttribute("reduce.graph"))) { %> - open -<%} else { %> - close - -
-<%} }%> - -
-<% if(JSPUtil.privateActionsAllowed(tracker.conf)) { %> -
- Change priority from <%=job.getPriority()%> to: -
- - -
-<% } %> - - - -<% if(JSPUtil.privateActionsAllowed(tracker.conf) - && runState == JobStatus.RUNNING) { %> -
Kill this job -<% } %> - -
- -
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobdetailshistory.jsp b/hadoop-mapreduce-project/src/webapps/job/jobdetailshistory.jsp deleted file mode 100644 index 232e71b3218..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobdetailshistory.jsp +++ /dev/null @@ -1,291 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapreduce.TaskAttemptID" - import="org.apache.hadoop.mapreduce.TaskID" - import="org.apache.hadoop.mapreduce.Counter" - import="org.apache.hadoop.mapreduce.Counters" - import="org.apache.hadoop.mapreduce.CounterGroup" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.*" - import="java.text.*" - import="org.apache.hadoop.mapreduce.jobhistory.*" - import="java.security.PrivilegedExceptionAction" - import="org.apache.hadoop.security.AccessControlException" - import="org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo" - import="org.apache.hadoop.mapreduce.JobACL" - import="org.apache.hadoop.security.authorize.AccessControlList" -%> -<%!private static final long serialVersionUID = 1L; -%> - -<%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %> -<% - String logFile = request.getParameter("logFile"); - String reasonforFailure = " "; - final Path jobFile = new Path(logFile); - String jobid = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString(); - - final FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - final JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, response, - jobTracker, fs, jobFile); - if (job == null) { - return; - } - if (job.getJobStatus().equals("FAILED")) - reasonforFailure = job.getErrorInfo(); -%> - - - - -Hadoop Job <%=jobid%> on History Viewer - - - - -

Hadoop Job <%=jobid %> on History Viewer

- -User: <%=HtmlQuoting.quoteHtmlChars(job.getUsername()) %>
-JobName: <%=HtmlQuoting.quoteHtmlChars(job.getJobname()) %>
-JobConf: - <%=job.getJobConfPath() %>
-<% - Map jobAcls = job.getJobACLs(); - JSPUtil.printJobACLs(jobTracker, jobAcls, out); -%> -Submitted At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getSubmitTime(), 0 ) %>
-Launched At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(), job.getSubmitTime()) %>
-Finished At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getFinishTime(), job.getLaunchTime()) %>
-Status: <%= ((job.getJobStatus()) == null ? "Incomplete" :job.getJobStatus()) %>
-ReasonForFailure: <%=reasonforFailure %>
-<% - HistoryViewer.SummarizedJob sj = new HistoryViewer.SummarizedJob(job); -%> -Analyse This Job -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KindTotal Tasks(successful+failed+killed)Successful tasksFailed tasksKilled tasksStart TimeFinish Time
Setup - <%=sj.getTotalSetups()%> - <%=sj.getNumFinishedSetups()%> - <%=sj.getNumFailedSetups()%> - <%=sj.getNumKilledSetups()%><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupStarted(), 0) %><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupFinished(), sj.getSetupStarted()) %>
Map - <%=sj.getTotalMaps()%> - <%=job.getFinishedMaps() %> - <%=sj.getNumFailedMaps()%> - <%=sj.getNumKilledMaps()%><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getMapStarted(), 0) %><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getMapFinished(), sj.getMapStarted()) %>
Reduce - <%=sj.getTotalReduces()%> - <%=job.getFinishedReduces()%> - <%=sj.getNumFailedReduces()%> - <%=sj.getNumKilledReduces()%><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(), 0) %><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(), sj.getReduceStarted()) %>
Cleanup - <%=sj.getTotalCleanups()%> - <%=sj.getNumFinishedCleanups()%> - <%=sj.getNumFailedCleanups()%> - <%=sj.getNumKilledCleanups()%><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getCleanupStarted(), 0) %><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getCleanupFinished(), sj.getCleanupStarted()) %>
- -
-
- - - - - - - - - - -<% - - Counters totalCounters = job.getTotalCounters(); - Counters mapCounters = job.getMapCounters(); - Counters reduceCounters = job.getReduceCounters(); - - if (totalCounters != null) { - for (String groupName : totalCounters.getGroupNames()) { - CounterGroup totalGroup = totalCounters.getGroup(groupName); - CounterGroup mapGroup = mapCounters.getGroup(groupName); - CounterGroup reduceGroup = reduceCounters.getGroup(groupName); - - Format decimal = new DecimalFormat(); - - boolean isFirst = true; - Iterator ctrItr = totalGroup.iterator(); - while(ctrItr.hasNext()) { - Counter counter = ctrItr.next(); - String name = counter.getName(); - String mapValue = - decimal.format(mapGroup.findCounter(name).getValue()); - String reduceValue = - decimal.format(reduceGroup.findCounter(name).getValue()); - String totalValue = - decimal.format(counter.getValue()); -%> - -<% - if (isFirst) { - isFirst = false; -%> - -<% - } -%> - - - - - -<% - } - } - } -%> -

CounterMapReduceTotal
- <%=HtmlQuoting.quoteHtmlChars(totalGroup.getDisplayName())%><%=HtmlQuoting.quoteHtmlChars(counter.getDisplayName())%><%=mapValue%><%=reduceValue%><%=totalValue%>
-
- -
- <% - HistoryViewer.FilteredJob filter = new HistoryViewer.FilteredJob(job,TaskStatus.State.FAILED.toString()); - Map> badNodes = filter.getFilteredMap(); - if (badNodes.size() > 0) { - %> -

Failed tasks attempts by nodes

- - - <% - for (Map.Entry> entry : badNodes.entrySet()) { - String node = entry.getKey(); - Set failedTasks = entry.getValue(); -%> - - - - -<% - } - } - %> -
HostnameFailed Tasks
<%=node %> -<% - boolean firstId = true; - for (TaskID tid : failedTasks) { - if (firstId) { - firstId = false; -%> - <%=tid %> -<% - } else { -%> - , <%=tid %> -<% - } - } -%> -
-
- - <% - filter = new HistoryViewer.FilteredJob(job, TaskStatus.State.KILLED.toString()); - badNodes = filter.getFilteredMap(); - if (badNodes.size() > 0) { - %> -

Killed tasks attempts by nodes

- - - <% - for (Map.Entry> entry : badNodes.entrySet()) { - String node = entry.getKey(); - Set killedTasks = entry.getValue(); -%> - - - - -<% - } - } -%> -
HostnameKilled Tasks
<%=node %> -<% - boolean firstId = true; - for (TaskID tid : killedTasks) { - if (firstId) { - firstId = false; -%> - <%=tid %> -<% - } else { -%> - , <%=tid %> -<% - } - } -%> -
- - diff --git a/hadoop-mapreduce-project/src/webapps/job/jobfailures.jsp b/hadoop-mapreduce-project/src/webapps/job/jobfailures.jsp deleted file mode 100644 index 57530999947..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobfailures.jsp +++ /dev/null @@ -1,202 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.mapreduce.TaskType" - import="org.apache.hadoop.util.*" -%> - -<%! private static final long serialVersionUID = 1L; -%> - -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); -%> -<%! - private void printFailedAttempts(JspWriter out, - JobTracker tracker, - TaskInProgress tip, - TaskStatus.State failState) throws IOException { - TaskStatus[] statuses = tip.getTaskStatuses(); - TaskID tipId = tip.getTIPId(); - for(int i=0; i < statuses.length; ++i) { - TaskStatus.State taskState = statuses[i].getRunState(); - if ((failState == null && (taskState == TaskStatus.State.FAILED || - taskState == TaskStatus.State.KILLED)) || taskState == failState) { - String taskTrackerName = statuses[i].getTaskTracker(); - TaskTrackerStatus taskTracker = tracker.getTaskTrackerStatus(taskTrackerName); - out.print("" + statuses[i].getTaskID() + - "" + - tipId + ""); - if (taskTracker == null) { - out.print("" + taskTrackerName + ""); - } else { - out.print("" + taskTracker.getHost() + - ""); - } - out.print("" + taskState + ""); - out.print("
");
-        String[] failures = 
-                     tracker.getTaskDiagnostics(statuses[i].getTaskID());
-        if (failures == null) {
-          out.print(" ");
-        } else {
-          for(int j = 0 ; j < failures.length ; j++){
-            out.print(HtmlQuoting.quoteHtmlChars(failures[j]));
-            if (j < (failures.length - 1)) {
-              out.print("\n-------\n");
-            }
-          }
-        }
-        out.print("
"); - - out.print(""); - String taskLogUrl = null; - if (taskTracker != null) { - taskLogUrl = TaskLogServlet.getTaskLogUrl(taskTracker.getHost(), - String.valueOf(taskTracker.getHttpPort()), - statuses[i].getTaskID().toString()); - } - if (taskLogUrl != null) { - String tailFourKBUrl = taskLogUrl + "&start=-4097"; - String tailEightKBUrl = taskLogUrl + "&start=-8193"; - String entireLogUrl = taskLogUrl; - out.print("Last 4KB
"); - out.print("Last 8KB
"); - out.print("All
"); - } else { - out.print("n/a"); // task tracker was lost - } - out.print(""); - - out.print("\n"); - } - } - } - - private void printFailures(JspWriter out, - JobTracker tracker, - JobInProgress job, - String kind, - String cause) - throws IOException, InterruptedException, ServletException { - - boolean includeMap = false; - boolean includeReduce = false; - if (kind == null) { - includeMap = true; - includeReduce = true; - } else if ("map".equals(kind)) { - includeMap = true; - } else if ("reduce".equals(kind)) { - includeReduce = true; - } else if ("all".equals(kind)) { - includeMap = true; - includeReduce = true; - } else { - out.print("Kind " + kind + - " not supported.
\n"); - return; - } - - TaskStatus.State state = null; - try { - if (cause != null) { - state = TaskStatus.State.valueOf(cause.toUpperCase()); - if (state != TaskStatus.State.FAILED && state != TaskStatus.State.KILLED) { - out.print("Cause '" + cause + - "' is not an 'unsuccessful' state.
\n"); - return; - } - } - } catch (IllegalArgumentException e) { - out.print("Cause '" + cause + "' not supported.
\n"); - return; - } - - out.print(""); - out.print("" + - "\n"); - if (includeMap) { - TaskInProgress[] tips = job.getTasks(TaskType.MAP); - for(int i=0; i < tips.length; ++i) { - printFailedAttempts(out, tracker, tips[i], state); - } - } - if (includeReduce) { - TaskInProgress[] tips = job.getTasks(TaskType.REDUCE); - for(int i=0; i < tips.length; ++i) { - printFailedAttempts(out, tracker, tips[i], state); - } - } - out.print("
AttemptTaskMachineStateErrorLogs
\n"); - } -%> - -<% - String jobId = request.getParameter("jobid"); - if (jobId == null) { - out.println("

Missing 'jobid'!

"); - return; - } - JobID jobIdObj = JobID.forName(jobId); - - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob( - tracker, jobIdObj, request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - if (job == null) { - out.print("Job " + jobId + " not found.
\n"); - return; - } - - String kind = request.getParameter("kind"); - String cause = request.getParameter("cause"); -%> - - - -Hadoop <%=jobId%> failures on <%=trackerName%> - -

Hadoop <%=jobId%> -failures on <%=trackerName%>

- -<% - printFailures(out, tracker, job, kind, cause); -%> - -
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobhistory.jsp b/hadoop-mapreduce-project/src/webapps/job/jobhistory.jsp deleted file mode 100644 index d9cc8e1bb53..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobhistory.jsp +++ /dev/null @@ -1,318 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="java.io.*" - import="java.util.*" - import="java.net.URLEncoder" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.*" - import="org.apache.hadoop.fs.*" - import="javax.servlet.jsp.*" - import="java.text.SimpleDateFormat" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapreduce.jobhistory.*" -%> - -<%! private static final long serialVersionUID = 1L; -%> - -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); -%> -<%! - private static SimpleDateFormat dateFormat = - new SimpleDateFormat("d/MM HH:mm:ss"); -%> - - - - - -<%= trackerName %> Hadoop Map/Reduce History Viewer - - - -

<%= trackerName %> Hadoop Map/Reduce - History Viewer

-
-<% - final String search = (request.getParameter("search") == null) - ? "" - : request.getParameter("search"); - - String parts[] = search.split(":"); - - final String user = (parts.length >= 1) - ? parts[0].toLowerCase() - : ""; - final String jobid = (parts.length >= 2) - ? parts[1].toLowerCase() - : ""; - final String rawUser = HtmlQuoting.unquoteHtmlChars(user); - final String rawJobid = HtmlQuoting.unquoteHtmlChars(jobid); - - PathFilter jobLogFileFilter = new PathFilter() { - private boolean matchUser(String fileName) { - // return true if - // - user is not specified - // - user matches - return "".equals(rawUser) || rawUser.equals(fileName.split("_")[3]); - } - - private boolean matchJobId(String fileName) { - // return true if - // - jobid is not specified - // - jobid matches - String[] jobDetails = fileName.split("_"); - String actualId = jobDetails[0] + "_" +jobDetails[1] + "_" + jobDetails[2] ; - return "".equals(rawJobid) || jobid.equalsIgnoreCase(actualId); - } - - public boolean accept(Path path) { - return (!(path.getName().endsWith(".xml") || - path.getName().endsWith(JobHistory.OLD_SUFFIX)) && - matchUser(path.getName()) && matchJobId(path.getName())); - } - }; - - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - String historyLogDir = (String) application.getAttribute("historyLogDir"); - if (fs == null) { - out.println("Null file system. May be namenode is in safemode!"); - return; - } - Path[] jobFiles = FileUtil.stat2Paths(fs.listStatus(new Path(historyLogDir), - jobLogFileFilter)); - out.println(""); - if (null == jobFiles || jobFiles.length == 0) { - out.println("No files found!"); - return ; - } - - // get the pageno - int pageno = request.getParameter("pageno") == null - ? 1 - : Integer.parseInt(request.getParameter("pageno")); - - // get the total number of files to display - int size = 100; - - // if show-all is requested or jobfiles < size(100) - if (pageno == -1 || size > jobFiles.length) { - size = jobFiles.length; - } - - if (pageno == -1) { // special case 'show all' - pageno = 1; - } - - int maxPageNo = (int)Math.ceil((float)jobFiles.length / size); - - // check and fix pageno - if (pageno < 1 || pageno > maxPageNo) { - out.println("Invalid page index"); - return ; - } - - int length = size ; // determine the length of job history files to be displayed - if (pageno == maxPageNo) { - // find the number of files to be shown on the last page - int startOnLast = ((pageno - 1) * size) + 1; - length = jobFiles.length - startOnLast + 1; - } - - // Display the search box - out.println("
Filter (username:jobid) "); // heading - out.println(""); // search box - out.println("
"); - out.println("Example: 'smith' will display jobs submitted by user 'smith'. "); - out.println("Job Ids need to be prefixed with a colon(:) For example, :job_200908311030_0001 will display the job with that id. "); // example - out.println("
"); - - //Show the status - int start = (pageno - 1) * size + 1; - - // DEBUG - out.println(""); - - out.println("Available Jobs in History "); - // display the number of jobs, start index, end index - out.println("( Displaying " + length + " jobs from " + start + " to " + (start + length - 1) + " out of " + jobFiles.length + " jobs"); - if (!"".equals(user)) { - out.println(" for user " + HtmlQuoting.quoteHtmlChars(user) + ""); // show the user if present - } - if (!"".equals(jobid)) { - out.println(" for jobid " + HtmlQuoting.quoteHtmlChars(jobid) + " in it."); // show the jobid keyword if present - } - out.print(")"); - - // show the 'show-all' link - out.println(" [show all]"); - - // show the 'first-page' link - if (pageno > 1) { - out.println(" [first page]"); - } else { - out.println("[first page]"); - } - - // show the 'last-page' link - if (pageno < maxPageNo) { - out.println(" [last page]"); - } else { - out.println("[last page]"); - } - - // sort the files on creation time. - Arrays.sort(jobFiles, new Comparator() { - public int compare(Path p1, Path p2) { - String dp1 = null; - String dp2 = null; - - dp1 = p1.getName(); - dp2 = p2.getName(); - - String[] split1 = dp1.split("_"); - String[] split2 = dp2.split("_"); - - // compare job tracker start time - int res = new Date(Long.parseLong(split1[1])).compareTo( - new Date(Long.parseLong(split2[1]))); - if (res == 0) { - Long l1 = Long.parseLong(split1[2]); - res = l1.compareTo(Long.parseLong(split2[2])); - } - return res; - } - }); - - out.println("

"); - - // print the navigation info (top) - printNavigation(pageno, size, maxPageNo, search, out); - - out.print(""); - out.print(""); - out.print( "") ; - out.print(""); - - Set displayedJobs = new HashSet(); - for (int i = start - 1; i < start + length - 1; ++i) { - Path jobFile = jobFiles[i]; - - String jobId = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString(); - String userName = JobHistory.getUserFromHistoryFilePath(jobFile); - - // Check if the job is already displayed. There can be multiple job - // history files for jobs that have restarted - if (displayedJobs.contains(jobId)) { - continue; - } else { - displayedJobs.add(jobId); - } - -%> -
-<% - printJob(jobId, userName, new Path(jobFile.getParent(), jobFile), - out) ; -%> -
-<% - } // end while trackers - out.print("
Job IdUser
"); - - // show the navigation info (bottom) - printNavigation(pageno, size, maxPageNo, search, out); -%> -<%! - private void printJob(String jobId, - String user, Path logFile, JspWriter out) - throws IOException { - out.print(""); - out.print("" + "" + HtmlQuoting.quoteHtmlChars(jobId) + ""); - out.print("" + HtmlQuoting.quoteHtmlChars(user) + ""); - out.print(""); - } - - private void printNavigation(int pageno, int size, int max, String search, - JspWriter out) throws IOException { - int numIndexToShow = 5; // num indexes to show on either side - - //TODO check this on boundary cases - out.print("
<"); - - // show previous link - if (pageno > 1) { - out.println("Previous"); - } - - // display the numbered index 1 2 3 4 - int firstPage = pageno - numIndexToShow; - if (firstPage < 1) { - firstPage = 1; // boundary condition - } - - int lastPage = pageno + numIndexToShow; - if (lastPage > max) { - lastPage = max; // boundary condition - } - - // debug - out.println(""); - - for (int i = firstPage; i <= lastPage; ++i) { - if (i != pageno) {// needs hyperlink - out.println(" " + i + " "); - } else { // current page - out.println(i); - } - } - - // show the next link - if (pageno < max) { - out.println("Next"); - } - out.print(">
"); - } -%> - diff --git a/hadoop-mapreduce-project/src/webapps/job/jobqueue_details.jsp b/hadoop-mapreduce-project/src/webapps/job/jobqueue_details.jsp deleted file mode 100644 index d429e785200..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobqueue_details.jsp +++ /dev/null @@ -1,107 +0,0 @@ -<%/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.util.Vector" - import="java.util.Collection" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.StringUtils" - import="org.apache.hadoop.util.ServletUtil" -%> -<%!private static final long serialVersionUID = 526456771152222127L;%> -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = StringUtils.simpleHostname(tracker - .getJobTrackerMachine()); - String queueName = request.getParameter("queueName"); - TaskScheduler scheduler = tracker.getTaskScheduler(); - JobQueueInfo schedInfo = tracker.getQueueInfo(queueName); -%> - - - -Queue details for <%=queueName != null ? queueName : ""%> - - - - -<% - if (!JSPUtil.processButtons(request, response, tracker)) { - return;// user is not authorized - } -%> -<% - String schedulingInfoString = schedInfo.getSchedulingInfo(); -%> -

Hadoop Job Queue Scheduling Information on - <%=trackerName%> -

-
-Scheduling Information : -<%=HtmlQuoting.quoteHtmlChars(schedulingInfoString).replaceAll("\n", "
")%> -
-
-<% - if (schedInfo.getChildren() != null && schedInfo.getChildren().size() > 0) { -%> -Child Queues : -<% - for (JobQueueInfo childQueue : schedInfo.getChildren()) { - String[] childNameSplits = childQueue.getQueueName().split(":"); - String childName = childNameSplits[childNameSplits.length -1]; -%> - - <%=childName%>    -<% - } -%> -
-<% - } else { - Collection jobs = scheduler.getJobs(queueName); - if (jobs == null || jobs.isEmpty()) { -%> -
-

No Jobs found for the Queue :: <%=queueName != null ? queueName : ""%>

-
-
-<% - } else { -%> -
-

Job Summary for the Queue :: <%=queueName != null ? queueName : ""%>

-
-
-(In the order maintained by the scheduler) -
-
-
-<%=JSPUtil.generateJobTable("Job List", jobs, 30, 0, tracker.conf)%> -
-<% - } - } -%> - -<% - out.println(ServletUtil.htmlFooter()); -%> - diff --git a/hadoop-mapreduce-project/src/webapps/job/jobtable.jsp b/hadoop-mapreduce-project/src/webapps/job/jobtable.jsp deleted file mode 100644 index 7c7fe3b8501..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobtable.jsp +++ /dev/null @@ -1,73 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page -contentType="text/html; charset=UTF-8" -import="org.apache.hadoop.mapred.*" -import="javax.servlet.*" -import="javax.servlet.http.*" -import="java.io.*" -import="java.util.*" -import="org.apache.hadoop.util.ServletUtil" -%> -<%! -private static final long serialVersionUID = 1L; -%> -<% -JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); -QueueManager qmgr = tracker.getQueueManager(); -String queue = request.getParameter("queue_name"); -TaskScheduler scheduler = tracker.getTaskScheduler(); -JobQueueInfo queueInfo = tracker.getQueueInfo(queue); -%> -<% -if(queueInfo == null || (queueInfo.getChildren() != null && - queueInfo.getChildren().size() != 0) ){ -%> -<% -} else { -%> -<% -Collection jobs = scheduler.getJobs(queue); -String[] queueLabelSplits = queue.split(":"); -String queueLabel = - queueLabelSplits.length==0?queue:queueLabelSplits[queueLabelSplits.length-1]; - -if(jobs == null || jobs.isEmpty()) { -%> -
-

No Jobs found for the QueueName:: <%=queueLabel%> -

-
-<% -}else { -%> -
-

-

Job Summary for the Queue :: <%=queueLabel%>

-(In the order maintained by the scheduler) -


-<%= - JSPUtil.generateJobTable("Job List", jobs, 30, 5, tracker.conf) -%> -
-<% -} -%> -<%} %> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobtasks.jsp b/hadoop-mapreduce-project/src/webapps/job/jobtasks.jsp deleted file mode 100644 index e9f574e0160..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobtasks.jsp +++ /dev/null @@ -1,173 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.util.*" - import="java.lang.Integer" - import="java.text.SimpleDateFormat" -%> -<%! private static final long serialVersionUID = 1L; -%> -<%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %> -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); - String jobid = request.getParameter("jobid"); - if (jobid == null) { - out.println("

Missing 'jobid'!

"); - return; - } - JobID jobidObj = JobID.forName(jobid); - - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, jobidObj, - request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - - String type = request.getParameter("type"); - String pagenum = request.getParameter("pagenum"); - String state = request.getParameter("state"); - state = (state!=null) ? state : "all"; - int pnum = Integer.parseInt(pagenum); - int next_page = pnum+1; - int numperpage = 2000; - - TaskReport[] reports = null; - int start_index = (pnum - 1) * numperpage; - int end_index = start_index + numperpage; - int report_len = 0; - if ("map".equals(type)) { - reports = (job != null) ? tracker.getMapTaskReports(jobidObj) : null; - } else if ("reduce".equals(type)) { - reports = (job != null) ? tracker.getReduceTaskReports(jobidObj) : null; - } else if ("cleanup".equals(type)) { - reports = (job != null) ? tracker.getCleanupTaskReports(jobidObj) : null; - } else if ("setup".equals(type)) { - reports = (job != null) ? tracker.getSetupTaskReports(jobidObj) : null; - } -%> - - - - - Hadoop <%=type%> task list for <%=jobid%> on <%=trackerName%> - - - -

Hadoop <%=type%> task list for -<%=jobid%> on -<%=trackerName%>

-<% - // redirect to history page if it cannot be found in memory - if (job == null) { - JobID jobIdObj = JobID.forName(jobid); - String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj); - if (historyFile == null) { - out.println("

Job " + jobid + " not known!

"); - return; - } - String historyUrl = "/jobtaskshistory.jsp?logFile=" + historyFile + - "&status=" + state + "&taskType=" + type; - response.sendRedirect(response.encodeRedirectURL(historyUrl)); - return; - } - // Filtering the reports if some filter is specified - if (!"all".equals(state)) { - List filteredReports = new ArrayList(); - for (int i = 0; i < reports.length; ++i) { - if (("completed".equals(state) && reports[i].getCurrentStatus() == TIPStatus.COMPLETE) - || ("running".equals(state) && reports[i].getCurrentStatus() == TIPStatus.RUNNING) - || ("killed".equals(state) && reports[i].getCurrentStatus() == TIPStatus.KILLED) - || ("pending".equals(state) && reports[i].getCurrentStatus() == TIPStatus.PENDING)) { - filteredReports.add(reports[i]); - } - } - // using filtered reports instead of all the reports - reports = filteredReports.toArray(new TaskReport[0]); - filteredReports = null; - } - report_len = reports.length; - - if (report_len <= start_index) { - out.print("No such tasks"); - } else { - out.print("
"); - out.print("

" + Character.toUpperCase(state.charAt(0)) - + state.substring(1).toLowerCase() + " Tasks

"); - out.print("
"); - out.print(""); - out.print("" + - ""); - if (end_index > report_len){ - end_index = report_len; - } - for (int i = start_index ; i < end_index; i++) { - TaskReport report = reports[i]; - out.print(""); - out.print(""); - out.print(""); - out.println(""); - out.println(""); - String[] diagnostics = report.getDiagnostics(); - out.print(""); - out.println(""); - } - out.print("
TaskCompleteStatusStart TimeFinish TimeErrorsCounters
" + report.getTaskID() + "" + StringUtils.formatPercent(report.getProgress(),2) + - ServletUtil.percentageGraph(report.getProgress() * 100f, 80) + "" + HtmlQuoting.quoteHtmlChars(report.getState()) + "
" + StringUtils.getFormattedTimeWithDiff(dateFormat, report.getStartTime(),0) + "
" + StringUtils.getFormattedTimeWithDiff(dateFormat, - report.getFinishTime(), report.getStartTime()) + "
");
-         for (int j = 0; j < diagnostics.length ; j++) {
-             out.println(HtmlQuoting.quoteHtmlChars(diagnostics[j]));
-         }
-         out.println("

" + - "" + report.getCounters().size() + - "
"); - out.print("
"); - } - if (end_index < report_len) { - out.print(""); - } - if (start_index != 0) { - out.print(""); - } -%> - -
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobtaskshistory.jsp b/hadoop-mapreduce-project/src/webapps/job/jobtaskshistory.jsp deleted file mode 100644 index 7a40fba7107..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobtaskshistory.jsp +++ /dev/null @@ -1,95 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapreduce.TaskAttemptID" - import="org.apache.hadoop.mapreduce.TaskID" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.util.*" - import="java.text.SimpleDateFormat" - import="org.apache.hadoop.mapreduce.jobhistory.*" -%> - -<%! - private static SimpleDateFormat dateFormat = - new SimpleDateFormat("d/MM HH:mm:ss") ; -%> -<%! private static final long serialVersionUID = 1L; -%> - -<% - String logFile = request.getParameter("logFile"); - String taskStatus = request.getParameter("status"); - String taskType = request.getParameter("taskType"); - - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - JobHistoryParser.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, - response, jobTracker, fs, new Path(logFile)); - if (job == null) { - return; - } - Map tasks = job.getAllTasks(); -%> - - - -

<%=taskStatus%> <%=taskType %> task list for <%=job.getJobId() %>

-
- - -<% - for (JobHistoryParser.TaskInfo task : tasks.values()) { - if (taskType.equalsIgnoreCase(task.getTaskType().toString())) { - Map taskAttempts = task.getAllTaskAttempts(); - for (JobHistoryParser.TaskAttemptInfo taskAttempt : taskAttempts.values()) { - if (taskStatus.equals(taskAttempt.getTaskStatus()) || - taskStatus.equalsIgnoreCase("all")){ - printTask(logFile, taskAttempt, out); - } - } - } - } -%> -
Task IdStart TimeFinish Time
Error
-<%! - private void printTask(String logFile, - JobHistoryParser.TaskAttemptInfo attempt, JspWriter out) throws IOException{ - out.print(""); - out.print("" + "" + - attempt.getAttemptId().getTaskID() + ""); - out.print("" + StringUtils.getFormattedTimeWithDiff(dateFormat, - attempt.getStartTime(), 0 ) + ""); - out.print("" + StringUtils.getFormattedTimeWithDiff(dateFormat, - attempt.getFinishTime(), - attempt.getStartTime() ) + ""); - out.print(""+ HtmlQuoting.quoteHtmlChars(attempt.getError()) +""); - out.print(""); - } -%> -
- - diff --git a/hadoop-mapreduce-project/src/webapps/job/jobtracker.jsp b/hadoop-mapreduce-project/src/webapps/job/jobtracker.jsp deleted file mode 100644 index 2b7651813fb..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobtracker.jsp +++ /dev/null @@ -1,185 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.lang.management.MemoryUsage" - import="java.lang.management.MemoryMXBean" - import="java.lang.management.ManagementFactory" - import="java.util.*" - import="java.text.DecimalFormat" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapreduce.*" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - ClusterStatus status = tracker.getClusterStatus(); - MemoryMXBean mem = ManagementFactory.getMemoryMXBean(); - ClusterMetrics metrics = tracker.getClusterMetrics(); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); - JobQueueInfo[] queues = tracker.getJobQueues(); - List runningJobs = tracker.getRunningJobs(); - List completedJobs = tracker.getCompletedJobs(); - List failedJobs = tracker.getFailedJobs(); -%> -<%! - private static DecimalFormat percentFormat = new DecimalFormat("##0.00"); - - public void generateSummaryTable(JspWriter out, ClusterMetrics metrics, - JobTracker tracker) throws IOException { - String tasksPerNode = metrics.getTaskTrackerCount() > 0 ? - percentFormat.format(((double)(metrics.getMapSlotCapacity() + - metrics.getReduceSlotCapacity())) / metrics.getTaskTrackerCount()): - "-"; - out.print("\n"+ - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "\n"); - out.print("
QueuesRunning Map TasksRunning Reduce TasksTotal SubmissionsNodesOccupied Map SlotsOccupied Reduce SlotsReserved Map SlotsReserved Reduce SlotsMap Slot CapacityReduce Slot CapacityAvg. Slots/NodeBlacklisted NodesExcluded Nodes
" + - tracker.getRootQueues().length + "" + - metrics.getRunningMaps() + "" + - metrics.getRunningReduces() + "" + - metrics.getTotalJobSubmissions() + - "" + - metrics.getTaskTrackerCount() + "" + - metrics.getOccupiedMapSlots() + "" + - metrics.getOccupiedReduceSlots() + "" + - metrics.getReservedMapSlots() + "" + - metrics.getReservedReduceSlots() + "" + - + metrics.getMapSlotCapacity() + - "" + metrics.getReduceSlotCapacity() + - "" + tasksPerNode + - "" + - metrics.getBlackListedTaskTrackerCount() + "" + - "" + - metrics.getDecommissionedTaskTrackerCount() + "" + - "
\n"); - - out.print("
"); - if (tracker.recoveryManager.shouldRecover()) { - out.print(""); - if (tracker.hasRecovered()) { - out.print("The JobTracker got restarted and recovered back in " ); - out.print(StringUtils.formatTime(tracker.getRecoveryDuration())); - } else { - out.print("The JobTracker got restarted and is still recovering"); - } - out.print(""); - } - }%> - - - - - -<%= trackerName %> Hadoop Map/Reduce Administration - - - - - -<% if (!JSPUtil.processButtons(request, response, tracker)) { - return;// user is not authorized - } -%> - -

<%= trackerName %> Hadoop Map/Reduce Administration

- - - -State: <%= status.getJobTrackerStatus() %>
-Started: <%= new Date(tracker.getStartTime())%>
-Version: <%= VersionInfo.getVersion()%>, - <%= VersionInfo.getRevision()%>
-Compiled: <%= VersionInfo.getDate()%> by - <%= VersionInfo.getUser()%> from - <%= VersionInfo.getBranch()%>
-Identifier: <%= tracker.getTrackerIdentifier()%>
- -
-

Cluster Summary (Heap Size is - <% MemoryUsage heap = mem.getHeapMemoryUsage(); - out.print(StringUtils.byteDesc(heap.getUsed()) + "/"); - out.print(StringUtils.byteDesc(heap.getCommitted()) + "/"); - out.print(StringUtils.byteDesc(heap.getMax()) + ")"); - %> -<% - generateSummaryTable(out, metrics, tracker); -%> -
-Filter (Jobid, Priority, User, Name)
-Example: 'user:smith 3200' will filter by 'smith' only in the user field and '3200' in all fields -
- -

Running Jobs

-<%=JSPUtil.generateJobTable("Running", runningJobs, 30, 0, tracker.conf)%> -
- -<% -if (completedJobs.size() > 0) { - out.print("

Completed Jobs

"); - out.print(JSPUtil.generateJobTable("Completed", completedJobs, 0, - runningJobs.size(), tracker.conf)); - out.print("
"); -} -%> - -<% -if (failedJobs.size() > 0) { - out.print("

Failed Jobs

"); - out.print(JSPUtil.generateJobTable("Failed", failedJobs, 0, - (runningJobs.size()+completedJobs.size()), tracker.conf)); - out.print("
"); -} -%> - -

Retired Jobs

-<%=JSPUtil.generateRetiredJobTable(tracker, - (runningJobs.size()+completedJobs.size()+failedJobs.size()))%> -
- -

Local Logs

-Log directory, -Job Tracker History - -<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/jobtracker.jspx b/hadoop-mapreduce-project/src/webapps/job/jobtracker.jspx deleted file mode 100644 index be7324de53a..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/jobtracker.jspx +++ /dev/null @@ -1,91 +0,0 @@ - - - - - - - - - - - - - private static final long serialVersionUID = 1L; - - - - response.setHeader("Pragma", "no-cache"); - response.setHeader("Cache-Control", "no-store"); - response.setDateHeader("Expires", -1); - - - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = StringUtils.simpleHostname(tracker.getJobTrackerMachine()); - JobTrackerJspHelper jspHelper = new JobTrackerJspHelper(); - - List<JobInProgress> runningJobs = tracker.getRunningJobs(); - List<JobInProgress> completedJobs = tracker.getCompletedJobs(); - List<JobInProgress> failedJobs = tracker.getFailedJobs(); - - - trackerName - - - tracker.getClusterStatus().getJobTrackerStatus() - new Date(tracker.getStartTime()) - VersionInfo.getVersion() - VersionInfo.getRevision() - VersionInfo.getDate() - VersionInfo.getUser() - tracker.getTrackerIdentifier() - - - - - jspHelper.generateSummaryTable(out, tracker); - - - - - - jspHelper.generateJobTable(out, "running", runningJobs); - - - - - - jspHelper.generateJobTable(out, "completed", completedJobs); - - - - - - jspHelper.generateJobTable(out, "failed", failedJobs); - - - - - - JSPUtil.generateRetiredJobXml(out, tracker, - runningJobs.size() + completedJobs.size() + failedJobs.size()); - - - diff --git a/hadoop-mapreduce-project/src/webapps/job/machines.jsp b/hadoop-mapreduce-project/src/webapps/job/machines.jsp deleted file mode 100644 index e58036a1ec2..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/machines.jsp +++ /dev/null @@ -1,186 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="java.text.DecimalFormat" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); - String type = request.getParameter("type"); -%> -<%! - public void generateTaskTrackerTable(JspWriter out, - String type, - JobTracker tracker) throws IOException { - Collection c; - if (("blacklisted").equals(type)) { - out.println("

Blacklisted Task Trackers

"); - c = tracker.blacklistedTaskTrackers(); - } else if (("active").equals(type)) { - out.println("

Active Task Trackers

"); - c = tracker.activeTaskTrackers(); - } else { - out.println("

Task Trackers

"); - c = tracker.taskTrackers(); - } - int noCols = 9 + - (3 * tracker.getStatistics().collector.DEFAULT_COLLECT_WINDOWS.length); - if(type.equals("blacklisted")) { - noCols = noCols + 1; - } - if (c.size() == 0) { - out.print("There are currently no known " + type + " Task Trackers."); - } else { - out.print("
\n"); - out.print("\n"); - out.print("\n"); - out.print("" + - "" + - "" + - "" + - "" + - "" + - ""); - if(type.equals("blacklisted")) { - out.print(""); - } - for(StatisticsCollector.TimeWindow window : tracker.getStatistics(). - collector.DEFAULT_COLLECT_WINDOWS) { - out.println(""); - out.println(""); - out.println(""); - } - out.print("\n"); - - int maxFailures = 0; - String failureKing = null; - for (Iterator it = c.iterator(); it.hasNext(); ) { - TaskTrackerStatus tt = (TaskTrackerStatus) it.next(); - long sinceHeartbeat = System.currentTimeMillis() - tt.getLastSeen(); - boolean isHealthy = tt.getHealthStatus().isNodeHealthy(); - long sinceHealthCheck = tt.getHealthStatus().getLastReported(); - String healthString = ""; - if(sinceHealthCheck == 0) { - healthString = "N/A"; - } else { - healthString = (isHealthy?"Healthy":"Unhealthy"); - sinceHealthCheck = System.currentTimeMillis() - sinceHealthCheck; - sinceHealthCheck = sinceHealthCheck/1000; - } - if (sinceHeartbeat > 0) { - sinceHeartbeat = sinceHeartbeat / 1000; - } - int numCurTasks = 0; - for (Iterator it2 = tt.getTaskReports().iterator(); it2.hasNext(); ) { - it2.next(); - numCurTasks++; - } - int numFailures = tt.getFailures(); - if (numFailures > maxFailures) { - maxFailures = numFailures; - failureKing = tt.getTrackerName(); - } - out.print("\n"); - } - out.print("
Task Trackers
NameHost# running tasksMax Map TasksMax Reduce TasksFailuresNode Health StatusSeconds Since Node Last HealthyReason For blacklistingTotal Tasks "+window.name+"Succeeded Tasks "+window.name+"Failed Health Checks " - + window.name+"Seconds since heartbeat
"); - out.print(tt.getTrackerName() + ""); - out.print(tt.getHost() + "" + numCurTasks + - "" + tt.getMaxMapSlots() + - "" + tt.getMaxReduceSlots() + - "" + numFailures + - "" + healthString + - "" + sinceHealthCheck); - if(type.equals("blacklisted")) { - out.print("" + tracker.getFaultReport(tt.getHost())); - } - for(StatisticsCollector.TimeWindow window : tracker.getStatistics(). - collector.DEFAULT_COLLECT_WINDOWS) { - JobTrackerStatistics.TaskTrackerStat ttStat = tracker.getStatistics(). - getTaskTrackerStat(tt.getTrackerName()); - out.println("" + ttStat.totalTasksStat.getValues(). - get(window).getValue()); - out.println("" + ttStat.succeededTasksStat.getValues(). - get(window).getValue()); - out.println("" + ttStat.healthCheckFailedStat. - getValues().get(window).getValue()); - } - - out.print("" + sinceHeartbeat + "
\n"); - out.print("
\n"); - if (maxFailures > 0) { - out.print("Highest Failures: " + failureKing + " with " + maxFailures + - " failures
\n"); - } - } - } - - public void generateTableForExcludedNodes(JspWriter out, JobTracker tracker) - throws IOException { - // excluded nodes - out.println("

Excluded Nodes

"); - Collection d = tracker.getExcludedNodes(); - if (d.size() == 0) { - out.print("There are currently no excluded hosts."); - } else { - out.print("
\n"); - out.print("\n"); - out.print(""); - out.print("\n"); - for (Iterator it = d.iterator(); it.hasNext(); ) { - String dt = (String)it.next(); - out.print("\n"); - } - out.print("
Host Name
" + dt + "
\n"); - out.print("
\n"); - } - } -%> - - - - -<%=trackerName%> Hadoop Machine List - - -

<%=trackerName%> Hadoop Machine List

- -<% - if (("excluded").equals(type)) { - generateTableForExcludedNodes(out, tracker); - } else { - generateTaskTrackerTable(out, type, tracker); - } -%> - -<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/queueinfo.jsp b/hadoop-mapreduce-project/src/webapps/job/queueinfo.jsp deleted file mode 100644 index ff4376d52fe..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/queueinfo.jsp +++ /dev/null @@ -1,145 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%!private static final long serialVersionUID = 1L;%> -<%@ page -contentType="text/html; charset=UTF-8" -import="org.apache.hadoop.mapred.*" -import="org.apache.commons.lang.StringUtils" -import="javax.servlet.*" -import="javax.servlet.http.*" -import="java.io.*" -import="java.util.*" -%> - - - - -Job Queue Information page - -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - QueueManager qmgr = tracker.getQueueManager(); - JobQueueInfo[] rootQueues = qmgr.getRootQueues(); -%> -<%! - public static String getTree(String parent, JobQueueInfo[] rootQueues) { - List rootQueueList = new ArrayList(); - for (JobQueueInfo queue : rootQueues) { - rootQueueList.add(queue); - } - return getTree(parent, rootQueueList); - } - - private static String getTree(String parent, List children) { - StringBuilder str = new StringBuilder(); - if (children == null) { - return ""; - } - for (JobQueueInfo queueInfo : children) { - String variableName = StringUtils.replaceChars(queueInfo.getQueueName(), - ":-*+#.^", "_______"); - String label = queueInfo.getQueueName().split(":")[queueInfo - .getQueueName().split(":").length - 1]; - str.append(String.format( - "var %sTreeNode = new YAHOO.widget.MenuNode(\"%s\", %s, false);\n", - variableName, label, parent)); - str.append(String.format("%sTreeNode.data=\"%s\";\n", variableName, - queueInfo.getSchedulingInfo().replaceAll("\n", "
"))); - str.append(String.format("%sTreeNode.name=\"%s\";\n", variableName, - queueInfo.getQueueName())); - str.append(getTree(variableName + "TreeNode", queueInfo.getChildren())); - } - return str.toString(); - } -%> - - - - - - - -
-
-
- - - - diff --git a/hadoop-mapreduce-project/src/webapps/job/queuetable.jsp b/hadoop-mapreduce-project/src/webapps/job/queuetable.jsp deleted file mode 100644 index 8924e7a8b05..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/queuetable.jsp +++ /dev/null @@ -1,78 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%! -private static final long serialVersionUID = 1L; -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="java.text.DecimalFormat" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.*" - import="org.apache.hadoop.util.ServletUtil" -%> - - - - -<% -JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); -JobQueueInfo[] queues = tracker.getRootJobQueues(); -%> -Queue Information - - -

Scheduling Information

- - - - - - - - -<% -for(JobQueueInfo queue: queues) { - String queueName = queue.getQueueName(); - String state = queue.getQueueState(); - String schedulingInformation = queue.getSchedulingInfo(); - if(schedulingInformation == null || schedulingInformation.trim().equals("")) { - schedulingInformation = "NA"; - } -%> - - - - -<% -} -%> - -
Queue Name Scheduling Information
<%=queueName%> - -<%=HtmlQuoting.quoteHtmlChars(schedulingInformation).replaceAll("\n","
")%> -
-

Job Tracker

 - - diff --git a/hadoop-mapreduce-project/src/webapps/job/taskdetails.jsp b/hadoop-mapreduce-project/src/webapps/job/taskdetails.jsp deleted file mode 100644 index dc557e47a37..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/taskdetails.jsp +++ /dev/null @@ -1,374 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.lang.String" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.util.*" - import="java.text.SimpleDateFormat" - import="org.apache.hadoop.security.UserGroupInformation" - import="java.security.PrivilegedExceptionAction" - import="org.apache.hadoop.security.AccessControlException" -%> -<%!static SimpleDateFormat dateFormat = new SimpleDateFormat( - "d-MMM-yyyy HH:mm:ss"); - -%> -<%! private static final long serialVersionUID = 1L; -%> -<%!private void printConfirm(JspWriter out, - String attemptid, String action) throws IOException { - String url = "taskdetails.jsp?attemptid=" + attemptid; - out.print("" + "

Are you sure you want to kill/fail " - + attemptid + " ?


" - + "
" - + "" - + "" - + "
" - + "
"); - }%> -<% - final JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - - String attemptid = request.getParameter("attemptid"); - final TaskAttemptID attemptidObj = TaskAttemptID.forName(attemptid); - - // Obtain tipid for attemptid, if attemptid is available. - TaskID tipidObj = - (attemptidObj == null) ? TaskID.forName(request.getParameter("tipid")) - : attemptidObj.getTaskID(); - if (tipidObj == null) { - out.print("tipid sent is not valid.
\n"); - return; - } - // Obtain jobid from tipid - final JobID jobidObj = tipidObj.getJobID(); - String jobid = jobidObj.toString(); - - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, jobidObj, - request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - // redirect to history page if it cannot be found in memory - if (job == null) { - String historyFile = tracker.getJobHistory().getHistoryFilePath(jobidObj); - if (historyFile == null) { - out.println("

Job " + jobid + " not known!

"); - return; - } - String historyUrl = "/taskdetailshistory.jsp?logFile=" + historyFile + - "&tipid=" + tipidObj.toString(); - response.sendRedirect(response.encodeRedirectURL(historyUrl)); - return; - } - boolean privateActions = JSPUtil.privateActionsAllowed(tracker.conf); - if (privateActions) { - String action = request.getParameter("action"); - if (action != null) { - String user = request.getRemoteUser(); - UserGroupInformation ugi = null; - if (user != null) { - ugi = UserGroupInformation.createRemoteUser(user); - } - if (action.equalsIgnoreCase("confirm")) { - String subAction = request.getParameter("subaction"); - if (subAction == null) - subAction = "fail-task"; - printConfirm(out, attemptid, subAction); - return; - } - else if (action.equalsIgnoreCase("kill-task") - && request.getMethod().equalsIgnoreCase("POST")) { - if (ugi != null) { - try { - ugi.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException{ - - tracker.killTask(attemptidObj, false);// checks job modify permission - return null; - } - }); - } catch(AccessControlException e) { - String errMsg = "User " + user + " failed to kill task " - + attemptidObj + "!

" + e.getMessage() + - "
Go back to Job
"; - JSPUtil.setErrorAndForward(errMsg, request, response); - return; - } - } else {// no authorization needed - tracker.killTask(attemptidObj, false); - } - - //redirect again so that refreshing the page will not attempt to rekill the task - response.sendRedirect("/taskdetails.jsp?subaction=kill-task" + - "&tipid=" + tipidObj.toString()); - } - else if (action.equalsIgnoreCase("fail-task") - && request.getMethod().equalsIgnoreCase("POST")) { - if (ugi != null) { - try { - ugi.doAs(new PrivilegedExceptionAction() { - public Void run() throws IOException{ - - tracker.killTask(attemptidObj, true);// checks job modify permission - return null; - } - }); - } catch(AccessControlException e) { - String errMsg = "User " + user + " failed to fail task " - + attemptidObj + "!

" + e.getMessage() + - "
Go back to Job
"; - JSPUtil.setErrorAndForward(errMsg, request, response); - return; - } - } else {// no authorization needed - tracker.killTask(attemptidObj, true); - } - - response.sendRedirect("/taskdetails.jsp?subaction=fail-task" + - "&tipid=" + tipidObj.toString()); - } - } - } - TaskInProgress tip = job.getTaskInProgress(tipidObj); - TaskStatus[] ts = null; - boolean isCleanupOrSetup = false; - if (tip != null) { - ts = tip.getTaskStatuses(); - isCleanupOrSetup = tip.isJobCleanupTask(); - if (!isCleanupOrSetup) { - isCleanupOrSetup = tip.isJobSetupTask(); - } - } -%> - - - - - - - Hadoop Task Details - - -

Job <%=jobid%>

- -
- -

All Task Attempts

-
-<% - if (ts == null || ts.length == 0) { -%> -

No Task Attempts found

-<% - } else { -%> - - - <% - if (ts[0].getIsMap()) { - %> - - <% - } - else if(!isCleanupOrSetup) { - %> - - <% - } - %> - - <% - for (int i = 0; i < ts.length; i++) { - TaskStatus status = ts[i]; - String taskTrackerName = status.getTaskTracker(); - TaskTrackerStatus taskTracker = tracker.getTaskTrackerStatus(taskTrackerName); - out.print(""); - String taskAttemptTracker = null; - String cleanupTrackerName = null; - TaskTrackerStatus cleanupTracker = null; - String cleanupAttemptTracker = null; - boolean hasCleanupAttempt = false; - if (tip != null && tip.isCleanupAttempt(status.getTaskID())) { - cleanupTrackerName = tip.machineWhereCleanupRan(status.getTaskID()); - cleanupTracker = tracker.getTaskTrackerStatus(cleanupTrackerName); - if (cleanupTracker != null) { - cleanupAttemptTracker = "http://" + cleanupTracker.getHost() + ":" - + cleanupTracker.getHttpPort(); - } - hasCleanupAttempt = true; - } - out.print(""); - out.print(""); - out.print(""); - out.print(""); - if (ts[i].getIsMap()) { - out.print(""); - } - else if (!isCleanupOrSetup) { - out.print(""); - out.println(""); - } - out.println(""); - - out.print(""); - out.print(""); - out.print(""); - } - %> -
Task AttemptsMachineStatusProgressStart TimeMap Phase FinishedShuffle FinishedSort FinishedFinish TimeErrorsTask LogsCountersActions
" + status.getTaskID() + ""); - if (hasCleanupAttempt) { - out.print("Task attempt: "); - } - if (taskTracker == null) { - out.print(taskTrackerName); - } else { - taskAttemptTracker = "http://" + taskTracker.getHost() + ":" - + taskTracker.getHttpPort(); - out.print("" - + tracker.getNode(taskTracker.getHost()) + ""); - } - if (hasCleanupAttempt) { - out.print("
Cleanup Attempt: "); - if (cleanupAttemptTracker == null ) { - out.print(cleanupTrackerName); - } else { - out.print("" - + tracker.getNode(cleanupTracker.getHost()) + ""); - } - } - out.print("		" + status.getRunState() + "" + StringUtils.formatPercent(status.getProgress(), 2) - + ServletUtil.percentageGraph(status.getProgress() * 100f, 80) + "" - + StringUtils.getFormattedTimeWithDiff(dateFormat, status - .getStartTime(), 0) + "" - + StringUtils.getFormattedTimeWithDiff(dateFormat, status - .getMapFinishTime(), status.getStartTime()) + "" - + StringUtils.getFormattedTimeWithDiff(dateFormat, status - .getShuffleFinishTime(), status.getStartTime()) + "" - + StringUtils.getFormattedTimeWithDiff(dateFormat, status - .getSortFinishTime(), status.getShuffleFinishTime()) - + "" - + StringUtils.getFormattedTimeWithDiff(dateFormat, status - .getFinishTime(), status.getStartTime()) + "
");
-        String [] failures = tracker.getTaskDiagnostics(status.getTaskID());
-        if (failures == null) {
-          out.print(" ");
-        } else {
-          for(int j = 0 ; j < failures.length ; j++){
-            out.print(HtmlQuoting.quoteHtmlChars(failures[j]));
-            if (j < (failures.length - 1)) {
-              out.print("\n-------\n");
-            }
-          }
-        }
-        out.print("
"); - String taskLogUrl = null; - if (taskTracker != null ) { - taskLogUrl = TaskLogServlet.getTaskLogUrl(taskTracker.getHost(), - String.valueOf(taskTracker.getHttpPort()), - status.getTaskID().toString()); - } - if (hasCleanupAttempt) { - out.print("Task attempt:
"); - } - if (taskLogUrl == null) { - out.print("n/a"); - } else { - String tailFourKBUrl = taskLogUrl + "&start=-4097"; - String tailEightKBUrl = taskLogUrl + "&start=-8193"; - String entireLogUrl = taskLogUrl + "&all=true"; - out.print("Last 4KB
"); - out.print("Last 8KB
"); - out.print("All
"); - } - if (hasCleanupAttempt) { - out.print("Cleanup attempt:
"); - taskLogUrl = null; - if (cleanupTracker != null ) { - taskLogUrl = TaskLogServlet.getTaskLogUrl(cleanupTracker.getHost(), - String.valueOf(cleanupTracker.getHttpPort()), - status.getTaskID().toString()); - } - if (taskLogUrl == null) { - out.print("n/a"); - } else { - String tailFourKBUrl = taskLogUrl + "&start=-4097&cleanup=true"; - String tailEightKBUrl = taskLogUrl + "&start=-8193&cleanup=true"; - String entireLogUrl = taskLogUrl + "&all=true&cleanup=true"; - out.print("Last 4KB
"); - out.print("Last 8KB
"); - out.print("All
"); - } - } - out.print("		" + "" - + ((status.getCounters() != null) ? status.getCounters().size() : 0) - + ""); - if (privateActions - && status.getRunState() == TaskStatus.State.RUNNING) { - out.print(" Kill "); - out.print("
Fail "); - } - else - out.print("
 
"); - out.println("
-
- -<% - if (ts[0].getIsMap() && !isCleanupOrSetup) { -%> -

Input Split Locations

- -<% - for (String split: StringUtils.split(tracker.getTip( - tipidObj).getSplitNodes())) { - out.println(""); - } -%> -
" + split + "
-<% - } - } -%> - -
-Go back to the job
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/taskdetailshistory.jsp b/hadoop-mapreduce-project/src/webapps/job/taskdetailshistory.jsp deleted file mode 100644 index bdacd8ad1f3..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/taskdetailshistory.jsp +++ /dev/null @@ -1,145 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.util.*" - import="java.text.SimpleDateFormat" - import="org.apache.hadoop.mapreduce.TaskType" - import="org.apache.hadoop.mapreduce.Counters" - import="org.apache.hadoop.mapreduce.TaskID" - import="org.apache.hadoop.mapreduce.TaskAttemptID" - import="org.apache.hadoop.mapreduce.jobhistory.*" -%> - -<%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ; %> -<%! private static final long serialVersionUID = 1L; -%> - -<% - String logFile = request.getParameter("logFile"); - String tipid = request.getParameter("tipid"); - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - JobHistoryParser.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, - response, jobTracker, fs, new Path(logFile)); - if (job == null) { - return; - } - JobHistoryParser.TaskInfo task = job.getAllTasks().get(TaskID.forName(tipid)); - TaskType type = task.getTaskType(); -%> - - - -

<%=tipid %> attempts for <%=job.getJobId() %>

-
- - -<% - if (TaskType.REDUCE.equals(type)) { -%> - -<% - } -%> - - - -<% - for (JobHistoryParser.TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) { - printTaskAttempt(attempt, type, out, logFile); - } -%> -
Task IdStart TimeShuffle FinishedSort FinishedFinish TimeHostErrorTask LogsCounters
-
-<% - if (TaskType.MAP.equals(type)) { -%> -

Input Split Locations

- -<% - for (String split : StringUtils.split(task.getSplitLocations())) - { - out.println(""); - } -%> -
" + split + "
-<% - } -%> -<%! - private void printTaskAttempt(JobHistoryParser.TaskAttemptInfo taskAttempt, - TaskType type, JspWriter out, String logFile) - throws IOException { - out.print(""); - out.print("" + taskAttempt.getAttemptId() + ""); - out.print("" + StringUtils.getFormattedTimeWithDiff(dateFormat, - taskAttempt.getStartTime(), 0 ) + ""); - if (TaskType.REDUCE.equals(type)) { - out.print("" + - StringUtils.getFormattedTimeWithDiff(dateFormat, - taskAttempt.getShuffleFinishTime(), - taskAttempt.getStartTime()) + ""); - out.print("" + StringUtils.getFormattedTimeWithDiff(dateFormat, - taskAttempt.getSortFinishTime(), - taskAttempt.getShuffleFinishTime()) + ""); - } - out.print(""+ StringUtils.getFormattedTimeWithDiff(dateFormat, - taskAttempt.getFinishTime(), - taskAttempt.getStartTime()) + ""); - out.print("" + taskAttempt.getHostname() + ""); - out.print("" + HtmlQuoting.quoteHtmlChars(taskAttempt.getError()) + - ""); - - // Print task log urls - out.print(""); - String taskLogsUrl = HistoryViewer.getTaskLogsUrl(taskAttempt); - if (taskLogsUrl != null) { - String tailFourKBUrl = taskLogsUrl + "&start=-4097"; - String tailEightKBUrl = taskLogsUrl + "&start=-8193"; - String entireLogUrl = taskLogsUrl + "&all=true"; - out.print("Last 4KB
"); - out.print("Last 8KB
"); - out.print("All
"); - } else { - out.print("n/a"); - } - out.print(""); - Counters counters = taskAttempt.getCounters(); - if (counters != null) { - TaskAttemptID attemptId = taskAttempt.getAttemptId(); - out.print("" - + "" - + counters.countCounters() + ""); - } else { - out.print(""); - } - out.print(""); - } -%> - - diff --git a/hadoop-mapreduce-project/src/webapps/job/taskstats.jsp b/hadoop-mapreduce-project/src/webapps/job/taskstats.jsp deleted file mode 100644 index e5bf0b72db3..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/taskstats.jsp +++ /dev/null @@ -1,135 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.lang.String" - import="java.text.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.mapred.JSPUtil.JobWithViewAccessCheck" - import="org.apache.hadoop.util.*" - import="java.text.SimpleDateFormat" -%> -<%! private static final long serialVersionUID = 1L; -%> -<% - JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); - String trackerName = - StringUtils.simpleHostname(tracker.getJobTrackerMachine()); - - String attemptid = request.getParameter("attemptid"); - TaskAttemptID attemptidObj = TaskAttemptID.forName(attemptid); - // Obtain tipid for attemptId, if attemptId is available. - TaskID tipidObj = - (attemptidObj == null) ? TaskID.forName(request.getParameter("tipid")) - : attemptidObj.getTaskID(); - // Obtain jobid from tipid - JobID jobidObj = tipidObj.getJobID(); - String jobid = jobidObj.toString(); - - JobWithViewAccessCheck myJob = JSPUtil.checkAccessAndGetJob(tracker, jobidObj, - request, response); - if (!myJob.isViewJobAllowed()) { - return; // user is not authorized to view this job - } - - JobInProgress job = myJob.getJob(); - // redirect to history page if it cannot be found in memory - if (job == null) { - JobID jobIdObj = JobID.forName(jobid); - String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj); - if (historyFile == null) { - out.println("

Job " + jobid + " not known!

"); - return; - } - String historyUrl = "/taskstatshistory.jsp?logFile=" + historyFile + - "&attemptid=" + attemptid; - response.sendRedirect(response.encodeRedirectURL(historyUrl)); - return; - } - - Format decimal = new DecimalFormat(); - Counters counters; - if (attemptid == null) { - counters = tracker.getTipCounters(tipidObj); - attemptid = tipidObj.toString(); // for page title etc - } - else { - TaskStatus taskStatus = tracker.getTaskStatus(attemptidObj); - counters = taskStatus.getCounters(); - } -%> - - - - - Counters for <%=attemptid%> - - -

Counters for <%=attemptid%>

- -
- -<% - if ( counters == null ) { -%> -

No counter information found for this task

-<% - } else { -%> - -<% - for (String groupName : counters.getGroupNames()) { - Counters.Group group = counters.getGroup(groupName); - String displayGroupName = group.getDisplayName(); -%> - - - -<% - for (Counters.Counter counter : group) { - String displayCounterName = counter.getDisplayName(); - long value = counter.getCounter(); -%> - - - - - -<% - } - } -%> -

- <%=HtmlQuoting.quoteHtmlChars(displayGroupName)%>
<%=HtmlQuoting.quoteHtmlChars(displayCounterName)%><%=decimal.format(value)%>
-<% - } -%> - -
-Go back to the job
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/job/taskstatshistory.jsp b/hadoop-mapreduce-project/src/webapps/job/taskstatshistory.jsp deleted file mode 100644 index d4cad15f5ee..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/taskstatshistory.jsp +++ /dev/null @@ -1,122 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.util.*" - import="java.text.*" - import="org.apache.hadoop.mapreduce.jobhistory.*" - import="org.apache.hadoop.mapreduce.TaskID" - import="org.apache.hadoop.mapreduce.TaskAttemptID" - import="org.apache.hadoop.mapreduce.Counter" - import="org.apache.hadoop.mapreduce.Counters" - import="org.apache.hadoop.mapreduce.CounterGroup" -%> -<%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ; - private static final long serialVersionUID = 1L; -%> - -<% - String attemptid = request.getParameter("attemptid"); - if(attemptid == null) { - out.println("No attemptid found! Pass a 'attemptid' parameter in the request."); - return; - } - TaskID tipid = TaskAttemptID.forName(attemptid).getTaskID(); - String logFile = request.getParameter("logFile"); - - Format decimal = new DecimalFormat(); - - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - JobHistoryParser.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, - response, jobTracker, fs, new Path(logFile)); - if (job == null) { - return; - } - - Map tasks = job.getAllTasks(); - JobHistoryParser.TaskInfo task = tasks.get(tipid); - - Map attempts = task.getAllTaskAttempts(); - JobHistoryParser.TaskAttemptInfo attempt = attempts.get(TaskAttemptID.forName(attemptid)); - - Counters counters = attempt.getCounters(); -%> - - - - - Counters for <%=attemptid%> - - -

Counters for <%=attemptid%>

- -
- -<% - if (counters == null) { -%> -

No counter information found for this attempt

-<% - } else { -%> - -<% - for (String groupName : counters.getGroupNames()) { - CounterGroup group = counters.getGroup(groupName); - String displayGroupName = group.getDisplayName(); -%> - - - -<% - Iterator ctrItr = group.iterator(); - while(ctrItr.hasNext()) { - Counter counter = ctrItr.next(); - String displayCounterName = counter.getDisplayName(); - long value = counter.getValue(); -%> - - - - - -<% - } - } -%> -

- <%=HtmlQuoting.quoteHtmlChars(displayGroupName)%>
<%=HtmlQuoting.quoteHtmlChars(displayCounterName)%><%=decimal.format(value)%>
-<% - } -%> - -
-Go back to the job
-Go back to JobTracker
-<% -out.println(ServletUtil.htmlFooter()); -%> diff --git a/hadoop-mapreduce-project/src/webapps/static/hadoop-logo.jpg b/hadoop-mapreduce-project/src/webapps/static/hadoop-logo.jpg deleted file mode 100644 index 809525d9f158f6163c7692b5e317153ae68bf23f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9443 zcmbVxWmFv9vUU$H!6AX*7GMY@L4yZLuwVlTGLYad8Qcj3LIS}DcOBeaLvRR?;4rwm zyMB}Rob#S@@1O79yLHfSiDX8@`P$Wf;~i05zl zbD#x)@y}-yP-8pF5*6_;uYZk_#&(7#465dKj&=^_b{`l7xt{_&PoD}1KjjtX;bB1a z&-jI(3IPCm8t5OV2)6lRBay)+ejE|3xo=r&XIgcneFCWhzCm>v0 z+-8J|gFk*OJAGr`Dtd?V6vn5D{E zDImZ1S$GZY{c!Q_KcJ*~$ohzlode9rFCh3_NLX6tM0!~(E;BD=pTf}oH{N{~z6I8UKhXX{ z_CEvW_kTk6H?V)>ngZ^jgHW4?P69{(CwbSh4!nH?MYUDNbyv?KckJAy&#lP~t zIfs~$AFDq7t%p>xLL?;?Di}p-qL)CXc}iH02g@Xw!$Qr1+dU`|lx&$B!+P{H zYwP_BJrwqpw|oa!vAMDad;+miVui8y0EVV2g=+z=B zRp!A0-4ekr#u?cHoXD1L-W{78Z|p5U#%71eElMpDLnZipJhr9`

*R*qiz_!A~?ZS89anRimGxt(}8A$LVLM=d**S}itKT4vq9r3n4X zFxE`i<_oF9o-pls&UTgZag1+{csUH}dIwa@I7aZP$$l^l@8Moy{ql|bg)Yf(O}4{R zS1YLBl*r3p#xf+`RB1u%OVjN}%SZC-IHI@OIXdU+y47SIH&M>s!>sLlOj&Zdha?|! zS>`dPvH2&r%@Hk&-)KFh-xJN>0VN~HxT0=pQlLny4vYY*($q+I(4kYtdpdLCTnpAF zRV{Xj$)giC^m8$}Vk`;#t?Kbu^{&k6Pj2={F}_SPpUyle-iQt)D|>lYC397n&$7mF zBDJ~J8UwiRagb)PJmvoVk+xhhTKsw_1kc{${lWWTuvWrvrNf=&lRJQUgucri!>S${ z92f57_BiJaJwzQXXrZzM*nHq}hU;a;+W6p^G|hk-iBu;71K3uqn2*{uFJz;9x$dYQ zE&pih`uE-BU&Mupj)BS$RDNJ!VxlPF-X99XCc?+Zy+=YsN=iaRLPB<*<^dV`LkbcS zN=C|uw6yf}^kfg1SeWQoXz1wa{vs~aqu5y3gg7{abmS!DbpPx258{#qJ^nAmMT=rr z6qo;<-6-lpaTbcX&_KAT{8uFg{;(E0Iu9VP@+P3%j(2i9uo!xg-k|br$L8*R;0J`ZJN1MsgshLb|;0vhKGHhng3d}Zg%=m02VPW zJBHJYEZlKx9Fg4jrYL?#WrW(uzHCEo*qUE{^-I-LmKj8)N3y06hcUYo#?ce+tmI9S zTqT7ln@?_IEOU0T9*uDBnZU`_ytq=ef$)#3y>dE3*=ZadWNYkz0LJLj7pg)e#JJs? z5%Q|P$==!U^q4|ue6I+6GK|AMCG#R#Zfcl{1{M|dX)<*5k+A(Pn4C|A*yngWF-Frf ze%4gErI$Nu``(q3&yZ{O;#%Cq!!2;DlTQ`g-!-2vz^&c)4lr$M-71^2 zOjK?>)jS{5`b`tTIu^WJ;Qe;OoYoIsq7WvNMc6<$FIXbkK~P<+GZ3ejyGBN(SW%4I z&Ac!=6Tmmzx$up97d>w2qEmGAh&G9(Q;i*lM^junf8JmFX%wH1;l}8LWQli$cada; zerR3FlHj>kv1&Gf|JS65-3FV>2Z@ksE#@@U2}Fx`^hC4OR-LV&nVFJGr0yy!p`NM2 z7er!Z%5EEco6gJXi1edJ4V%G;GpC>e2e;#+inmw?gUD1<{>{!ep5a2FlR8Y%TIt-&4J`8`7E~~xVVt%IooFHiWb1Bf<*5!{Gpb4Sou9!?3u+dNX`gaboNnfsr!@i%s&1~0oX4*!}=BS=N zYK3&m>wJU+Y1RVU-~wBzFY+u?&(cWJ$5R@G^kLdniCQ2f>q{f6Pz0}+Vf{1s+?0h% zeV&nrQx#wfX~XVUfM9p2vh1`d^*@EGUnI{y$~Gpxj+y!)*KXY_3pOnG$a$W!JK26B zcOw(AC{M5Z5|?lTo{=-_Jrxg2YOEeu8AT2qzWx{{zl;rApUnZc=wwycys0`(4HHA} z#_$T#_@#Wk^qqmtsKrH9q#HYN?6}&(=1@aWR;;6lRv#3p0z1i%n*dz#`vusC4>>gB zLm0=2mz28LO2E_@cfc+*lN2ItKHu-gTqg~GOaL#eif9;zWCeyzvN5y2$DnY+NykNhLNn&_#RMi#>6D|1E0#4)~i(h}HvhFIk*E?4&auW6flx8Zd# ztvs#rjivLVDuQ%DZcUC6d_cX^a)Gxc8TylwWI-15ahC;`0Pe2B3-n#`H9b}*eDhYu zw6(80gM*B2rRdc$TuVW-(>7XPg(`hlVwNc`N21y7c2~|c#$P)sVkaX`<VKtLY7;Ya$lZZDO1tK@9tQf({y?Y-)r8kdg*)|RW9ILXW|h)Bf=*Ju)lh)2yg{LJSVJ}a+J-@ISG z4C70+`E`rD6HONhrIn+qvIa_Kt zQ?q)cvwe`9WVTb?e8W#5!~Db+&GP|RC(yd{IyE+EhP=?qqy|Rx2#>CQi|iK*yRDWb zm~Q2S!aAa5wDx4&W!}s2(mwv0pMD^Y%gNZxsFh_lSF-bH-7}h^(^kfwn~8_Gw|BCJ zqru??{ryO^t|^Mq4#QtX5=x;!-6XB^hi`N6lD>B;yh)jH+Yd^&nqC_2AAN9nYQ@xK zN-&+Qdmx~%!4xg0=8)`>;^nJ0S3Jw_v?{3L`;4<`=>r8GSqZ|2f)&Ge53R^%?B(I#d+UvdK_dOFrld24c5 z<;#Y4Au?_HUG^KsNwm2+?OF&l4Ii5E&`ERa#6cn>m673dUnZ_B4E@qtTq;V)haIv= z_5?b4Lr*IIy#$ok!uAKK^dLKzmI!=(s`q|*U*X@;gO;x4+a)8&-V{~5lhr7_DT1dD zs)@xT?|_+6_F-j?ZSJLlki5s>{wPWKvi3Jw#+4CX3Oy_mn=93DtwVx4@f{UoCrX7(2i2-xAIg5<}nXZ;Pf$Z)dB#tXTdNEX| zZz?o74GMc{6IVCGMK1hpKx5K!J-ZLj=oE&c5q4I2xA&~tpZaYX+F3wrIBM!x!dNZlVXHJiG2KYoZo&%w^11B=JB#mtMmBqj2|+z*+9mWrg;hP# z*D8e8{sF~YfC+0h{5ezSL@gwJ6wIoRzFA!DljSyMZXMcJCYS=%`bJ^dBJyE@ zhR|MIer4IlR)2f@*&@-EPKe`y1XGlA-E(AQyCYqzzrch|&coGBReu6sF?p7h%TKT! zZ`-c6<3x4cT)kBX;vx9)B!{MThIlv8RMA-Mxs{a>@^L^{iE%{O=NH&`yajZQ`1Izg zsc~CBIh*?1SogPTaJfkq;VJKPG08#e8#;;V_IBOZbSn{9s)vh=Vu^|(m_-V>=!nA6 z2dz#8DNEtAV`19~mYkNi2Vm7Z08Q+~Vhckud~RQqW*geH{P;q_Tf@0=4Qoyfdr~iF z7L&EI=@U(Zu-|WP=>wOyQm*&V7a><7dWCUWjVWsg0lCxehT864dk$9AT1UBcTuq!rs`(dJyPwO znzF4jP4gCaYSq&*?fK4M?|Popbg=aYQ;~aOqMl2t0=2<@cu^qZ1zG9Q< z7k!AO?~9-VACHM7Iwx~=uZmR9zF6Xz>rf`_mv9d-YgB(qtr^Wa&lY>R`xOSwg&Ek0=R|unFg_ta`?ABR{9LxsI@NZRuglt5^zF~M`(I!? z`*E)}R{*=x&FtJ0wjm|xAo;Kc8TIK4%Bu@7p4-O8#>ePHlP(6chaZl#*Dl*l=Xedg zm*xgi$Jhie67kRIZk^ndrCUJ^tIh;nw+!RgBi9S_{yP^Y`VyC)dZ1u^H- zhgZ~GRi=7rMFT@GxCO1d*u}g_8jD517q2_zEHFHzHmyV0oPRhB*19cd%FJVw^XVI* zH|}4UxF^KN7LZiR=qkDwyFlU9^T_mU-KL5@5kTeRt-kA?8m&wZe;d%5)Lov0*NQhC zsQUYZE@XjIZ32MT%AdJvR&fjUxTX&$IdR~dj>PWqxZ9HT5}B&(R7<$G_+%yAQguAj zH6Wz3aL`Lw1l_!i=uW;VPIXe#E#~D(D;LzqtkB`oID}NoLnAe;q?`;da^3E&lNAvg zbbNhK&HePPytXQ?ih7OOPp@d^@#ws39xmT1^5Pq(V4E4S1x|-}*)!D=FUfQMN}O}A z#pA5)x(d=CDpv$-K1Z;3n~5g=dR3XE6`rO$5iB!vDp|)S726QFWiNL3ua5Z6{ZmbYLUdFGV}RI&zytft6?DE?+q7DlBLZM%V+hDj6%W-MDiZcCJ`uCv`;R zt?45Z^u8_i(G){>ta;9E^H2t*XpQ_iQn^=543^oZu61=vJFB2^u(jSm&01s-tO zdGfXD{_<{`mCdTB{aZW1stoZtM&&~R3741}G<7-*0RC~rwci@sPc;Jd!<|z^?DgXSnn)E&V)5rtI!oYAwE@NXz>~qk#T?3W65j%29PHsj467~YSqAPc*K7kk+JiJ;$O053qgh#1o=UTBi_tRB6 zKXNF}jAaQ=qa)P(Jo%F`-9EjKO;a{u$(cnoKOrJ>R#0kbsl@LMEvIF{naXGpvm~69lnAWLa>87znV2Em|3theOKVMQ@^j@Y zjn6X%L+Pt|3;}4up0<~jK3VjP0!4(j15@4hC-x=*zKckYdY^WGk1g?iJ@ZGV(q1~w zI5xQ1}V67FWOa_ERn;l1Zs=NL=wO7B-*y^!i+J(w==!7|R~h!OE`h;9m=az;wmpc!_a3P?Wv{vV6tFS z;kJbKXqO*+1N^hSUv0Zg)lxp!T@a;P$?r1P+NhSbD@&rY2|#C=Nn)A%o|#mSMCwXK z7QX@9{UotRocD@U(qLM{yA<0MDKD&FmesjL^bh9`fGXwp+C>qti!v5Xk-t&aLpGo-P zw}$Ju?Q_g@5GMGHbFN8ua%WP227Khof=)_S%sf^a)hmWDP%kEa4DLF2?a+pm2Ik0W zoDt4ZSL-CnOO&eZ7T_PJ{u(%W^ZQH7NYuQT7kRKw>C3f=AV$d)uGj|8oZb6w*9~6| z6U8l@iO4k|o30=2CED!Z8I7`O+m7IRSQ<$Zv`p=SJ;z2)zCT58y)b!)JW_}GCH5euj5R7;PKwN@j4BSU(W=%O_nFdX|thakp z`z>F$*KSJZvs34>JgHDn{(c^ETJoN$yGnrI9kd9CfxlT{d3Q z{_vL>*vu2RZy8f@HJtsOdY#6aaysF3?-K%XrpdZVi8330)YV7}6N!mwgw;z4y!eid zL6dSzsSXB9n%@KG{FX~djn3~+G8K9(;TL+C4vqVkVisN<<6mTn{d2wPLQ=ED5Ios! z_td(2ZFL|V$;C4Xvw}fcacMW%zQ57tn>kiq5G1SpUf*D}LZMuYL>dMaK2xR%=YAXdBWIG!(l*-XQFV_IP z44$h~=6GrL3r{xSou@#*%nkYvdb>npZKG1Rz+iKNRMQQ9deP?=EgW{KkGAC|Utf!H zhKEVg+>BZnjd_?#7zG$tNZZmjlW~}e;LI5_GV2SUNN8_bWlcP`s}j~j_|bj?o_z!X zo{g6yLMB8%_QsfRBZDvXy2~Ex)d+V49j`?asSM>Z(qAi{jZGIux4RIyf6ALGf%Wd3 z{qzdBEl*Y+JHV8R!4nh zX@vN22e6-4eX9ytGK|6OraqSoFP6N;w7cS8Te{?G=la}3*l*KY`gT#;f{QN_nROT@ zzg?eDQR^^#4}o28B1C9sUo~wPyJqCKOlT{^H-z`%C^EU>DlqL$hmxa8mNm&5Gu6p~ z26;?RyzXT@>{F5pih6D%lfF<<#;{CXW)tbYR~eJGNv)gO*pIM$>R6>d77`0fSW17# ziaS|$mi_KB7RK8aCCXcXHE6x7Bd9 z=T5=aj0IAOw@Mh^k#>9Rz&6=-@-$|lBoob-^}Fst-31;);nw=$F5dHc~BLH$!?f1AdLu316E zT|({L{%1(xr^$~*W|n(q!e-d4J=#6Y?@OFH3t-pD5Uz3$R<5QV0QC=%9(^pMrU~Th z=~h#s-Kp;2RC!ok7fE^;nS3@ufnJ)ow_5S_N%92Sq{z9(^%#uxr~%`O%?B@*)udkb zflwgZINwCknRy}e~Cs#hf5p}*i-sOvtxRl-YCq*u`3P?+~pCb`~qXFInUA1t<} zeH8o=#;I488`CKp1VM{gyXwCg-Sf739(gh2=M~iwpmxvWbBKNEN)(TksZkKWxI?gR z(X^XV!$7eio+8wy#U=Acxz|(@WgDnX1&NQd?Ia_&K_iePUWakw%#oA1u+RI=z**>! z<(0hC6TTx1Oo7TXx+WsGjji0wdW$&W-gpO{itOr2cpf-u0fWw;UOd|;k&gM+e&Y_9 zY&mzlo;aXCidDJozXRGbEVn#&ry{aa^>Pu_GaSe=vhQ>aP-RkjWq((^|{}#cs#plVP zN{D%P55YTtFyk`pM{$nH0acxD)%{UeL$wBmEHhVJ2pi(mQhCiso}K!5YpB#~Ac0+v zRz!}KM?}o{gdj3+-gk?JeD9%uM-5FiG9z3om?Q_vT${}f(0|B%d8L>o-?vL8Ad+OP z^|K&cHs;av#fuMF*1~>-Ga{ik^2NHEvaZox+dok?cP;gUOhiN~*lnELKXm!X`9-*> z;GFH|DOpaVV#?KON?6Ucu+^@k`YS4C<&?UtF0)$o-uL*OD(}J>SpzL52C41u0@k9KFt*p{ zQDPT_sYH`L`LnS@9Fd&}+=(fr?KWB1-c7VcPV`B} zLx+Or^nqkY?=)sOdQ2x{P2&qL;K7zt_llV#HFDsyNPF_=oxU2ShJw0mccHL(O1(KMomh?vNQBY)y(e5_>UN#qe~AOPC31Ah0{VQw=|QO#2nxp4j7tFIzF8T04`Z fcAfm8ha<7VMx5Bv*{CHRB$`C|Pvy(!Zu0*Dl(S0n diff --git a/hadoop-mapreduce-project/src/webapps/static/hadoop.css b/hadoop-mapreduce-project/src/webapps/static/hadoop.css deleted file mode 100644 index 0560cb3075e..00000000000 --- a/hadoop-mapreduce-project/src/webapps/static/hadoop.css +++ /dev/null @@ -1,134 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -body { - background-color : #ffffff; - font-family : sans-serif; -} - -.small { - font-size : smaller; -} - -div#dfsnodetable tr#row1, div#dfstable td#col1 { - font-weight : bolder; -} - -div#dfstable td#col1 { - vertical-align : top; -} - -div#dfstable td#col3 { - text-align : right; -} - -div#dfsnodetable caption { - text-align : left; -} - -div#dfsnodetable a#title { - font-size : larger; - font-weight : bolder; -} - -div#dfsnodetable td, th { - border-bottom-style : none; - padding-bottom : 4px; - padding-top : 4px; -} - -div#dfsnodetable A:link, A:visited { - text-decoration : none; -} - -div#dfsnodetable th.header, th.headerASC, th.headerDSC { - padding-bottom : 8px; - padding-top : 8px; -} -div#dfsnodetable th.header:hover, th.headerASC:hover, th.headerDSC:hover, - td.name:hover { - text-decoration : underline; - cursor : pointer; -} - -div#dfsnodetable td.blocks, td.size, td.pcused, td.adminstate, td.lastcontact { - text-align : right; -} - -div#dfsnodetable .rowNormal .header { - background-color : #ffffff; -} -div#dfsnodetable .rowAlt, .headerASC, .headerDSC { - background-color : lightyellow; -} - -.warning { - font-weight : bolder; - color : red; -} - -div#dfstable table { - white-space : pre; -} - -div#dfsnodetable td, div#dfsnodetable th, div#dfstable td { - padding-left : 10px; - padding-right : 10px; -} - -td.perc_filled { - background-color:#AAAAFF; -} - -td.perc_nonfilled { - background-color:#FFFFFF; -} - -line.taskgraphline { - stroke-width:1;stroke-linecap:round; -} - -#quicklinks { - margin: 0; - padding: 2px 4px; - position: fixed; - top: 0; - right: 0; - text-align: right; - background-color: #eee; - font-weight: bold; -} - -#quicklinks ul { - margin: 0; - padding: 0; - list-style-type: none; - font-weight: normal; -} - -#quicklinks ul { - display: none; -} - -#quicklinks a { - font-size: smaller; - text-decoration: none; -} - -#quicklinks ul a { - text-decoration: underline; -} diff --git a/hadoop-mapreduce-project/src/webapps/static/jobconf.xsl b/hadoop-mapreduce-project/src/webapps/static/jobconf.xsl deleted file mode 100644 index 75363651e81..00000000000 --- a/hadoop-mapreduce-project/src/webapps/static/jobconf.xsl +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - -
namevalue
- - diff --git a/hadoop-mapreduce-project/src/webapps/static/jobtracker.js b/hadoop-mapreduce-project/src/webapps/static/jobtracker.js deleted file mode 100644 index 7da16c1fcc6..00000000000 --- a/hadoop-mapreduce-project/src/webapps/static/jobtracker.js +++ /dev/null @@ -1,151 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -function checkButtonVerbage() -{ - var inputs = document.getElementsByName("jobCheckBox"); - var check = getCheckStatus(inputs); - - setCheckButtonVerbage(! check); -} - -function selectAll() -{ - var inputs = document.getElementsByName("jobCheckBox"); - var check = getCheckStatus(inputs); - - for (var i in inputs) { - if ('jobCheckBox' == inputs[i].name) { - if ( inputs[i].parentNode.parentNode.style.display != 'none') { - inputs[i].checked = ! check; - } - } - } - - setCheckButtonVerbage(check); -} - -function getCheckStatus(inputs) -{ - var check = true; - - for (var i in inputs) { - if ('jobCheckBox' == inputs[i].name) { - if ( inputs[i].parentNode.parentNode.style.display != 'none') { - check = (inputs[i].checked && check); - } - } - } - - return check; -} - - -function setCheckButtonVerbage(check) -{ - var op = document.getElementById("checkEm"); - op.value = check ? "Select All" : "Deselect All"; -} - -function applyfilter() -{ - var cols = ["job","priority","user","name"]; - var nodes = []; - var filters = []; - - for (var i = 0; i < cols.length; ++i) { - nodes[i] = document.getElementById(cols[i] + "_0" ); - } - - var filter = document.getElementById("filter"); - filters = filter.value.split(' '); - - var row = 0; - while ( nodes[0] != null ) { - //default display status - var display = true; - - // for each filter - for (var filter_idx = 0; filter_idx < filters.length; ++filter_idx) { - - // go check each column - if ((getDisplayStatus(nodes, filters[filter_idx], cols)) == 0) { - display = false; - break; - } - } - - // set the display status - nodes[0].parentNode.style.display = display ? '' : 'none'; - - // next row - ++row; - - // next set of controls - for (var i = 0; i < cols.length; ++i) { - nodes[i] = document.getElementById(cols[i] + "_" + row); - } - } // while -} - -function getDisplayStatus(nodes, filter, cols) -{ - var offset = filter.indexOf(':'); - - var search = offset != -1 ? filter.substring(offset + 1).toLowerCase() : filter.toLowerCase(); - - for (var col = 0; col < cols.length; ++col) { - // a column specific filter - if (offset != -1 ) { - var searchCol = filter.substring(0, offset).toLowerCase(); - - if (searchCol == cols[col]) { - // special case jobs to remove unnecessary stuff - return containsIgnoreCase(stripHtml(nodes[col].innerHTML), search); - } - } else if (containsIgnoreCase(stripHtml(nodes[col].innerHTML), filter)) { - return true; - } - } - - return false; -} - -function stripHtml(text) -{ - return text.replace(/<[^>]*>/g,'').replace(/&[^;]*;/g,''); -} - -function containsIgnoreCase(haystack, needle) -{ - return haystack.toLowerCase().indexOf(needle.toLowerCase()) != -1; -} - -function confirmAction() -{ - return confirm("Are you sure?"); -} - -function toggle(id) -{ - if ( document.getElementById(id).style.display != 'block') { - document.getElementById(id).style.display = 'block'; - } - else { - document.getElementById(id).style.display = 'none'; - } -} diff --git a/hadoop-mapreduce-project/src/webapps/task/index.html b/hadoop-mapreduce-project/src/webapps/task/index.html deleted file mode 100644 index a1b41522bcc..00000000000 --- a/hadoop-mapreduce-project/src/webapps/task/index.html +++ /dev/null @@ -1,17 +0,0 @@ - - diff --git a/hadoop-mapreduce-project/src/webapps/task/tasktracker.jsp b/hadoop-mapreduce-project/src/webapps/task/tasktracker.jsp deleted file mode 100644 index 5cc04f7fecd..00000000000 --- a/hadoop-mapreduce-project/src/webapps/task/tasktracker.jsp +++ /dev/null @@ -1,116 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.*" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="java.text.DecimalFormat" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.util.*" -%> -<%! private static final long serialVersionUID = 1L; -%> -<% - TaskTracker tracker = (TaskTracker) application.getAttribute("task.tracker"); - String trackerName = tracker.getName(); -%> - - - - -<%= trackerName %> Task Tracker Status - - -

<%= trackerName %> Task Tracker Status

-
-Version: <%= VersionInfo.getVersion()%>, - <%= VersionInfo.getRevision()%>
-Compiled: <%= VersionInfo.getDate()%> by - <%= VersionInfo.getUser()%> from - <%= VersionInfo.getBranch()%>
- -

Running tasks

-
- - - - - <% - Iterator itr = tracker.getRunningTaskStatuses().iterator(); - while (itr.hasNext()) { - TaskStatus status = (TaskStatus) itr.next(); - out.print(""); - out.print("\n"); - } - %> -
Task AttemptsStatusProgressErrors
" + status.getTaskID()); - out.print("" + status.getRunState()); - out.print("" + - StringUtils.formatPercent(status.getProgress(), 2)); - out.print("
" +
-           HtmlQuoting.quoteHtmlChars(status.getDiagnosticInfo()) +
-           "
-
- -

Non-Running Tasks

- - - <% - for(TaskStatus status: tracker.getNonRunningTasks()) { - out.print(""); - out.print("\n"); - } - %> -
Task AttemptsStatus
" + status.getTaskID() + "" + status.getRunState() + "
- - -

Tasks from Running Jobs

-
- - - - - <% - itr = tracker.getTasksFromRunningJobs().iterator(); - while (itr.hasNext()) { - TaskStatus status = (TaskStatus) itr.next(); - out.print(""); - out.print("\n"); - } - %> -
Task AttemptsStatusProgressErrors
" + status.getTaskID()); - out.print("" + status.getRunState()); - out.print("" + - StringUtils.formatPercent(status.getProgress(), 2)); - out.print("
" +
-           HtmlQuoting.quoteHtmlChars(status.getDiagnosticInfo()) +
-           "
-
- - -

Local Logs

-Log directory - -<% -out.println(ServletUtil.htmlFooter()); -%> From c7ab9779d2a9b17f4e1ffa3f030840a3dd1ef9ae Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 9 Nov 2012 17:39:20 +0000 Subject: [PATCH 22/31] HDFS-4162. Some malformed and unquoted HTML strings are returned from datanode web ui. Contributed by Darek Dagit. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407556 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/common/JspHelper.java | 16 +++---- .../server/datanode/DatanodeJspHelper.java | 5 ++- .../hdfs/server/common/TestJspHelper.java | 42 ++++++++++++++++++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0898deeeb34..95a6b1c380f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -574,6 +574,9 @@ Release 2.0.3-alpha - Unreleased HDFS-3810. Implement format() for BKJM (Ivan Kelly via umamahesh) + HDFS-4162. Some malformed and unquoted HTML strings are returned from + datanode web ui. (Darek Dagit via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index a2e0f501bde..3d03447a6c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -408,15 +408,15 @@ public class JspHelper { if (!parts[i].equals("")) { tempPath.append(parts[i]); out.print("" + parts[i] + "" + Path.SEPARATOR); + out.print("\">" + HtmlQuoting.quoteHtmlChars(parts[i]) + "" + Path.SEPARATOR); tempPath.append(Path.SEPARATOR); } } if(parts.length > 0) { - out.print(parts[parts.length-1]); + out.print(HtmlQuoting.quoteHtmlChars(parts[parts.length-1])); } } catch (UnsupportedEncodingException ex) { @@ -431,16 +431,16 @@ public class JspHelper { String nnAddress) throws IOException { out.print("
"); out.print("Goto : "); - out.print(""); - out.print(""); + out.print(""); + out.print(""); out.print(""); + + "value=\"" + namenodeInfoPort + "\"/>"); if (UserGroupInformation.isSecurityEnabled()) { out.print(""); + + "\" type=\"hidden\" value=\"" + tokenString + "\"/>"); } out.print(""); + + "value=\"" + nnAddress + "\"/>"); out.print("
"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java index d19e54e1122..98dceceb5d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.http.HtmlQuoting; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -119,7 +120,7 @@ public class DatanodeJspHelper { String target = dir; final HdfsFileStatus targetStatus = dfs.getFileInfo(target); if (targetStatus == null) { // not exists - out.print("

File or directory : " + target + " does not exist

"); + out.print("

File or directory : " + StringEscapeUtils.escapeHtml(target) + " does not exist

"); JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, target, nnAddr); } else { @@ -203,7 +204,7 @@ public class DatanodeJspHelper { + JspHelper.getDelegationTokenUrlParam(tokenString) + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr); cols[0] = "" - + localFileName + ""; + + HtmlQuoting.quoteHtmlChars(localFileName) + ""; cols[5] = lsDateFormat.format(new Date((files[i] .getModificationTime()))); cols[6] = files[i].getPermission().toString(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index bad1fff3ef2..ab6ed12492e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -19,13 +19,20 @@ package org.apache.hadoop.hdfs.server.common; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doAnswer; import java.io.IOException; +import java.io.StringReader; import java.net.InetSocketAddress; import java.util.ArrayList; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; +import javax.servlet.jsp.JspWriter; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -46,10 +53,17 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + public class TestJspHelper { private Configuration conf = new HdfsConfiguration(); + private String jspWriterOutput = ""; public static class DummySecretManager extends AbstractDelegationTokenSecretManager { @@ -368,7 +382,33 @@ public class TestJspHelper { ae.getMessage()); } } - + + @Test + public void testPrintGotoFormWritesValidXML() throws IOException, + ParserConfigurationException, SAXException { + JspWriter mockJspWriter = mock(JspWriter.class); + ArgumentCaptor arg = ArgumentCaptor.forClass(String.class); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invok) { + Object[] args = invok.getArguments(); + jspWriterOutput += (String) args[0]; + return null; + } + }).when(mockJspWriter).print(arg.capture()); + + jspWriterOutput = ""; + + JspHelper.printGotoForm(mockJspWriter, 424242, "a token string", + "foobar/file", "0.0.0.0"); + + DocumentBuilder parser = + DocumentBuilderFactory.newInstance().newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(jspWriterOutput)); + parser.parse(is); + } + private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getParameter(UserParam.NAME)).thenReturn(user); From db71de2e11cfa56a254ef4c92fea5ef4f8c19100 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 9 Nov 2012 18:07:04 +0000 Subject: [PATCH 23/31] HDFS-4080. Add a separate logger for block state change logs to enable turning off those logs. Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407566 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/conf/log4j.properties | 7 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../BlockInfoUnderConstruction.java | 4 +- .../server/blockmanagement/BlockManager.java | 87 ++++++++++--------- .../blockmanagement/CorruptReplicasMap.java | 4 +- .../blockmanagement/InvalidateBlocks.java | 2 +- .../UnderReplicatedBlocks.java | 16 ++-- .../hadoop/hdfs/server/namenode/NameNode.java | 1 + .../server/namenode/NameNodeRpcServer.java | 9 +- .../apache/hadoop/hdfs/TestDatanodeDeath.java | 1 + .../apache/hadoop/hdfs/TestFileAppend2.java | 1 + 11 files changed, 74 insertions(+), 60 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 4c49da0ad9f..b92ad27b67a 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -99,6 +99,13 @@ log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} log4j.appender.TLA.layout=org.apache.log4j.PatternLayout log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# +# HDFS block state change log from block manager +# +# Uncomment the following to suppress normal block state change +# messages from BlockManager in NameNode. +#log4j.logger.BlockStateChange=WARN + # #Security appender # diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 95a6b1c380f..78f92cbd5ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1961,6 +1961,8 @@ Release 0.23.5 - UNRELEASED NEW FEATURES IMPROVEMENTS + HDFS-4080. Add a separate logger for block state change logs to enable turning + off those logs. (Kihwal Lee via suresh) OPTIMIZATIONS diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index 5c0db1bb9b4..36b3598b2c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -233,7 +233,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { setBlockUCState(BlockUCState.UNDER_RECOVERY); blockRecoveryId = recoveryId; if (replicas.size() == 0) { - NameNode.stateChangeLog.warn("BLOCK*" + NameNode.blockStateChangeLog.warn("BLOCK*" + " BlockInfoUnderConstruction.initLeaseRecovery:" + " No blocks found, lease removed."); } @@ -245,7 +245,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { primaryNodeIndex = j; DatanodeDescriptor primary = replicas.get(j).getExpectedLocation(); primary.addBlockToBeRecovered(this); - NameNode.stateChangeLog.info("BLOCK* " + this + NameNode.blockStateChangeLog.info("BLOCK* " + this + " recovery started, primary=" + primary); return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 3a971fbf0d4..74dbfbc295f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -83,6 +83,7 @@ import com.google.common.collect.Sets; public class BlockManager { static final Log LOG = LogFactory.getLog(BlockManager.class); + static final Log blockLog = NameNode.blockStateChangeLog; /** Default load factor of map */ public static final float DEFAULT_MAP_LOAD_FACTOR = 0.75f; @@ -872,7 +873,7 @@ public class BlockManager { final long size) throws UnregisteredNodeException { final DatanodeDescriptor node = getDatanodeManager().getDatanode(datanode); if (node == null) { - NameNode.stateChangeLog.warn("BLOCK* getBlocks: " + blockLog.warn("BLOCK* getBlocks: " + "Asking for blocks from an unrecorded node " + datanode); throw new HadoopIllegalArgumentException( "Datanode " + datanode + " not found."); @@ -950,7 +951,7 @@ public class BlockManager { datanodes.append(node).append(" "); } if (datanodes.length() != 0) { - NameNode.stateChangeLog.info("BLOCK* addToInvalidates: " + b + " " + blockLog.info("BLOCK* addToInvalidates: " + b + " " + datanodes); } } @@ -971,7 +972,7 @@ public class BlockManager { // ignore the request for now. This could happen when BlockScanner // thread of Datanode reports bad block before Block reports are sent // by the Datanode on startup - NameNode.stateChangeLog.info("BLOCK* findAndMarkBlockAsCorrupt: " + blockLog.info("BLOCK* findAndMarkBlockAsCorrupt: " + blk + " not found"); return; } @@ -988,7 +989,7 @@ public class BlockManager { BlockCollection bc = b.corrupted.getBlockCollection(); if (bc == null) { - NameNode.stateChangeLog.info("BLOCK markBlockAsCorrupt: " + b + blockLog.info("BLOCK markBlockAsCorrupt: " + b + " cannot be marked as corrupt as it does not belong to any file"); addToInvalidates(b.corrupted, node); return; @@ -1013,7 +1014,7 @@ public class BlockManager { */ private void invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn ) throws IOException { - NameNode.stateChangeLog.info("BLOCK* invalidateBlock: " + b + " on " + dn); + blockLog.info("BLOCK* invalidateBlock: " + b + " on " + dn); DatanodeDescriptor node = getDatanodeManager().getDatanode(dn); if (node == null) { throw new IOException("Cannot invalidate " + b @@ -1023,7 +1024,7 @@ public class BlockManager { // Check how many copies we have of the block NumberReplicas nr = countNodes(b.stored); if (nr.replicasOnStaleNodes() > 0) { - NameNode.stateChangeLog.info("BLOCK* invalidateBlocks: postponing " + + blockLog.info("BLOCK* invalidateBlocks: postponing " + "invalidation of " + b + " on " + dn + " because " + nr.replicasOnStaleNodes() + " replica(s) are located on nodes " + "with potentially out-of-date block reports"); @@ -1033,12 +1034,12 @@ public class BlockManager { // If we have at least one copy on a live node, then we can delete it. addToInvalidates(b.corrupted, dn); removeStoredBlock(b.stored, node); - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* invalidateBlocks: " + if(blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* invalidateBlocks: " + b + " on " + dn + " listed for deletion."); } } else { - NameNode.stateChangeLog.info("BLOCK* invalidateBlocks: " + b + blockLog.info("BLOCK* invalidateBlocks: " + b + " on " + dn + " is the only copy and was not deleted"); } } @@ -1160,7 +1161,7 @@ public class BlockManager { (blockHasEnoughRacks(block)) ) { neededReplications.remove(block, priority); // remove from neededReplications neededReplications.decrementReplicationIndex(priority); - NameNode.stateChangeLog.info("BLOCK* Removing " + block + blockLog.info("BLOCK* Removing " + block + " from neededReplications as it has enough replicas"); continue; } @@ -1235,7 +1236,7 @@ public class BlockManager { neededReplications.remove(block, priority); // remove from neededReplications neededReplications.decrementReplicationIndex(priority); rw.targets = null; - NameNode.stateChangeLog.info("BLOCK* Removing " + block + blockLog.info("BLOCK* Removing " + block + " from neededReplications as it has enough replicas"); continue; } @@ -1261,8 +1262,8 @@ public class BlockManager { // The reason we use 'pending' is so we can retry // replications that fail after an appropriate amount of time. pendingReplications.increment(block, targets.length); - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(blockLog.isDebugEnabled()) { + blockLog.debug( "BLOCK* block " + block + " is moved from neededReplications to pendingReplications"); } @@ -1278,7 +1279,7 @@ public class BlockManager { namesystem.writeUnlock(); } - if (NameNode.stateChangeLog.isInfoEnabled()) { + if (blockLog.isInfoEnabled()) { // log which blocks have been scheduled for replication for(ReplicationWork rw : work){ DatanodeDescriptor[] targets = rw.targets; @@ -1288,13 +1289,13 @@ public class BlockManager { targetList.append(' '); targetList.append(targets[k]); } - NameNode.stateChangeLog.info("BLOCK* ask " + rw.srcNode + blockLog.info("BLOCK* ask " + rw.srcNode + " to replicate " + rw.block + " to " + targetList); } } } - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(blockLog.isDebugEnabled()) { + blockLog.debug( "BLOCK* neededReplications = " + neededReplications.size() + " pendingReplications = " + pendingReplications.size()); } @@ -1504,7 +1505,7 @@ public class BlockManager { // To minimize startup time, we discard any second (or later) block reports // that we receive while still in startup phase. if (namesystem.isInStartupSafeMode() && !node.isFirstBlockReport()) { - NameNode.stateChangeLog.info("BLOCK* processReport: " + blockLog.info("BLOCK* processReport: " + "discarded non-initial block report from " + nodeID + " because namenode still in startup phase"); return; @@ -1536,7 +1537,7 @@ public class BlockManager { // Log the block report processing stats from Namenode perspective NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime)); - NameNode.stateChangeLog.info("BLOCK* processReport: from " + blockLog.info("BLOCK* processReport: from " + nodeID + ", blocks: " + newReport.getNumberOfBlocks() + ", processing time: " + (endTime - startTime) + " msecs"); } @@ -1596,7 +1597,7 @@ public class BlockManager { addStoredBlock(b, node, null, true); } for (Block b : toInvalidate) { - NameNode.stateChangeLog.info("BLOCK* processReport: " + blockLog.info("BLOCK* processReport: " + b + " on " + node + " size " + b.getNumBytes() + " does not belong to any file"); addToInvalidates(b, node); @@ -2034,7 +2035,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } if (storedBlock == null || storedBlock.getBlockCollection() == null) { // If this block does not belong to anyfile, then we are done. - NameNode.stateChangeLog.info("BLOCK* addStoredBlock: " + block + " on " + blockLog.info("BLOCK* addStoredBlock: " + block + " on " + node + " size " + block.getNumBytes() + " but it does not belong to any file"); // we could add this block to invalidate set of this datanode. @@ -2056,7 +2057,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } } else { curReplicaDelta = 0; - NameNode.stateChangeLog.warn("BLOCK* addStoredBlock: " + blockLog.warn("BLOCK* addStoredBlock: " + "Redundant addStoredBlock request received for " + storedBlock + " on " + node + " size " + storedBlock.getNumBytes()); } @@ -2115,7 +2116,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) { - if (!NameNode.stateChangeLog.isInfoEnabled()) { + if (!blockLog.isInfoEnabled()) { return; } @@ -2126,7 +2127,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block storedBlock.appendStringTo(sb); sb.append(" size " ) .append(storedBlock.getNumBytes()); - NameNode.stateChangeLog.info(sb); + blockLog.info(sb); } /** * Invalidate corrupt replicas. @@ -2153,7 +2154,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block try { invalidateBlock(new BlockToMarkCorrupt(blk, null), node); } catch (IOException e) { - NameNode.stateChangeLog.info("invalidateCorruptReplicas " + blockLog.info("invalidateCorruptReplicas " + "error in deleting bad block " + blk + " on " + node, e); gotException = true; } @@ -2391,7 +2392,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // upon giving instructions to the namenode. // addToInvalidates(b, cur); - NameNode.stateChangeLog.info("BLOCK* chooseExcessReplicates: " + blockLog.info("BLOCK* chooseExcessReplicates: " +"("+cur+", "+b+") is added to invalidated blocks set"); } } @@ -2405,8 +2406,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } if (excessBlocks.add(block)) { excessBlocksCount++; - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* addToExcessReplicate:" + if(blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* addToExcessReplicate:" + " (" + dn + ", " + block + ") is added to excessReplicateMap"); } @@ -2418,15 +2419,15 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * removed block is still valid. */ public void removeStoredBlock(Block block, DatanodeDescriptor node) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + if(blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* removeStoredBlock: " + block + " from " + node); } assert (namesystem.hasWriteLock()); { if (!blocksMap.removeNode(block, node)) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + if(blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* removeStoredBlock: " + block + " has already been removed from node " + node); } return; @@ -2453,8 +2454,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block if (excessBlocks != null) { if (excessBlocks.remove(block)) { excessBlocksCount--; - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* removeStoredBlock: " + if(blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* removeStoredBlock: " + block + " is removed from excessBlocks"); } if (excessBlocks.size() == 0) { @@ -2497,7 +2498,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block if (delHint != null && delHint.length() != 0) { delHintNode = datanodeManager.getDatanode(delHint); if (delHintNode == null) { - NameNode.stateChangeLog.warn("BLOCK* blockReceived: " + block + blockLog.warn("BLOCK* blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint); } } @@ -2532,7 +2533,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block addStoredBlock(b, node, delHintNode, true); } for (Block b : toInvalidate) { - NameNode.stateChangeLog.info("BLOCK* addBlock: block " + blockLog.info("BLOCK* addBlock: block " + b + " on " + node + " size " + b.getNumBytes() + " does not belong to any file"); addToInvalidates(b, node); @@ -2558,7 +2559,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block try { final DatanodeDescriptor node = datanodeManager.getDatanode(nodeID); if (node == null || !node.isAlive) { - NameNode.stateChangeLog + blockLog .warn("BLOCK* processIncrementalBlockReport" + " is received from dead or unregistered node " + nodeID); @@ -2585,19 +2586,19 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block String msg = "Unknown block status code reported by " + nodeID + ": " + rdbi; - NameNode.stateChangeLog.warn(msg); + blockLog.warn(msg); assert false : msg; // if assertions are enabled, throw. break; } - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* block " + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* block " + (rdbi.getStatus()) + ": " + rdbi.getBlock() + " is received from " + nodeID); } } } finally { namesystem.writeUnlock(); - NameNode.stateChangeLog + blockLog .debug("*BLOCK* NameNode.processIncrementalBlockReport: " + "from " + nodeID + " receiving: " + receiving + ", " @@ -2890,8 +2891,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } finally { namesystem.writeUnlock(); } - if (NameNode.stateChangeLog.isInfoEnabled()) { - NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName() + if (blockLog.isInfoEnabled()) { + blockLog.info("BLOCK* " + getClass().getSimpleName() + ": ask " + dn + " to delete " + toInvalidate); } return toInvalidate.size(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java index 440e3d40569..4613199ee6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java @@ -63,13 +63,13 @@ public class CorruptReplicasMap{ if (!nodes.contains(dn)) { nodes.add(dn); - NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ + NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ blk.getBlockName() + " added as corrupt on " + dn + " by " + Server.getRemoteIp() + reasonText); } else { - NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ + NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ "duplicate requested for " + blk.getBlockName() + " to add as corrupt " + "on " + dn + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java index 2a10ee2253c..841ca41755f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java @@ -86,7 +86,7 @@ class InvalidateBlocks { if (set.add(block)) { numBlocks++; if (log) { - NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName() + NameNode.blockStateChangeLog.info("BLOCK* " + getClass().getSimpleName() + ": add " + block + " to " + datanode); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java index 0759d533633..779b445cece 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java @@ -190,8 +190,8 @@ class UnderReplicatedBlocks implements Iterable { int priLevel = getPriority(block, curReplicas, decomissionedReplicas, expectedReplicas); if(priorityQueues.get(priLevel).add(block)) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(NameNode.blockStateChangeLog.isDebugEnabled()) { + NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.add:" + block + " has only " + curReplicas @@ -233,8 +233,8 @@ class UnderReplicatedBlocks implements Iterable { boolean remove(Block block, int priLevel) { if(priLevel >= 0 && priLevel < LEVEL && priorityQueues.get(priLevel).remove(block)) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(NameNode.blockStateChangeLog.isDebugEnabled()) { + NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.remove: " + "Removing block " + block + " from priority queue "+ priLevel); @@ -245,8 +245,8 @@ class UnderReplicatedBlocks implements Iterable { // not found in the queue for the given priority level. for (int i = 0; i < LEVEL; i++) { if (priorityQueues.get(i).remove(block)) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(NameNode.blockStateChangeLog.isDebugEnabled()) { + NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.remove: " + "Removing block " + block + " from priority queue "+ i); @@ -296,8 +296,8 @@ class UnderReplicatedBlocks implements Iterable { remove(block, oldPri); } if(priorityQueues.get(curPri).add(block)) { - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( + if(NameNode.blockStateChangeLog.isDebugEnabled()) { + NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.update:" + block + " has only "+ curReplicas diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index fb5c88d8bbc..309811c242f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -227,6 +227,7 @@ public class NameNode { public static final int DEFAULT_PORT = 8020; public static final Log LOG = LogFactory.getLog(NameNode.class.getName()); public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.hdfs.StateChange"); + public static final Log blockStateChangeLog = LogFactory.getLog("BlockStateChange"); public static final HAState ACTIVE_STATE = new ActiveState(); public static final HAState STANDBY_STATE = new StandbyState(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 6d505836f9f..de8a909e37f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -132,6 +132,7 @@ class NameNodeRpcServer implements NamenodeProtocols { private static final Log LOG = NameNode.LOG; private static final Log stateChangeLog = NameNode.stateChangeLog; + private static final Log blockStateChangeLog = NameNode.blockStateChangeLog; // Dependencies from other parts of NN. protected final FSNamesystem namesystem; @@ -889,8 +890,8 @@ class NameNodeRpcServer implements NamenodeProtocols { String poolId, StorageBlockReport[] reports) throws IOException { verifyRequest(nodeReg); BlockListAsLongs blist = new BlockListAsLongs(reports[0].getBlocks()); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*BLOCK* NameNode.blockReport: " + if(blockStateChangeLog.isDebugEnabled()) { + blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: " + "from " + nodeReg + " " + blist.getNumberOfBlocks() + " blocks"); } @@ -905,8 +906,8 @@ class NameNodeRpcServer implements NamenodeProtocols { public void blockReceivedAndDeleted(DatanodeRegistration nodeReg, String poolId, StorageReceivedDeletedBlocks[] receivedAndDeletedBlocks) throws IOException { verifyRequest(nodeReg); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: " + if(blockStateChangeLog.isDebugEnabled()) { + blockStateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: " +"from "+nodeReg+" "+receivedAndDeletedBlocks.length +" blocks."); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java index a6625431fdc..c0b3994aed9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java @@ -46,6 +46,7 @@ import org.junit.Test; public class TestDatanodeDeath { { ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + ((Log4JLogger)NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL); ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java index f488040c492..d33052e4c6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java @@ -51,6 +51,7 @@ public class TestFileAppend2 { { ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); + ((Log4JLogger)NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL); ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL); ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); From a4b1c675b623ec1c174131f2c645996157c14ae2 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 9 Nov 2012 18:09:21 +0000 Subject: [PATCH 24/31] Moving HDFS-4080 to Incompatible Change section. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407567 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 78f92cbd5ef..2e407129b86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1958,11 +1958,12 @@ Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES + HDFS-4080. Add a separate logger for block state change logs to enable turning + off those logs. (Kihwal Lee via suresh) + NEW FEATURES IMPROVEMENTS - HDFS-4080. Add a separate logger for block state change logs to enable turning - off those logs. (Kihwal Lee via suresh) OPTIMIZATIONS From f8c486fbc8b9ea2380b6f7caa2fb8e2364774f5d Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 9 Nov 2012 18:25:49 +0000 Subject: [PATCH 25/31] HADOOP-7115. Add a cache for getpwuid_r and getpwgid_r calls (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407577 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../hadoop/fs/CommonConfigurationKeys.java | 8 +- .../org/apache/hadoop/io/SecureIOUtils.java | 2 +- .../apache/hadoop/io/nativeio/NativeIO.java | 83 ++++++- .../org/apache/hadoop/io/nativeio/NativeIO.c | 204 ++++++++++++------ .../src/main/resources/core-default.xml | 11 + .../hadoop/io/nativeio/TestNativeIO.java | 16 +- 7 files changed, 252 insertions(+), 74 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4d6b88c93d0..6a9cacc5015 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -413,6 +413,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-9012. IPC Client sends wrong connection context (daryn via bobby) + HADOOP-7115. Add a cache for getpwuid_r and getpwgid_r calls (tucu) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 28f8dd0532e..a7579a96406 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -184,5 +184,11 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { */ public static final String KERBEROS_TICKET_CACHE_PATH = "hadoop.security.kerberos.ticket.cache.path"; -} + public static final String HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY = + "hadoop.security.uid.cache.secs"; + + public static final long HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT = + 4*60*60; // 4 hours + +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java index b30c4a4da44..3d01810e712 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java @@ -120,7 +120,7 @@ public class SecureIOUtils { FileInputStream fis = new FileInputStream(f); boolean success = false; try { - Stat stat = NativeIO.fstat(fis.getFD()); + Stat stat = NativeIO.getFstat(fis.getFD()); checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner, expectedGroup); success = true; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 4cfa0761edc..94ff5f6057f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -19,8 +19,13 @@ package org.apache.hadoop.io.nativeio; import java.io.FileDescriptor; import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.commons.logging.Log; @@ -30,6 +35,8 @@ import org.apache.commons.logging.LogFactory; * These functions should generally be used alongside a fallback to another * more portable mechanism. */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public class NativeIO { // Flags for open() call from bits/fcntl.h public static final int O_RDONLY = 00; @@ -86,6 +93,8 @@ public class NativeIO { "hadoop.workaround.non.threadsafe.getpwuid"; static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false; + private static long cacheTimeout = -1; + static { if (NativeCodeLoader.isNativeCodeLoaded()) { try { @@ -96,6 +105,14 @@ public class NativeIO { initNative(); nativeLoaded = true; + + cacheTimeout = conf.getLong( + CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY, + CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * + 1000; + LOG.debug("Initialized cache for IDs to User/Group mapping with a" + + " cache timeout of " + cacheTimeout/1000 + " seconds."); + } catch (Throwable t) { // This can happen if the user has an older version of libhadoop.so // installed - in this case we can continue without native IO @@ -115,7 +132,7 @@ public class NativeIO { /** Wrapper around open(2) */ public static native FileDescriptor open(String path, int flags, int mode) throws IOException; /** Wrapper around fstat(2) */ - public static native Stat fstat(FileDescriptor fd) throws IOException; + private static native Stat fstat(FileDescriptor fd) throws IOException; /** Wrapper around chmod(2) */ public static native void chmod(String path, int mode) throws IOException; @@ -176,6 +193,7 @@ public class NativeIO { * Result type of the fstat call */ public static class Stat { + private int ownerId, groupId; private String owner, group; private int mode; @@ -196,9 +214,9 @@ public class NativeIO { public static final int S_IWUSR = 0000200; /* write permission, owner */ public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ - Stat(String owner, String group, int mode) { - this.owner = owner; - this.group = group; + Stat(int ownerId, int groupId, int mode) { + this.ownerId = ownerId; + this.groupId = groupId; this.mode = mode; } @@ -218,4 +236,61 @@ public class NativeIO { return mode; } } + + static native String getUserName(int uid) throws IOException; + + static native String getGroupName(int uid) throws IOException; + + private static class CachedName { + final long timestamp; + final String name; + + public CachedName(String name, long timestamp) { + this.name = name; + this.timestamp = timestamp; + } + } + + private static final Map USER_ID_NAME_CACHE = + new ConcurrentHashMap(); + + private static final Map GROUP_ID_NAME_CACHE = + new ConcurrentHashMap(); + + private enum IdCache { USER, GROUP } + + private static String getName(IdCache domain, int id) throws IOException { + Map idNameCache = (domain == IdCache.USER) + ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE; + String name; + CachedName cachedName = idNameCache.get(id); + long now = System.currentTimeMillis(); + if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) { + name = cachedName.name; + } else { + name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id); + if (LOG.isDebugEnabled()) { + String type = (domain == IdCache.USER) ? "UserName" : "GroupName"; + LOG.debug("Got " + type + " " + name + " for ID " + id + + " from the native implementation"); + } + cachedName = new CachedName(name, now); + idNameCache.put(id, cachedName); + } + return name; + } + + /** + * Returns the file stat for a file descriptor. + * + * @param fd file descriptor. + * @return the file descriptor file stat. + * @throws IOException thrown if there was an IO error while obtaining the file stat. + */ + public static Stat getFstat(FileDescriptor fd) throws IOException { + Stat stat = fstat(fd); + stat.owner = getName(IdCache.USER, stat.ownerId); + stat.group = getName(IdCache.GROUP, stat.groupId); + return stat; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 4a91d0af954..7e82152c1fe 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -72,16 +72,27 @@ static int workaround_non_threadsafe_calls(JNIEnv *env, jclass clazz) { static void stat_init(JNIEnv *env, jclass nativeio_class) { // Init Stat jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$Stat"); - PASS_EXCEPTIONS(env); + if (!clazz) { + return; // exception has been raised + } stat_clazz = (*env)->NewGlobalRef(env, clazz); + if (!stat_clazz) { + return; // exception has been raised + } stat_ctor = (*env)->GetMethodID(env, stat_clazz, "", - "(Ljava/lang/String;Ljava/lang/String;I)V"); - + "(III)V"); + if (!stat_ctor) { + return; // exception has been raised + } jclass obj_class = (*env)->FindClass(env, "java/lang/Object"); - assert(obj_class != NULL); + if (!obj_class) { + return; // exception has been raised + } jmethodID obj_ctor = (*env)->GetMethodID(env, obj_class, "", "()V"); - assert(obj_ctor != NULL); + if (!obj_ctor) { + return; // exception has been raised + } if (workaround_non_threadsafe_calls(env, nativeio_class)) { pw_lock_object = (*env)->NewObject(env, obj_class, obj_ctor); @@ -158,8 +169,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_fstat( JNIEnv *env, jclass clazz, jobject fd_object) { jobject ret = NULL; - char *pw_buf = NULL; - int pw_lock_locked = 0; int fd = fd_get(env, fd_object); PASS_EXCEPTIONS_GOTO(env, cleanup); @@ -171,71 +180,14 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_fstat( goto cleanup; } - size_t pw_buflen = get_pw_buflen(); - if ((pw_buf = malloc(pw_buflen)) == NULL) { - THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); - goto cleanup; - } - - if (pw_lock_object != NULL) { - if ((*env)->MonitorEnter(env, pw_lock_object) != JNI_OK) { - goto cleanup; - } - pw_lock_locked = 1; - } - - // Grab username - struct passwd pwd, *pwdp; - while ((rc = getpwuid_r(s.st_uid, &pwd, pw_buf, pw_buflen, &pwdp)) != 0) { - if (rc != ERANGE) { - throw_ioe(env, rc); - goto cleanup; - } - free(pw_buf); - pw_buflen *= 2; - if ((pw_buf = malloc(pw_buflen)) == NULL) { - THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); - goto cleanup; - } - } - assert(pwdp == &pwd); - - jstring jstr_username = (*env)->NewStringUTF(env, pwd.pw_name); - if (jstr_username == NULL) goto cleanup; - - // Grab group - struct group grp, *grpp; - while ((rc = getgrgid_r(s.st_gid, &grp, pw_buf, pw_buflen, &grpp)) != 0) { - if (rc != ERANGE) { - throw_ioe(env, rc); - goto cleanup; - } - free(pw_buf); - pw_buflen *= 2; - if ((pw_buf = malloc(pw_buflen)) == NULL) { - THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); - goto cleanup; - } - } - assert(grpp == &grp); - - jstring jstr_groupname = (*env)->NewStringUTF(env, grp.gr_name); - PASS_EXCEPTIONS_GOTO(env, cleanup); - // Construct result ret = (*env)->NewObject(env, stat_clazz, stat_ctor, - jstr_username, jstr_groupname, s.st_mode); + (jint)s.st_uid, (jint)s.st_gid, (jint)s.st_mode); cleanup: - if (pw_buf != NULL) free(pw_buf); - if (pw_lock_locked) { - (*env)->MonitorExit(env, pw_lock_object); - } return ret; } - - /** * public static native void posix_fadvise( * FileDescriptor fd, long offset, long len, int flags); @@ -385,6 +337,128 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_chmod( (*env)->ReleaseStringUTFChars(env, j_path, path); } +/* + * static native String getUserName(int uid); + */ +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_getUserName(JNIEnv *env, +jclass clazz, jint uid) +{ + int pw_lock_locked = 0; + if (pw_lock_object != NULL) { + if ((*env)->MonitorEnter(env, pw_lock_object) != JNI_OK) { + goto cleanup; + } + pw_lock_locked = 1; + } + + char *pw_buf = NULL; + int rc; + size_t pw_buflen = get_pw_buflen(); + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + + // Grab username + struct passwd pwd, *pwdp; + while ((rc = getpwuid_r((uid_t)uid, &pwd, pw_buf, pw_buflen, &pwdp)) != 0) { + if (rc != ERANGE) { + throw_ioe(env, rc); + goto cleanup; + } + free(pw_buf); + pw_buflen *= 2; + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + } + if (pwdp == NULL) { + char msg[80]; + snprintf(msg, sizeof(msg), "uid not found: %d", uid); + THROW(env, "java/io/IOException", msg); + goto cleanup; + } + if (pwdp != &pwd) { + char msg[80]; + snprintf(msg, sizeof(msg), "pwd pointer inconsistent with reference. uid: %d", uid); + THROW(env, "java/lang/IllegalStateException", msg); + goto cleanup; + } + + jstring jstr_username = (*env)->NewStringUTF(env, pwd.pw_name); + +cleanup: + if (pw_lock_locked) { + (*env)->MonitorExit(env, pw_lock_object); + } + if (pw_buf != NULL) free(pw_buf); + return jstr_username; +} + +/* + * static native String getGroupName(int gid); + */ +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_getGroupName(JNIEnv *env, +jclass clazz, jint gid) +{ + int pw_lock_locked = 0; + + if (pw_lock_object != NULL) { + if ((*env)->MonitorEnter(env, pw_lock_object) != JNI_OK) { + goto cleanup; + } + pw_lock_locked = 1; + } + + char *pw_buf = NULL; + int rc; + size_t pw_buflen = get_pw_buflen(); + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + + // Grab group + struct group grp, *grpp; + while ((rc = getgrgid_r((uid_t)gid, &grp, pw_buf, pw_buflen, &grpp)) != 0) { + if (rc != ERANGE) { + throw_ioe(env, rc); + goto cleanup; + } + free(pw_buf); + pw_buflen *= 2; + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + } + if (grpp == NULL) { + char msg[80]; + snprintf(msg, sizeof(msg), "gid not found: %d", gid); + THROW(env, "java/io/IOException", msg); + goto cleanup; + } + if (grpp != &grp) { + char msg[80]; + snprintf(msg, sizeof(msg), "pwd pointer inconsistent with reference. gid: %d", gid); + THROW(env, "java/lang/IllegalStateException", msg); + goto cleanup; + } + + jstring jstr_groupname = (*env)->NewStringUTF(env, grp.gr_name); + PASS_EXCEPTIONS_GOTO(env, cleanup); + +cleanup: + if (pw_lock_locked) { + (*env)->MonitorExit(env, pw_lock_object); + } + if (pw_buf != NULL) free(pw_buf); + return jstr_groupname; +} + /* * Throw a java.IO.IOException, generating the message from errno. diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index affcc523520..bd6e9420305 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -214,6 +214,17 @@ + + + hadoop.security.uid.cache.secs + 14400 + + This is the config controlling the validity of the entries in the cache + containing the userId to userName and groupId to groupName used by + NativeIO getFstat(). + + + hadoop.rpc.protection authentication diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java index f947e02efb4..9ee1516863b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java @@ -61,7 +61,7 @@ public class TestNativeIO { public void testFstat() throws Exception { FileOutputStream fos = new FileOutputStream( new File(TEST_DIR, "testfstat")); - NativeIO.Stat stat = NativeIO.fstat(fos.getFD()); + NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); fos.close(); LOG.info("Stat: " + String.valueOf(stat)); @@ -93,7 +93,7 @@ public class TestNativeIO { long et = Time.now() + 5000; while (Time.now() < et) { try { - NativeIO.Stat stat = NativeIO.fstat(fos.getFD()); + NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); assertEquals(System.getProperty("user.name"), stat.getOwner()); assertNotNull(stat.getGroup()); assertTrue(!stat.getGroup().isEmpty()); @@ -125,7 +125,7 @@ public class TestNativeIO { new File(TEST_DIR, "testfstat2")); fos.close(); try { - NativeIO.Stat stat = NativeIO.fstat(fos.getFD()); + NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); } catch (NativeIOException nioe) { LOG.info("Got expected exception", nioe); assertEquals(Errno.EBADF, nioe.getErrno()); @@ -283,4 +283,14 @@ public class TestNativeIO { assertEquals(expected, perms.toShort()); } + @Test + public void testGetUserName() throws IOException { + assertFalse(NativeIO.getUserName(0).isEmpty()); + } + + @Test + public void testGetGroupName() throws IOException { + assertFalse(NativeIO.getGroupName(0).isEmpty()); + } + } From 9e3af0ccd25f237c56f9a307510fa68599f00cc1 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Fri, 9 Nov 2012 19:52:28 +0000 Subject: [PATCH 26/31] YARN-206. TestApplicationCleanup.testContainerCleanup occasionally fails. (jlowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407606 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../TestApplicationCleanup.java | 58 +++++++++---------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 97636dabc70..03d293aa6ae 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -213,6 +213,9 @@ Release 0.23.5 - UNRELEASED YARN-201. Fix CapacityScheduler to be less conservative for starved off-switch requests. (jlowe via acmurthy) + YARN-206. TestApplicationCleanup.testContainerCleanup occasionally fails. + (jlowe via jeagles) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java index 349de1e44c8..ca640b39e59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java @@ -81,38 +81,38 @@ public class TestApplicationCleanup { new ArrayList()).getAllocatedContainers(); int contReceived = conts.size(); int waitCount = 0; - while (contReceived < request && waitCount++ < 20) { + while (contReceived < request && waitCount++ < 200) { + LOG.info("Got " + contReceived + " containers. Waiting to get " + + request); + Thread.sleep(100); conts = am.allocate(new ArrayList(), new ArrayList()).getAllocatedContainers(); contReceived += conts.size(); - LOG.info("Got " + contReceived + " containers. Waiting to get " - + request); - Thread.sleep(2000); } - Assert.assertEquals(request, conts.size()); + Assert.assertEquals(request, contReceived); am.unregisterAppAttempt(); HeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE); am.waitForState(RMAppAttemptState.FINISHED); - int cleanedConts = 0; - int cleanedApps = 0; - List contsToClean = null; - List apps = null; - //currently only containers are cleaned via this //AM container is cleaned via container launcher + resp = nm1.nodeHeartbeat(true); + List contsToClean = resp.getContainersToCleanupList(); + List apps = resp.getApplicationsToCleanupList(); + int cleanedConts = contsToClean.size(); + int cleanedApps = apps.size(); waitCount = 0; - while ((cleanedConts < 2 || cleanedApps < 1) && waitCount++ < 20) { - contsToClean = resp.getContainersToCleanupList(); - apps = resp.getApplicationsToCleanupList(); + while ((cleanedConts < 2 || cleanedApps < 1) && waitCount++ < 200) { LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts + " cleanedApps: " + cleanedApps); + Thread.sleep(100); + resp = nm1.nodeHeartbeat(true); + contsToClean = resp.getContainersToCleanupList(); + apps = resp.getApplicationsToCleanupList(); cleanedConts += contsToClean.size(); cleanedApps += apps.size(); - Thread.sleep(1000); - resp = nm1.nodeHeartbeat(true); } Assert.assertEquals(1, apps.size()); @@ -170,20 +170,20 @@ public class TestApplicationCleanup { new ArrayList()).getAllocatedContainers(); int contReceived = conts.size(); int waitCount = 0; - while (contReceived < request && waitCount++ < 20) { + while (contReceived < request && waitCount++ < 200) { + LOG.info("Got " + contReceived + " containers. Waiting to get " + + request); + Thread.sleep(100); conts = am.allocate(new ArrayList(), new ArrayList()).getAllocatedContainers(); dispatcher.await(); contReceived += conts.size(); - LOG.info("Got " + contReceived + " containers. Waiting to get " - + request); - Thread.sleep(2000); } - Assert.assertEquals(request, conts.size()); + Assert.assertEquals(request, contReceived); // Release a container. ArrayList release = new ArrayList(); - release.add(conts.get(1).getId()); + release.add(conts.get(0).getId()); am.allocate(new ArrayList(), release); dispatcher.await(); @@ -194,7 +194,7 @@ public class TestApplicationCleanup { new HashMap>(); ArrayList containerStatusList = new ArrayList(); - containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1) + containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0) .getId(), ContainerState.RUNNING, "nothing", 0)); containerStatuses.put(app.getApplicationId(), containerStatusList); @@ -203,13 +203,13 @@ public class TestApplicationCleanup { List contsToClean = resp.getContainersToCleanupList(); int cleanedConts = contsToClean.size(); waitCount = 0; - while (cleanedConts < 1 && waitCount++ < 20) { + while (cleanedConts < 1 && waitCount++ < 200) { + LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts); + Thread.sleep(100); resp = nm1.nodeHeartbeat(true); dispatcher.await(); contsToClean = resp.getContainersToCleanupList(); - LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts); cleanedConts += contsToClean.size(); - Thread.sleep(1000); } LOG.info("Got cleanup for " + contsToClean.get(0)); Assert.assertEquals(1, cleanedConts); @@ -220,7 +220,7 @@ public class TestApplicationCleanup { + "NM getting cleanup"); containerStatuses.clear(); containerStatusList.clear(); - containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1) + containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0) .getId(), ContainerState.RUNNING, "nothing", 0)); containerStatuses.put(app.getApplicationId(), containerStatusList); @@ -231,13 +231,13 @@ public class TestApplicationCleanup { // The cleanup list won't be instantaneous as it is given out by scheduler // and not RMNodeImpl. waitCount = 0; - while (cleanedConts < 1 && waitCount++ < 20) { + while (cleanedConts < 1 && waitCount++ < 200) { + LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts); + Thread.sleep(100); resp = nm1.nodeHeartbeat(true); dispatcher.await(); contsToClean = resp.getContainersToCleanupList(); - LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts); cleanedConts += contsToClean.size(); - Thread.sleep(1000); } LOG.info("Got cleanup for " + contsToClean.get(0)); Assert.assertEquals(1, cleanedConts); From de1f33dc35599d3169475d82f3052f1b04025add Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 9 Nov 2012 20:40:09 +0000 Subject: [PATCH 27/31] HADOOP-9020. Add a SASL PLAIN server (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407622 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../hadoop/security/SaslPlainServer.java | 159 ++++++++++++++++++ .../apache/hadoop/security/SaslRpcServer.java | 5 +- .../org/apache/hadoop/ipc/TestSaslRPC.java | 124 +++++++++++++- 4 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6a9cacc5015..5ea2434aca8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -291,6 +291,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-8597. Permit FsShell's text command to read Avro files. (Ivan Vladimirov Ivanov via cutting) + HADOOP-9020. Add a SASL PLAIN server (daryn via bobby) + IMPROVEMENTS HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java new file mode 100644 index 00000000000..7d1b98062b0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import java.security.Provider; +import java.util.Map; + +import javax.security.auth.callback.*; +import javax.security.sasl.AuthorizeCallback; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import javax.security.sasl.SaslServerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class SaslPlainServer implements SaslServer { + @SuppressWarnings("serial") + public static class SecurityProvider extends Provider { + public SecurityProvider() { + super("SaslPlainServer", 1.0, "SASL PLAIN Authentication Server"); + put("SaslServerFactory.PLAIN", + SaslPlainServerFactory.class.getName()); + } + } + + public static class SaslPlainServerFactory implements SaslServerFactory { + @Override + public SaslServer createSaslServer(String mechanism, String protocol, + String serverName, Map props, CallbackHandler cbh) + throws SaslException { + return "PLAIN".equals(mechanism) ? new SaslPlainServer(cbh) : null; + } + @Override + public String[] getMechanismNames(Map props){ + return (props == null) || "false".equals(props.get(Sasl.POLICY_NOPLAINTEXT)) + ? new String[]{"PLAIN"} + : new String[0]; + } + } + + private CallbackHandler cbh; + private boolean completed; + private String authz; + + SaslPlainServer(CallbackHandler callback) { + this.cbh = callback; + } + + @Override + public String getMechanismName() { + return "PLAIN"; + } + + @Override + public byte[] evaluateResponse(byte[] response) throws SaslException { + if (completed) { + throw new IllegalStateException("PLAIN authentication has completed"); + } + if (response == null) { + throw new IllegalArgumentException("Received null response"); + } + try { + String payload; + try { + payload = new String(response, "UTF-8"); + } catch (Exception e) { + throw new IllegalArgumentException("Received corrupt response", e); + } + // [ authz, authn, password ] + String[] parts = payload.split("\u0000", 3); + if (parts.length != 3) { + throw new IllegalArgumentException("Received corrupt response"); + } + if (parts[0].isEmpty()) { // authz = authn + parts[0] = parts[1]; + } + + NameCallback nc = new NameCallback("SASL PLAIN"); + nc.setName(parts[1]); + PasswordCallback pc = new PasswordCallback("SASL PLAIN", false); + pc.setPassword(parts[2].toCharArray()); + AuthorizeCallback ac = new AuthorizeCallback(parts[1], parts[0]); + cbh.handle(new Callback[]{nc, pc, ac}); + if (ac.isAuthorized()) { + authz = ac.getAuthorizedID(); + } + } catch (Exception e) { + throw new SaslException("PLAIN auth failed: " + e.getMessage()); + } finally { + completed = true; + } + return null; + } + + private void throwIfNotComplete() { + if (!completed) { + throw new IllegalStateException("PLAIN authentication not completed"); + } + } + + @Override + public boolean isComplete() { + return completed; + } + + @Override + public String getAuthorizationID() { + throwIfNotComplete(); + return authz; + } + + @Override + public Object getNegotiatedProperty(String propName) { + throwIfNotComplete(); + return Sasl.QOP.equals(propName) ? "auth" : null; + } + + @Override + public byte[] wrap(byte[] outgoing, int offset, int len) + throws SaslException { + throwIfNotComplete(); + throw new IllegalStateException( + "PLAIN supports neither integrity nor privacy"); + } + + @Override + public byte[] unwrap(byte[] incoming, int offset, int len) + throws SaslException { + throwIfNotComplete(); + throw new IllegalStateException( + "PLAIN supports neither integrity nor privacy"); + } + + @Override + public void dispose() throws SaslException { + cbh = null; + authz = null; + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java index 31b4c35dae2..33942dc0885 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java @@ -23,6 +23,7 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import java.security.Security; import java.util.Map; import java.util.TreeMap; @@ -89,6 +90,7 @@ public class SaslRpcServer { SASL_PROPS.put(Sasl.QOP, saslQOP.getSaslQop()); SASL_PROPS.put(Sasl.SERVER_AUTH, "true"); + Security.addProvider(new SaslPlainServer.SecurityProvider()); } static String encodeIdentifier(byte[] identifier) { @@ -138,7 +140,8 @@ public class SaslRpcServer { public static enum AuthMethod { SIMPLE((byte) 80, ""), KERBEROS((byte) 81, "GSSAPI"), - DIGEST((byte) 82, "DIGEST-MD5"); + DIGEST((byte) 82, "DIGEST-MD5"), + PLAIN((byte) 83, "PLAIN"); /** The code for this method. */ public final byte code; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java index 68dd2c83930..be3064d20b3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java @@ -27,12 +27,13 @@ import java.io.IOException; import java.lang.annotation.Annotation; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; +import java.security.Security; import java.util.Collection; import java.util.Set; import java.util.regex.Pattern; -import javax.security.sasl.Sasl; - +import javax.security.auth.callback.*; +import javax.security.sasl.*; import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -44,6 +45,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.*; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; @@ -53,7 +55,6 @@ import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.log4j.Level; -import org.apache.tools.ant.types.Assertions.EnabledAssertion; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -76,7 +77,8 @@ public class TestSaslRPC { @BeforeClass public static void setupKerb() { System.setProperty("java.security.krb5.kdc", ""); - System.setProperty("java.security.krb5.realm", "NONE"); + System.setProperty("java.security.krb5.realm", "NONE"); + Security.addProvider(new SaslPlainServer.SecurityProvider()); } @Before @@ -448,6 +450,120 @@ public class TestSaslRPC { System.out.println("Test is successful."); } + @Test + public void testSaslPlainServer() throws IOException { + runNegotiation( + new TestPlainCallbacks.Client("user", "pass"), + new TestPlainCallbacks.Server("user", "pass")); + } + + @Test + public void testSaslPlainServerBadPassword() throws IOException { + SaslException e = null; + try { + runNegotiation( + new TestPlainCallbacks.Client("user", "pass1"), + new TestPlainCallbacks.Server("user", "pass2")); + } catch (SaslException se) { + e = se; + } + assertNotNull(e); + assertEquals("PLAIN auth failed: wrong password", e.getMessage()); + } + + + private void runNegotiation(CallbackHandler clientCbh, + CallbackHandler serverCbh) + throws SaslException { + String mechanism = AuthMethod.PLAIN.getMechanismName(); + + SaslClient saslClient = Sasl.createSaslClient( + new String[]{ mechanism }, null, null, null, null, clientCbh); + assertNotNull(saslClient); + + SaslServer saslServer = Sasl.createSaslServer( + mechanism, null, "localhost", null, serverCbh); + assertNotNull("failed to find PLAIN server", saslServer); + + byte[] response = saslClient.evaluateChallenge(new byte[0]); + assertNotNull(response); + assertTrue(saslClient.isComplete()); + + response = saslServer.evaluateResponse(response); + assertNull(response); + assertTrue(saslServer.isComplete()); + assertNotNull(saslServer.getAuthorizationID()); + } + + static class TestPlainCallbacks { + public static class Client implements CallbackHandler { + String user = null; + String password = null; + + Client(String user, String password) { + this.user = user; + this.password = password; + } + + @Override + public void handle(Callback[] callbacks) + throws UnsupportedCallbackException { + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) { + ((NameCallback) callback).setName(user); + } else if (callback instanceof PasswordCallback) { + ((PasswordCallback) callback).setPassword(password.toCharArray()); + } else { + throw new UnsupportedCallbackException(callback, + "Unrecognized SASL PLAIN Callback"); + } + } + } + } + + public static class Server implements CallbackHandler { + String user = null; + String password = null; + + Server(String user, String password) { + this.user = user; + this.password = password; + } + + @Override + public void handle(Callback[] callbacks) + throws UnsupportedCallbackException, SaslException { + NameCallback nc = null; + PasswordCallback pc = null; + AuthorizeCallback ac = null; + + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) { + nc = (NameCallback)callback; + assertEquals(user, nc.getName()); + } else if (callback instanceof PasswordCallback) { + pc = (PasswordCallback)callback; + if (!password.equals(new String(pc.getPassword()))) { + throw new IllegalArgumentException("wrong password"); + } + } else if (callback instanceof AuthorizeCallback) { + ac = (AuthorizeCallback)callback; + assertEquals(user, ac.getAuthorizationID()); + assertEquals(user, ac.getAuthenticationID()); + ac.setAuthorized(true); + ac.setAuthorizedID(ac.getAuthenticationID()); + } else { + throw new UnsupportedCallbackException(callback, + "Unsupported SASL PLAIN Callback"); + } + } + assertNotNull(nc); + assertNotNull(pc); + assertNotNull(ac); + } + } + } + private static Pattern BadToken = Pattern.compile(".*DIGEST-MD5: digest response format violation.*"); private static Pattern KrbFailed = From 9d5205aadde9e188180bef4f29c1cfb6566f4772 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 9 Nov 2012 21:44:00 +0000 Subject: [PATCH 28/31] HADOOP-8860. Split MapReduce and YARN sections in documentation navigation. (tomwhite via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407658 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../src/site/apt/CLIMiniCluster.apt.vm | 0 .../src/site/apt/ClusterSetup.apt.vm | 0 .../src/site/apt/SingleCluster.apt.vm | 0 .../src/site/apt/Federation.apt.vm | 0 .../apt/HDFSHighAvailabilityWithNFS.apt.vm | 0 .../apt/HDFSHighAvailabilityWithQJM.apt.vm | 0 .../hadoop-hdfs}/src/site/apt/WebHDFS.apt.vm | 0 .../src/site/apt/EncryptedShuffle.apt.vm | 0 hadoop-project/src/site/apt/index.apt.vm | 6 ++-- hadoop-project/src/site/site.xml | 32 ++++++++++--------- .../src/site/apt/index.apt.vm | 4 +-- 12 files changed, 25 insertions(+), 20 deletions(-) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-common-project/hadoop-common}/src/site/apt/CLIMiniCluster.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-common-project/hadoop-common}/src/site/apt/ClusterSetup.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-common-project/hadoop-common}/src/site/apt/SingleCluster.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-hdfs-project/hadoop-hdfs}/src/site/apt/Federation.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-hdfs-project/hadoop-hdfs}/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-hdfs-project/hadoop-hdfs}/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-hdfs-project/hadoop-hdfs}/src/site/apt/WebHDFS.apt.vm (100%) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site => hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core}/src/site/apt/EncryptedShuffle.apt.vm (100%) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5ea2434aca8..b79403cd5d8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -132,6 +132,9 @@ Trunk (Unreleased) HADOOP-9004. Allow security unit tests to use external KDC. (Stephen Chu via suresh) + HADOOP-8860. Split MapReduce and YARN sections in documentation navigation. + (tomwhite via tucu) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm rename to hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm rename to hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm rename to hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm rename to hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm rename to hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm rename to hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebHDFS.apt.vm rename to hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm diff --git a/hadoop-project/src/site/apt/index.apt.vm b/hadoop-project/src/site/apt/index.apt.vm index 32e708ec542..aacae46da26 100644 --- a/hadoop-project/src/site/apt/index.apt.vm +++ b/hadoop-project/src/site/apt/index.apt.vm @@ -34,7 +34,7 @@ Apache Hadoop ${project.version} Namenodes. More details are available in the - {{{./hadoop-yarn/hadoop-yarn-site/Federation.html}HDFS Federation}} + {{{./hadoop-project-dist/hadoop-hdfs/Federation.html}HDFS Federation}} document. * {MapReduce NextGen aka YARN aka MRv2} @@ -65,9 +65,9 @@ Getting Started The Hadoop documentation includes the information you need to get started using Hadoop. Begin with the - {{{./hadoop-yarn/hadoop-yarn-site/SingleCluster.html}Single Node Setup}} which + {{{./hadoop-project-dist/hadoop-common/SingleCluster.html}Single Node Setup}} which shows you how to set up a single-node Hadoop installation. Then move on to the - {{{./hadoop-yarn/hadoop-yarn-site/ClusterSetup.html}Cluster Setup}} to learn how + {{{./hadoop-project-dist/hadoop-common/ClusterSetup.html}Cluster Setup}} to learn how to set up a multi-node Hadoop installation. diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index ae2ab37cad4..da976f635ef 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -48,34 +48,36 @@ - - - + + + - - - - + + + + - - + + + + + - + - - + - + - + @@ -97,8 +99,8 @@ - + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm index badd9155509..7215172b7f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm @@ -49,7 +49,7 @@ MapReduce NextGen aka YARN aka MRv2 * {{{./WebApplicationProxy.html}Web Application Proxy}} - * {{{./CLIMiniCluster.html}CLI MiniCluster}} + * {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}} - * {{{./EncryptedShuffle.html}Encrypted Shuffle}} + * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted Shuffle}} From e8d7891e780ee89a5adeacf8eb8df233f73c69fa Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 9 Nov 2012 21:47:12 +0000 Subject: [PATCH 29/31] HADOOP-8860. Amendment to CHANGES.txt (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407662 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b79403cd5d8..4416a152b80 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -132,9 +132,6 @@ Trunk (Unreleased) HADOOP-9004. Allow security unit tests to use external KDC. (Stephen Chu via suresh) - HADOOP-8860. Split MapReduce and YARN sections in documentation navigation. - (tomwhite via tucu) - BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. @@ -285,6 +282,7 @@ Trunk (Unreleased) HADOOP-7761. Improve the performance of raw comparisons. (todd) HADOOP-8589 ViewFs tests fail when tests and home dirs are nested (sanjay Radia) + Release 2.0.3-alpha - Unreleased INCOMPATIBLE CHANGES @@ -360,6 +358,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby) + HADOOP-8860. Split MapReduce and YARN sections in documentation navigation. + (tomwhite via tucu) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang From 3ccc76c29b88d010c176264a1ea0eae06c2cde14 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Fri, 9 Nov 2012 22:14:06 +0000 Subject: [PATCH 30/31] MAPREDUCE-4666. JVM metrics for history server. (jlowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407669 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dfc9f9d73df..5d202dde222 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -589,6 +589,8 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4266. remove Ant remnants from MR (tgraves via bobby) + MAPREDUCE-4666. JVM metrics for history server (jlowe via jeagles) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java index 1eb62b05671..94d25f60f8b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java @@ -27,6 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; @@ -106,6 +108,8 @@ public class JobHistoryServer extends CompositeService { @Override public void start() { + DefaultMetricsSystem.initialize("JobHistoryServer"); + JvmMetrics.initSingleton("JobHistoryServer", null); try { jhsDTSecretManager.startThreads(); } catch(IOException io) { @@ -118,6 +122,7 @@ public class JobHistoryServer extends CompositeService { @Override public void stop() { jhsDTSecretManager.stopThreads(); + DefaultMetricsSystem.shutdown(); super.stop(); } From 7c9778e2c3bc030daa995a8fb630047739cd484b Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 9 Nov 2012 22:55:57 +0000 Subject: [PATCH 31/31] MAPREDUCE-4774. JobImpl does not handle asynchronous task events in FAILED state (jlowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407679 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/impl/JobImpl.java | 3 + .../v2/app/job/impl/TestJobImpl.java | 67 ++++++++++++++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5d202dde222..985cd28a6f6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -647,6 +647,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit (Mark Fuhs via bobby) + + MAPREDUCE-4774. JobImpl does not handle asynchronous task events in FAILED + state (jlowe via bobby) Release 0.23.4 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 4612d066ee3..19488b2e81e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -348,6 +348,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, .addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED, EnumSet.of(JobEventType.JOB_KILL, JobEventType.JOB_UPDATED_NODES, + JobEventType.JOB_TASK_COMPLETED, + JobEventType.JOB_TASK_ATTEMPT_COMPLETED, + JobEventType.JOB_MAP_TASK_RESCHEDULED, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from KILLED state diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index 7a77a6b2088..92742274a9f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -42,6 +43,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.Task; @@ -51,10 +53,14 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.InitTransition; import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.JobNoTasksCompletedTransition; import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; +import org.apache.hadoop.yarn.state.StateMachine; +import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -340,7 +346,7 @@ public class TestJobImpl { return isUber; } - private InitTransition getInitTransition() { + private static InitTransition getInitTransition() { InitTransition initTransition = new InitTransition() { @Override protected TaskSplitMetaInfo[] createSplits(JobImpl job, JobId jobId) { @@ -350,4 +356,63 @@ public class TestJobImpl { }; return initTransition; } + + @Test + public void testTransitionsAtFailed() throws IOException { + Configuration conf = new Configuration(); + JobID jobID = JobID.forName("job_1234567890000_0001"); + JobId jobId = TypeConverter.toYarn(jobID); + OutputCommitter committer = mock(OutputCommitter.class); + doThrow(new IOException("forcefail")) + .when(committer).setupJob(any(JobContext.class)); + InlineDispatcher dispatcher = new InlineDispatcher(); + JobImpl job = new StubbedJob(jobId, Records + .newRecord(ApplicationAttemptId.class), conf, + dispatcher.getEventHandler(), committer, true, null); + + dispatcher.register(JobEventType.class, job); + job.handle(new JobEvent(jobId, JobEventType.JOB_INIT)); + Assert.assertEquals(JobState.FAILED, job.getState()); + + job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED)); + Assert.assertEquals(JobState.FAILED, job.getState()); + job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED)); + Assert.assertEquals(JobState.FAILED, job.getState()); + job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED)); + Assert.assertEquals(JobState.FAILED, job.getState()); + job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)); + Assert.assertEquals(JobState.FAILED, job.getState()); + } + + private static class StubbedJob extends JobImpl { + //override the init transition + private final InitTransition initTransition = getInitTransition(); + StateMachineFactory localFactory + = stateMachineFactory.addTransition(JobStateInternal.NEW, + EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED), + JobEventType.JOB_INIT, + // This is abusive. + initTransition); + + private final StateMachine + localStateMachine; + + @Override + protected StateMachine getStateMachine() { + return localStateMachine; + } + + public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId, + Configuration conf, EventHandler eventHandler, + OutputCommitter committer, boolean newApiCommitter, String user) { + super(jobId, applicationAttemptId, conf, eventHandler, + null, new JobTokenSecretManager(), new Credentials(), + new SystemClock(), null, MRAppMetrics.create(), committer, + newApiCommitter, user, System.currentTimeMillis(), null, null); + + // This "this leak" is okay because the retained pointer is in an + // instance variable. + localStateMachine = localFactory.make(this); + } + } }

(dQXy$M01DsODN@^sF z5lcCmQJ=Ok{br!&u5-b7`WP$TKZ!jeQ(5^&O?NkX|Jx4?(Ex^BNb>u&=@+13@%Ys4 z5;a@@e@;SuxM}rb`^uh=ZDeQXooF{>XvqFA!t;NjrEP`w<6;S>7L?ipHO6tYLC2mZDk-~S0#$f2Zl;a*G*YGT4Xv5a9~wCng8^`KPA?O!*DP*G9e zM5q!b9o*OB9o!emxf)5Xeu?yp2{ zN6-S-K}3;KOkYgu@N(9=+*R(Th-fjhI@D;4daI^x5JxvVt5bsj-iw~odUgC{|g{^G&oa6(HfF*9G&=C40A${&J?>8nR8?Ed5V(r~_Rpzn*rNCKl`0g~;id z?dNF_5Ww)jel@DlL4z-~RH#EBpp5^Sm^|&LVT0P218C|bW|Rb1VI%zW^exT!m#nv znNthI#f45HI&bJVCSOlxkj@yiEH(^bM!#HNI@b@72?9%>L~2oN#N&H0`Q%`UL%xa` zNC#FrMqqIjp2Z82u42lY&n-0-)(1s_JW*lN;dh={8D81<3p% zmAIdXX-#pNZ_%}@$`^_0UPUyXoHqJ0V{4t(pv^Jw@I_8+yn9W|0so$wh6bOUaaujF zZs!3#uA;}c36&>mozrPj)Y;&M!_Z1rfwr^!1lMoB6%a4Ke2Bwx=aOKZ9#J*J7sx-Xq)1=ZK z+1>8w_lONYg*B!LSP^lm(^)tFwiv17t^Tp<@p<$hj0yaG z>${Q*Svg_%s&(1x4w;<=NWWiAP?1Y+@pFoR>s@e#)9a>uARXqYILPESk|t29GBaQ& z$3*JPf>P8qAtp(`7axsA~-!W#oe|9@JReK<2t9Xh#jjmmiEFT2|$ zQa$;ndW`+&56_f_I!^{u_m$ky0+?UFr`-?TP4dt^|H0^%fUs1wtaN+toGdtsoqT0< zIJ_qE1LuxNv3O(TwZ6=4Ly8^wIg*|4m?acgWzK7{C{?IxM91^TnUB@Gup*JUV|`gk z(a%9#E-|9Uj~0SmN%C%~^0c<#_vd6>{@>^4e=$>P3O}}ay-uXqe8t1CZy61DoG2rR zxo|#yCBVQo-3K>JKSk0mV2=L9RE%>n6SBe!#T&ypBrBG_oSYouC{Z?j&KS=`tZ$}{ zNr4=JRj1RA(3G5xuBygl5*4 zx;}aSs5-k>4#(D=Dl=sT`XNl%wB}9}5aB(MY$4YI@m+GbO4g4_fjC(q4)F zSamDtFDeZbH`0teTJViopG-Y~Nwg%`t*oG-NNuJ0l#nO9=%&g8ro%-X3~}N$m9b*G zB{`H$23;<3Gq6n&kgcjIcK#VOb|F~*2{@4T3Z56ZQnvWBHmIXUD;!DA`7#==>q!+) z{)e)Z7yg8gp2J~zP~d7toR9v>81Cd%hL+~YQb=vb$7*y83!XJ}wL$9w^H%-MPGC0d zjSXFoqNJGB88=r-=))kLx1a%L%vIKOc77fD8S{*+6Vz!vKd>Mw;rDL=xmJedWwY2UvamB%A+fzP>iF%uZ2kM zCbp4?BqKpEM4{1FCO?9N zNWbp!_|kONE^3nfBJf`yjyc*~X?tPwLmYPj4~-!KckpegvuXEJN(ZQR|&cNw_r{!`?_x5pCulIERH-3N1V)a>z}Od>>Z zpFK<7BdX+=5bh?$_fqjKT&@Rg4Z(VhaPEN_n@J-s{PIwE) z-2s{Jq`Iy;wua5uz^d!Wv&f@g`%}=MKAmoI1clbyqP zW%Q`JLZn_~>zSo(@NSi^?}hlEL^Jc0wm{MHm_hbsUc!<%zfsvm=kIL$Bs@sP7Ow?} z$E`Z4MKE`!n}}<|`-d^wibuGmq$w(AjhB|AOFwW4k{G;qpN~#S6ag4CjvE%pP$<)~ zCwg;uC22nnLSsa4FJ24Rd{)O_!Nqe0EUIFk7(fA%GS#EqWO4K!Yj za&Pn!>kt@A3>fkf3>Pdf>swjAR&mT3uR(LeSR7a&!Vgr`O8A(%a7Ay9*UGoaYHgFD z;bkW?xB&$OD8zn6G}jls3Oesb*9&MXJAzCSi#lcDyej?PnzE zpLuRLvV}wNK}*=Jfau#Wrca{2YtV(Rxds?D@Qpo}qD@(33AK`5_EllO?uv1Ug{w^Q z@6<@=GVgNq{eMdi%1c&6YceAHoA9H29#9H8QGU6x5{?P1O=`WYI-l`}CRZupxTev! z#=lkdS$*Z-U)@e9Vp>WwU`}8|m#7U1GL=kS#1k4w^e7GP&)brZNL<{Ls?8@d7AH<{dbg)I zzQBTLW>Qmo#K>Po!c`uz8^V_ZjW>~f^TuU@~8_9wv4D!;=?TROdtF3laCBja4puB>#(Au&`bO8W+|dXlhUVQo1ARt2naT7v@`V&eHd{($L7p)rP*sS)k>~VKM8dhn35z zwWpC})UXq%a({)d2W%qN@TQ_}|J6nhM0-)wLqTqS=zZ$%cjvhlpU4X`?hc2$O=y)( z2VLfIg%WTFr@2=C<2c`u234dmXrXleeeqfiQ?7@fhgJO2gx)_w>*SBUX&Y97A_-qx zgDzS$`^O@WL>rVGP=ms3WRYMCOwKK%?gIl4;8|lQ`lEBmMht(9-F-HnqzulO3j3pT zKg1JWgY*@XtT#tJa-cD!8?i8s(&=f6I8~=?`DuusNh8Fc@1E# z=DqxdlcuK4A@p-TwCHh&(-Z5%nbqZclhxE%0dX5X3b8Q)%@co^#N`axKq*!9`{kxG zs^$ot)IUERGz#-u;jKg|191XDNK{F`ql4jwB}55iK%zYDJ+0!Z2r`U(7u-fsJKg+o z%eJtO$2@qb}4}5Z#WJNdV?#%FQ*kpBYG|% zXG<<8Z-jPq$`p&)x`r#bJa@oMpvJny9lP9R6R7by3ZB3ag;c6! zp-%k>#B7pB+>EsXrs+njRBOpb?TlXwxxu0h=f_?Cs7=l4Y0^w`84xR5%}Yb#DjdBp zi?`)V!SEa1KjvSuLA6b9_vb<3dN%8CbKkZQ?JLJ!LFm z#&%&wO*5UnB$lpTg<+H{I;b)D7b;+0ow7>fH}8GxS9v^8S&aw<}f)bKTOrHSZ8Xv65WwLuK39;5Q!vpB_M3r}+kQuLGyVWxey~Y6tH}kI+ z2YMShBY->~HsB#pK-7)L<-{3nXrEu{_(sxMj)i(3+K5Er+H3Dp2QB1-Xo!Fg-eRZg z0#&8~aB?ZDjPsLW!&dOTuslOa!hVO#UiCiSD4^3~fhov+9_~dnLo-H-L2j$d6kEF} z=B>o0j>4~-<1@{#t{9kiSV(H;*&MmxYpAm#CTLFbo(6!p1TOak|KqI&<5#N}EWB+@ zC#Gj~4n%PQ3WN(53?(kF?G0sq+bc7zbf4I4Phx}55`bC9hThd8Sd9|8E7l<1%cLZY z5$)6zJ@#O4YbplSqbw{sTYW0*){585ed6q-p|{XO8Q?fId{|WZsR3!cfGY#UMif1r z&9vsLC{A6j{RX}E_V8qU!!wm=J88Z0yQQG1vl)WOZ)Av)#|353v};b{H|JN*=CrCOBWwQVccD?`XmyJ!^YR7m=_1kO%F}W^ZkK z!zm~d`pZMU3FtF-G|mquUT1S@guI8HFA@XI)92f0rXuf-ewwdp@po>OS6l2YAs|Hc}&@T-XYJEx*; zzj&^s4hHqF(g;N>Ajy2qj5X+iSCuHMEZq$nlw~%I-9(W^dfOxoeFtM5NiFEIEt-}H zBO5wV2m?Cv$4}`Gz^a zzMDYrHE@=KGB&Cp+sQnmHz(9kKp^-|?3X$l;LX)$&31+!=@9Kg5$03u2d-V+bc<;$ zBk}V7sDi3fN7ow!B~}Ta+2=5<3)57L5Tq zOsKr5iXonh&mT~@t*o4Ll2P)t8ZSS^BYXeAM0HR!w5$`(xQEv3x>-uMlw; z&8rdM4ErmLjvpFFPy zEzoJ3STUVkwlW}Eu&w{Evg-Bb)Mj`yOb6?`HX_TWyA2ibJZ!Kww11>}`>h{PkNunB zirTTv8M}WMVvN4(3Tt!+Y2(#+j3OaH7oH)-8@)|#psXGo&X*yxc2Aent4ZuWi2K5R zN%^s?p|uCauu8L3l(z8z-Z4W5#_gurT|ABCM6>2vscc(MjKx&IfFiy0-?7M2bAJ!a zSkA-c@f(jpFN)d+Tspj4^M0w@-}Kxr1hLLOYS-WZx|+;1gKT;v1MuZ&j2 zd>5`=LWoKMiblsx+MOZ#7=KS?A2E|_0;so>?dlud(Cc)3H-j6~h)@1Z zb5g46pWck?>$*GKR$^rLI7ZZo{+fjkIG3#5viJ`?fWE8*E`VYeh^2TPT1Jmel5NCZ zV^hWXV&Zd*hzF4mhczqR* z>-5^O{T(553a{ayEAW`=xNP#j_}ShX{pZ|X8LLT}W1!Pk=1-_;EAE7^jsFrK9c@2s zM255}QSl=EZsXs&$1>^!Y)y5xv-n2kjdLkQXJO5!s(Q3BX<2nsXV%mYaVln-7}=C`i?U#%-uH4b9n>yIR~ z(U2?%4=|K6IFrT(Yw~^b5;}9)51%#<(h#EC6AW0fp!@a!99|rRaenyY+GnL>_0@mc zv*DE~yz&r*ae{$AJ`K>!ZkAyjyEMB|#B7Kw-ZnjFZ`*yf%kH03W9ctcknk70WMdn| z45z>7veGUgHXT$6>U;|FiYGJPKM}Mod*tNlXny*YQ`d|q5sz)C&8H0}@m-1$e>H%8 zGIq;ujmd#pd0Y<h0LOu*2BiZ^hW>vAu+Ry{`EOTfWm% zjqOY>%|8*YC*eObtoRrNl<3FQ05?fL*zYYP0QA{|J6%{ikz@Jy)0#b}J-ZUfs0QkB zKP|88Q2CBWqce5YwTj`;DFN?HO3P-R(KSQiPWkOeP8JG>vIVOJp$1Lw8)!Ydl^*=t zW(k=U>GJ^aqfgMrJ3KdyT;z5;UE~tHUrAu-t@3u|T?j$^H(Nx0Mws|)g7jK6H7JI0 zLyQv|$wbuPbn}QTy$`r+}NYxo;GyCIO*9&05h##5xK8B!{fMj4H%PLE!soqyf$}UxIe) zMGibG+&sPU6+YZIg?`00`W<%BZn9by1Ms$^Si@Jdmylyad;3o|gzH0Uwe8+h{ehu+ zUGQxg?z$mXZo}MevZH4sBh~yNEVL zhl*5FRR$z`|CXbk@^Ol9q6$=!P_q!54l|rR=VRhGtDk2=<(J<6`c}-VX?$7uFFk&wqlxZp8$8BJzFM{8Z^D zruKa8;_|dV!#w1=&VD+KbPVF!{eVJTb-@U zbLJAQ{&giz&rB+83}2q&%9MUU#WTwC$pV`KQMI{gIj(F$rj4o1S8|*+`bQMOS>w2I zJ~n+`oN~nNKW%td0y+E=KTer1^LoeZPiZ%xwmVW_EgeN}2;kBVylj{w;7s*#|ESx^ zc=#Ez*JP{?W_18s^A=N;o0()C=|Sxp^v{}ONG ztX@2Tp26^*SA}=_-0U-h+ov(czM!`ivkus1gZ^7hZKyE;>ozYxwu?AV~!mL+q zy3(~BZLVBCp@Z6QLbkcSq|dIr8-T|JGT@^KOK)S|8nEhQKFg;Y84W6tcjXai-& zi(ednsEc&HEs_Cyww4?&Tg?GUaf=7rP>I5c!%a$XLbVWF^&GyYf zaJQPTkhx5zyfq&^o_3>v9q@>NMly?Y1jlyM;L)fw%CAsOj8De@sIIdD_D{T=aqN}w z-}YdquePy!R||L<_orrCjwjZz|HaG~8!jeyjPP~nvf-=!Ln9@h`dAb^Oc$HEHhGtV`$>#V+3d@nm*Q>4qcSQ7z#E7%;n!;yAls+K z6@2jX-sft%vJ4Si%Xy;O*3NTjmWLb5h0@4vIY6g=^Cns&@?&tV|h^j0y z1aC9yyyGTOR^5y!x+OoPAx$WK%Yf;J4rL?(lbCz$X~K(H9B2odz2x`+0`UWBEFm)p zvlz30`jd?E2H3ffrOEJz3@jU)r+guA9<05I1IUJN*}%_RwwAMc^U&G&1T}p(z^p5O zFo2IW!xdgu{fz*!xWfBIea}zEqV2+g35R|kXw7MmXv9!#6XbA=G|ZY4Vhr@@Cn8?(5?m=j^!8 zDZ^dP@L?BypWPC?nS<)Xm17NBqsNw)Bwa@sxpG>_z?y2eeYuSHAjz3tPVeMt&sSXv zvcTwnL0OjDDhr3+Pb#DnvS*3oSP?G$AE7PABDi^rUtb*-G6aLi5SPf8XEykAZ*1`VgT#-^nq>97I{T~d_R5C)I&R-_ zSguJu-T6J8X?!6*xN7@i;Vp$NtV#aoLs%Cx7>Oq45RIk5>DK%50<4%-~ zLQ8~?y-F#D*h&4iJA(B-s^#FbpO~>??%$I=Pp&DRr_IIQo8c*}x5urumu~xL$Yu{W zi#!_}Lytc)d7c6^jQ$=-_56KEd@G>zuFmQPYLWT4e}zbw`}h7eM6tl_aZI2$q^Iz= z&fw_d-PZBb#XEI9g*DKSyW#!mRM<$Rp3-+D#)W(z%^TuUM&)sGxUC*QM zdTb}&-h>5a+g7s06L#shpK+^U_VmT= z@nY*s`Ij}p{&lk$sb~=Q{S91b_WsMp{p|+u{l!D3+d#wL4YSaor%Tw)qs#lj$Kew_ z$=wdR^(R@J%#f15WU)QdAqiF+dp(1?&j;~msp99?Tb~bp&xh-flEVZkR|LP6zl(Kn zsFo%_?S(H&WcX}%OL*joTGStfK8IQuD~3V^9*JN@b^=@^_~7eGM5NA_dKjxc@(aJh z!ud%TMVadEJ!u)IaM3Qo70hHB*4NG4>q(Jm@p|Z3V!Y#!f_#l4qq>k6Pm zZT|VozyXZ75fRD#ClDV?Y)6ZuxUfDxU{aNpHQK6&?=4AQmjnt{v4NaSNmIwlI(rPIq+%eTIB_8+fRF>a!~AW$J2*2=hZ5VETp>bi zz+XiJ+f4V>zVBi|Fm$;@$zOb34H2B@${(U@@8V`7d znC{m5rMFF#Ht-4FhHwf=PEa~-m3q94HA1--dAhu^8@U<2K4!U{corJp9@tdekCOTu zCe<18$?ReKuUpUG$6UI{`R=Y2UhJ zj-a#fU<*@>aZ|^pk+Eg?gbq$OP~Fa+%Pa4d1m#bi*^r$(LmD1WNaVN>)ZW4!p1Z^t z)@Fjbf!pU2rPY|-?wLs;wil61x)bYDR^#MwgybvbK=_|TxF={ z3Kxrmx<=(r%;cS9NGaH~ zZ-hk2a`L%=vEDS5$8ArG3E7jk{^-$-mK9cyO95UnY4zOmGo4?RF7Z+*!l)Vf{4!=} zLsm`6_rW7meXZ3kMyh;zP-dK5@e*blyN?WX*P#%pS4<3aXy#C7 zM-VA=%2aZ{D|VIC8WPziSTynziT( z)8mQFke6=2jUa)W_g3z)9?A|J%O zW%MxV?74xVZLRW4p5s&FvJs#5*vBQ10i$B(ypln>o|}2X3dg~o(Vd|+>H!Z+6kRUO zG=}7L<+=u~{D@|aEFxzQ@Jj!Y5fdmstKQ=K?=v>kWpLTq3Ul8O?+H_tAUW+4^sxD! z^zYmE&b8}gY2A5~K0*tp(zPqpGJk|qWkZH`_jSy{s!6)$6E7BnH|r&}o_y0PEs!~8 z?plNL-q};K_zC)=73y;8GE#!4x7pEec3!+u&ArjITDP%rW!p5UPP=>!7pbOSN_oHt5G*(n_Xqy_ z{Of&GRevLYzgT3${zXa^o7Q%QWEAV~_yg6Yqz{>6#@Za{>=_LEv8aAhc%y3$vKK5( z?!UVD*b@F2Usd_h##X~e_Gq^vo7d|Ywlnag3{N>oij zXE%%Yt9=LkH;e1x*VJ^xvM4%B^Na+Ks3+YU46xsTtYMLOj!njIrwQ_U$WMs!I^Se^W@hbQ#qzv=)Me#pgLJi%isS;UjU*m zlVU0=kNit+K-Yz@r8}>{c7Eczb3g=Fmav+Ltf+Pv7ihhaK|=Mt!RyVtaPT=EeD#u7 ze<#U)AP$O@wA&eVE?{&mBQ87`P|sv>BC8U8N>Y>VBvSO@LBsx_yX0oh3${O2b1lD} zxvtGA#!h~jemu4go{gP{8S1(yN2HIH<{d5Qco~D%VBu?fDeh@EZ~UUKg@j8Xz|*7M zY9BZHeeRxzm-fO8Vc8_Q5YJx|{0F1cQ{->iq&U@8Kl5J``E{zd=M!^ku%Peh{d7x- zX+4nRufV7b!50M;NDgWv@4R}9SOw?N=hTi!Nd7t*rLwpbBdx{^21<8dD5hW@G0x z?UTd?pi8YK&X?O!6W!8qWy5Z3U$PYcG?;sPT zRoz2Xoe$36dssw6mvA4MXtZ9>-`~7P+o7$~D&NrQ?c%XiDji`Y%6? zd9_KGm)jt5w^}6;ujKNLRmUHan?ugzV$KD7`5qB2NG3%X{7j!g*0+UgmQD-F4`-gMGpvE{ERIrygP%`v+ywx;&(UXk4-Q;3=HKUX{BTxS3?!5BAq_e(a z`q23=e;sIPz3;x?-OrF5?(u}>%!q5m`dD6>nbb7^+IU*gjz4t}wotgD?RkB8!2)D= zJqdC4%hLkN%=$6tP-Gh$zPPJD!t+(A1sndjW ztxHy{?`WXp{}1zA8j4rO?tdYZ>G;YH_s%J&q#K!s9#BkD`_9C$d{8? znIKZ<`51EfF8myzmdn_Phk1!*=kLK&#{658L(k`m0@<>_60big>UPd>IaZ*F}CiV4X;pR zQ%9$Dox|lhdE2IGF`n%z<(tNK zZ((3t%TZNAnpUP(1LdMYVl~2HXHvEJTMbuPtWLJ;)0x>$Oo%^AGCAGkqMm;;d+G7Q zP59;oy)968BAeA1(jmlD$W48Fe zDm4{2(c0@n2P0^KA%lq)7?jw~=GKW(8Qq2s%GS`z@gMnE>*&vkImO;ngf2VXD|Yxn zziDcuFqx8}E-0+KPxC(4N~&AaJQ#*qqUJU!2kT(ZV)wJv|9x|vNXUJr-AuGjgkpaE O2Jp1M_){!ZfbPGH diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/goldFoldedTrace.json.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/goldFoldedTrace.json.gz deleted file mode 100644 index 0d3a3b836152b019a97de56442cd4587381783e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 159847 zcmZ^~bwC`?(k_fckYK@G5&{I503i?{L4pN$SlrzuEE*uVySux)ySqbh3+~JAx9~gX zyYIR0xqr~l(B0G1QuS2LP8CHk3QC0CCs`PHV`~coQRttcfufzRz9F-jy|oq0@smb1 zZM`is`udT|``x7}xCkCk=bEpr=X|t_LUNugEzUf@wxc-bY<{`(Y*pw7Kpq1QwO2!n zgz<0!_yF&AP?C<7#7kzCt|>9cqwj&h^I@H~hlB3qTZ%Og(5Xs02=a6>O98o07WBA& zRDrO9Hm1kNDVQn1*Z23Qpoc|Z3*>KjInaY#!1MOtWO>;ba%#;&@Bn;t?o8N{QXTVu zavlRg${2y*tJQF80nf+XrUbOyr?+!D$e@eM({;$*bbk2h`V)v{oC19IkPo>{g_Hw9 zr{VF%K)lxNbitpR?Yqf)6aqgXDOC=w3}(kCN3y4FkmW09BvZ0x5QxQkt@YmZ#=bC? zBR(B+a(^ugbO9e7owh8d2T0bmFoW*ETSxU^*#?gR3JNV?{BZs}+B)!IA2=U;uL69y zwD!EbP6s=*V^csL^N%pW2gSgr!-oS*YXN>BLfwY0{ z`sW3IK0fTVw|m?GpTxRD@9!SEkK4R#l|kLtKH;8&!x-dXF1rB8Q$Nu7*t1wC&cprj z8hkd}z9OJ=*X2cWYuB5+y!vo&OJPL;yaYWxVhV-;&%mDRdg=YjE#SNJ`~Gx+ur|L< z?RE(Gy8q+pItInMKnp({T1i5ba9M!!Gm;|STF~K{<&Zj_^IGCw>Oz;Ev z1hX6RcxH456nt=poOeGugLR&?Jg$c!PkOgN@Mhi7`Wgguek0uKmb45MCTULwoq;a% z>nV(7M}}JK)Ege(qScG@P*E6xZw^?1u8*MR;oNaSKb_OF>01f`X3+jlGtf0j8*F~! z{os6aw*$H9*70mx*|$#9(LuNG>jmXQ(vhpf$KxLdxX{35_2b#Io#l=YwF>UL^KcfM zH~i9A;Is6tall1W+ec4zv@5V8LF4N`AI_W6Qo`4# zj{0HA*MRJR;dFsE@ZG`QBnLS7F^>hP;N>tMz77N}Um|}1yN1NOfF5mrYP!CGfUmCW zfijdC0U7{f;g03aG#GmG03fQnKRps0w;z3O!}pFigs@H+dHBpJl*%9mPM6iRUrJ}F zYZZhJy%UA6?0P3&p=ou7qg4j?dXoCb@z_ZN+kDE@8m<{VDJ`D58Q!>_4peoZjNt`~ zOj>(Fu2q0i-jl1q&uE0YD{L4lv`wE3z$ZJ>WL3+rWlb)3ROxaj_;#WSAF*rmB}EJ7 z;>DcaSGBrm3j!b3SRU?K(tjn~79Y{vB(H<5jsu?V9@^JHcL)8O9CQeG)$!hS--)w7 zG;dCZgQ+|l_f;;7wZ{%sD74!jACsMdpOMXbCGFgJ3CkPTsQ_7D<2j0 zDrqD?7t7B@b5U=x2FssRl(FpgmYdgom=5@pH{mD7Wka*GuKZNt;_B z^td&}d($8t&05FfnjV(2B|azhKLje4WS^MMgNy6RG&F1fvnDJx?hg8lei=#7pV?9Z zsz}J6xAYfw%6~d+#;5-f%mZ%o_b}HzT`obF^`{U{%3H41bL;l)cBAR)HJ$rr%<}g3 zx3T<7+Mb?|b5;kjNBNMw>C1fE*vH5EbWgBLP%z{yJpAnT=KT7R=xJMU4fwHj0m#{Q zMgXzD$FDEzWNcISbyn9on*BVxf!7S?Pa~~fIUjcSE$u{3N8}AV!tcl194%!{9sx0U zN=_6`QS&Dxk55U@%`-q`nNB5i-i25sUHx3#cxCxne9JI?69Y{l<2;#mWD9GBXODzm zzL-}BCbMJc9?=q~-Fp~I0z^zFeRnBw7;kQAx$W~74eiy&B-fr4?D+JS^Kdr?&UfiB zP%i)3mZ4@zExNTgWCOf(PbWsB(>$R&xag(7$d!QMB@O6S_vQkAL7Yg3$C2!FYpKaP zCjFtwy1K)Qr3@h~sGDtn5j!qA?>KtlIO3F)Ct0{knF0BZ26;EnZ`s*j#=kJT6y!J? z&JTIbsZz~Ldh6q&1$5qJ{gv?@mX7M;{Pa2`HLKR-_go}KgQe@*l{2XcO2|eSyb2+{ zP1)DLM=erhtDezo9bYqd!Up7z1o|z9?`ry>Mg_I>fPR;6nCwA%qMtpFzvAd zfv^2xi0<#By!{mb4${s6UhSWUDx9!k)9-XT5=~tZR5`p&N-M!ynLmU4L-fO7hAUG# z`bmn>!eKUhP>$Sr;I6i%2*v27o*GVEB#C~6VKeueHzJ_geLotCx<<>_i zqkw}OS)!s$RXPgj80hGtVr|N67U@_gAFOc0@Js&tt~`ZRdfcn==RD_;LlAuzt1tcV z*iRI%S0Nj(VKs=6-!JN@i_6G3t|j1tt2%L)+(xM~5D;7%Y@#>3F1)l@Fipy85+NS( z;$L{Uk?#WtEL4y8B$48%z0Z>VzJ>+5(@G86?*w!pvUQ7p5ntuTnTU2)Zwfux0z&j( z!=~!cFuyS~-*T}Y%8;~U89?NntP%F$g80>G*~56DSF*bK)@+1}>(;{qwDK_eD6oka zK)*j1!Of|+D&rBhaT?7J!|9Wt^>^H5fycZzndyHip-}}lz(&|1@5i7PiSw7$74(36IfnE zW-tGj!GRp5BCwF^Gw(`bRH(fO%P0QKQ57;ItU5mR2I&hg_DW4b^af|NMY#ZP!=cei zuq~k~9i?4oR#0v?z(Ece;N31puA5ATYk5Cedek%jJxHAqeD6pzh?ept5Uc1P#q5hE1Dr0c;*@8Ok z6>sqT<{g}|?v`mRo2$;}dVSC`1-D-l)%>WLXcM-NcCz$uUNVic_qTLA(%Hf^uidt8 z1D~V=FRvdq0kjPYrf3yf*3Q|G;mo|`FV+yIB_(69Cl#v;Yhaw)+C#AvY#y*f(FeXF zXtPoEV9v4^WZXHm(ckPGAeG^Q+T&WZcft=yn2wqI0@@y>x?HkjjEfw(tH|^FR=dN! zr`JXUYbO6oYHywaB$=zkunHp@tOT*!?CLFB*;HyosVxIYQ!4uR-xoJ>I3|H>XJS8n z`c3W;AM)PK@e525;16SNJD;ABccFIR@-;!(QK7;l4$q5oDNnC^5g0W3=?i zB9$$y=U#%bYtuP1wKN~;oVhN0Q`0x13Z?%XE~7rEWuYl$ z^@^|LsjFped2SG^BX7dMhu4NaZ5F#D`cM>)0hS+hNVp-PJ;_Va&Nqjq`u>=P+!857 zhx`Z$U8=xt1TBumCr!iQ!s&EEX7thYQFQbXwx<;7)8X1_NErC_J>3nU=&on%c^M_{ zf}T*bSo!zq#Y{EbZb7Y?R(*rnzgr@4H9nO%Kga`sn6@c6+a; z+kaOUy8kmmYyPhScfs8-`Ijfy5&pOZFMrN@o+5N4gpn*n$sk+b|vbX&~?$*;MYaI7C^lAUv1E3+- zzpmQN*j}y*W?vqgUJ87QZ~g3eTCVobs8;_pL&x3w`vjP4x`hugM{O$p`*iVLO*N)S z-18|pV)c=N0x@y+$iJWK)R=fV3q#6KAF0sWdA>!EddXa>IX$Mc-e46<%(0!h_(rui zGf_rF{PIU7j}d@3#T=^?B|3G?oI-cqEqn@5nu!b?WHP0i^jt*n(U(tN>eC@Ss={-T z_>yz3{AG9sWPQ!qNz10@<`6_cFT6NW5k{>T;U(Vtf*i?fT0$#@GrePfpRbPxXb7@W zPv>KCJ#{N?{9br7>CK;3c&jq$p9}}$xOKuTrra+F{|5Um7y%7v@Uf|u_r_=+vg+%< z2i|*&j$_5z-ZT&DBXe0wm6nb7AFe zwnQ~59Y2ZJ)Y%f%eN;(p?%?*~SN1(jpI5?Q$m-ZVzroi?g@=noosqOoB#0rVO(w~D z!!ek)f(k?TD_yf8;|rONe;>1a2D2~${f}x!y-a(*+a3l6J?Go&4?nK_UG>L;Vm{_Q z-Cv2LpX`QWXI<}v$Z;Ky@HXIDVLr@B{`|0rUD||-o>w+*5NUwi3Z*y7T z>PC=6%`ydAA3fCw`r$j(25!GIyAQ14s)PDe61yjxs;XO;b>RdnwAxtl6m{Jvg+IR- zBmTJ2=drtqhArN1Xz$753^!;vlo`~`TjliIuw3*tnvsUrZ`(DnVMvr&o_C%E(fKuw<&MwvVTZ>&{>o%g+@nF>fK0_;ELOG8%<249F?KLhQB2x{NZbzacZ=>?-EXY+d@w+?dMa^F>1)E+8O3pD*n37?VE!7xsu#MKH*)x z5BFo!!DTAQu&Z!LE?I4F>X_Z!^NFOyMMYx8cI?{MtdJ^Jtie)0xYZ;C;w8_g?p0qn zNLmZ%wVU2|4Vl(ZPIQA`^ATCurG=PK2{fZ)Gx2=pgKOd;%5#qSG+)W__ zX%dM2(MJZ5{BpP+if--ihN&3)F8c0( z%1R`moR>(mDF4(ZdBZDu^K7a5yc37kPJ*FVDOQ;^@;A}UUWbyBMA;z{oyZ+EdoX!t zf-QH5^Mv%#IQ9o2<6KNWvWqPwFZs8=T&qe%%ppFM-Gvi>zay~(ea;M>H;Dc~du@sI zdOrZx=4z0#)S)wB&*fzC8v{yqv~D0#)DtTl<7ygubh#c-IqFNi)nrNL8JR4mRsnnY z7#S;18UsFy`z*={ zQc`jzf7vcHQRwn0L*@-4bEMg$Vs;;mBYQS0eQ*Sn#SfMovf3zx8fahOvvwv_4&2n1D0tAjQ*Szp$LQ5ITm4BM|48t2=HDN_&a4{E~+fu|%mPA%eYV z@Y7(?ADh8yhqXcJn6%tr&L%=KjNgj@%|&_Q1Kr&9?9S00a*gkVJv^BDNl_AeyOWj} zKQw16|0MeIxt0Jf5{vR!l?fZpSIJPL)6_%MENp`KaWxaZ3agg$;aF5vQQpCd!F?1H zLnGe?vBdJ==qdTy?(rAvh2(x?d?J6Pr;=v&A!w$phIK8YuuNJi8*%L1BVYeDQZgCS znS54J0e%KcX2zPN_me{;r&7zQC}V`#LDB^ z+0HOtssj{?*0~5Tm-7r*`G^UqIa!R3!yvsbD)O19Y+q`TFHbbg5zGxE&@@ zk6P&PD%7ls? zw{-}osXc1d!TwI*&q?iS*6*J54xe^?#}EmKJ5$kis#B$b$1{sUn4z;4?EC~+bH!6) z2fGB|cPA{iy8*IJbLt{?Rn-!;^GJZoP4r;prq2XWB>`hxF3C8hweQ?{IvL(XTqx9@uxcg*;c8%R69uPyOzqeA%FnMkfZsib$y#VRpF#X+@D8URPIC0r0e>nA z@rHLhVxf#TCjrS7-qWl=n%&|kh##r7lZ_63T!vOims*0D@lkn)bUHn&}qT zp3f|m0P{)~{4Spebtq?ndh9B*rk8F6jBzn-PUgC>1NR<0W~%O*Ox9EHj7+|<$o5e} z%Io^%xxWrr`V`Wf2f0l~Iv?NiEe>w>!LCN#XJo@xiCqNho%kHe&fuF@h33ZNaWBdw zmGhE!AuR<%`Lsjsn+ZYP=(`{)`!vEiB!*nAiLr(4JH%@uljp72#kkBPL-yiI3ulcb zsF0pT^P-UXIntPMA@b`_nCl^anX?1)NdHBKZ%0)nzv(5{9-RIlgD)&kMkGU)@mk#h z?PktQ;h{a215=3krAzF0I)CMd&jfwYfn##i=!ijBZ#?SgLI`8b)1^^Q%$+XSMLbE+ z--i;`u&t(|M`BgmUf;j)TW=#kP5r3k8g^@GD~=4-iH&D3dQ#0;PH zDpI!XJNhYO_|Q63$3{eW)iVCd?i=<8artBaXs4yHGb8nYWw#TD#%L zchRt1g?#eA$Kie!RYmVhSxlB!BVnd%-Wl(w9T#xGiayn{&6!t;^E0_XKNFCWHY=ru zrw?TNMF;V8>O=8}}2NI63FZ-$6osw)BCaPOLI;<$V2e9VM*M##9^vOvGD zrcRg&-e&9}Wscj-zw)gXmu1VYK}AFq8_e{B*W_5hFkud)`kUE>k92KU1f}bC(zkzE zD0`yrnJxUOx>A-A?jI$F$tUp-!h{?f0X-BB%mc@qI5D;2RY;_#n4*;%6jv|3j3FsSac_;bm$7}+x|XvVuLHN1o=I74BG!wMBUyE4gvX=odnL%`0X z{;mb_{Qj=vWI_rwur^z9sku8tOsltx^aV<_>sII3C5B9yB`)GuCG7<9sxpz;%aAZX zt1Z>9V&65&dETUiBSLqZG5kHxWj3zFko$v^K$%O|s8r1Y5>1NKROFTMgr56_5^1D3 zj)`?MPvi_BAfQfK_Sinu`pRKsGD0w3?9D~wFQKJXG7S4TVKnxs88GgWOK&IO4XvZ4 zdgYVhI9%t~CGK#8MyyEQ8+}f{H@Fh^ktwauP7lCKi_T^2s7qQRXIf>Y*3zF%1j|8T zS6>EU_Av;Akg;gqpubtyjl1MA+6bWJ%9l2w!fA3Q0DAKUEsT(rl@1T9hu$;W~TxCU)to632|+kTJ6qHWCyYYx}Z=M=hn zkQ{gY=4gAn&bH?>^qN4`w0F_#$Xb56<jriRS&+uKLILEpUY@w#@DJ# ze_bbtT{Vw@`~<#1ky4tJ^!6lH6_DV)_ytEq6?D+jOR;3kg%n2~%aum=HKFSTCvlZ* zmX8n(;Of5e*UfHMKm`#Qn}DW0%m+6`kIQRo2q?LHUIvYg_2XTkOt5qMTALQaRV>otpR@r*gM&cC#P5&Wx+p4oKx#*0ZKWT06L`e6bxd8 z&t?b$LT;?^AVThT>XaIe*7rWQo#pJvj|-Sv=EIcQLCs;t*gN+Vb8|AxNz_l2I7<8p z>2+SWNFf-s$M%U!F4aW4kl!?&p3Vf$9i=HF{QEO3L_|gj36>5`L z+_fCYOnEikUU7mN^FoppyaTsz>O{8p!}i41df{+PxmJ;qDjAM=bS)?@A#2RJhwZiy zp4>0PkFr7=HAw0m@bMO}YDAd^?S;%+pFD7C0B4sa$s$u^H8ptQ*;cP=@(5BfYYZl+ zfv;*}d~)&)L#w3T(wM`4*bgIuFP$`M^;Y)uBc#jOzGw-w^5GJHX6 zh5>KNTF^`Cw_Wvi1_z7t@4&ZG$9HPPwJlo1qomQ~1%~{fCck2?)_sqHw{W@p)MmDy zTTQ~$)iGl`Xw6;`OdMqu^j@RR-pv$T?ra_qoOF=B#*wYSgsHB>$D-#9HO`tlzcwaHbUZ!T$c_C`n z1=}G55F{rw<6XI9&{70b>iCMGYnJ_cu#!r!#vU8}rkOM|AB71KMx|ulw^q|W0ePV- zy2!L9OEwkB65lzK>a{P7A}05`rR$$iYpzzs_EBrf_)3;J&#pqe7-Si#{7Kusy)IU) z6Jt-~KrkUPqBzxLHYvR|lXX{Lf{l+hgIH;CiJ5Xe)5N>CQuV>E z>I{(~AI(#h#xT_D-IS0_7Jt{oO+Dq+T3Il`9*A=9RB+8;MOPq(&{UsOQGYPr3S}7Y zTV8^ZBPpXvQ-smzW-Q-T3DGt8n0%!xDWf2Vy-;Q3CWlbB!20+OLjwnA(ZyVuuxamC zX-XbG%ebj#{x5GP{nOZ(GJtsztIX@Q0wJb2!Z2AEB>-qKj%{0zT^8C=%J&)Fq?BJRi+= zq&yo1@>^0!%3G4{7`!dRsvX6cHn~U4ZSzLl>_;>GOb~t>45f7Y1qxs4=>)5=Z~5Z` z7S**S8IMG=6a#r0p`wd-XchlMk~s1;b}YU<+)uyzj77Iswl!+j1+gdQ=fp-WjTks5 zsI_IQ!q#|SYfTa5f;l;~2WCxuZ(HKpeNmd?&_{QCl$kivi1M^(L1+o*7C(mHYQE?= z)2l!Oen}N9G6m&NU!gZ5aHL&p2#p+AU0 z;QE)^RF?sU+2SxYc2-oKE;sHhlyKA?NP?Y|;#~+5N`O{tiEI<>#hO8 zmMPJh_x5YOz|#aUe1-3g1sk%L7ZYq5|CbVjK1oJ@Zu7P{?6um!$shf0Y3I-T!Z_u@ z)NLf3D(c_xJW2&2vbfPk!lH{t^Pb2TtOCB&{>VOcvZlmC;6tR_=Gi+y(HJN2z#8Dw>urc-JYwb&9$KCXeRY3d#!+ltrf zo6p=?mlB}Dx+T=a%ou2qlX|nao7aY)6NIUpb)oKCj;GnMZ_cKqEmE@6hpDBae4xH0 z0ncA3OseC+w?M3!Olx6U7De@kQWGvzyiYQALM9;D5lMrZ)5o_I)e0m+xPoV4iam!V zG$4Ji{$Vt9UiV|NlC}98NoF^VJh(cu#`6sdw6-OkVO1)J?ccV1&||v?NKykN{q>XL zAI%3V_M2987%T9k`(=%Uoe?$esq}b1YM+1VxJA_1!m<@G|w;upajVu{6i#rQ#}EhK@RL+uORtG#7C;yl4WRaqRH1t z>Wh3|<%B}xHLv&0weHlN&G^@qKc6T@l_3&Gtk!B0zWX2=NL6Q;6<3hc2anD)p^}I> z=c2;;RwU(_X;G~g*)MhHyx!qcsS7CRiD@#W1!9RMJN{CS(E6&G5bP&u;eEhf$(82V zSwBnlOMW-EC$4X_=}X=Gg{+OOUqXUe%rB-wp@rJIbMnZNP_DT2j=BA;%g(q&E|sv_ zdEqN|M*=-7;<~RtYD4Ph4{PHK14=!Ni+xC0P7ycC07p2uYnSJm&C>K5UGEw(*S171 zq;kKRxW!;r{MIe5vjt7GzMl(vH_&gct0y&MEAX<(!(&VD#4gT#r2Pf47vPxG9S}(^ z=ZGwJ(6GND@{%_kMUk?I#V|nvES8FY1;m1RKCh`82&4^O8L&;{@z=g(I5?NLCApCg zm8)YGSmd*iX5pi-;0=nPen4SG)Ckj{P@(bZiA-Tc(%3Xg?Tj13#2G#~UlLT+%B#Dg zp}<`Gyns~$noqr`t!YQePgl-{J&?>W{6-lyj99wVcD`2iOKCz+cv!Byr@+$X-6AQ^WFPh|R>W#N7MtM00{%f+&?J?ooM z-4FS%n=Y zAZct*@fGni`QL1sW*;VVA8oEA>ymVtaQtW<`YdFoP&1>Up-c7HE0>5V6xCEG-GDH8 z`lqYytV?fYaZ|N-7Nb!*_7hbCdCCefA-P*>)MOM^b3{<5H4if@g`6do`}1z?l?i1Q zN3zkDbd(VT)5iv*yzhSWi*NhlFt!FXXNrvb?OPcRC&z>rwmqmw1lyqOo{JsAG^%Sx zr~Gw>uc4TcDFK(P4!U}gQV>d|51DntQKXTX*tf5lAJYUzqp`D)EV#*eevy^?9$PV6 zC{=CcZY?!(84vgKi54Lkn!70EePs0>Kyd{Lg%lbo5_`fP++E&7v{Je)9R|2+EH z%GoL32J>jJ`eut`$UU%-$%)iBeJkcsgEq^P zb{X-dX=J{rfJ&I@)0#JcEm3#DcQM_munrSCBC@)@sRM0!?sO0G5t#y)ezxcr$-%Ge z+P9J(6^mu~iWOU_by_$S?>;A7=Vg(9HU2^aqa?E3#U|xPC__2yU%E7RZf2rdJvNUc zf2oMM`;YTAbs|-pCbv)17B?lVcc&haA>jASSj^9iW#`o5iT-vUwa%hrUQgaiL&Shy zA{_FX%qpxnh9NZ+Szo%bTxl>=WrV2ywL185chHsKWtx|N)go)+B21$S9i%QDrjcJc z{R}&LaejDjY$QTOZ9kS55d2D0a9X#D zQ#6oJe&1fVIF6JBp`Ks(CnhmGAAw_kUnp!pK`K_AkzFd`xMHqro>AZd_QOF1i&cs- z@qxtLD~liywH`08_M!WIG5kUkWgLkc9BQ8MU*FZaQ5|%nr(Qdpaj8E9_s4cl5n-DR9OGXpkH|{>mzZfp&MU{ zeaJ9HZ&1{^_sPZ)EY!cECyoa*I0P|Jvzw#)RP@$eQGL4H6#|NO(1+ zV5s@8SUhZ9-ZKuKi+n`LR2DK%2xn6God#VuJX5HNky%1wUNQ9uG9#H0xF&|4EGuz% z)E&0;xFoKGvY*7UTKFEW9c+0RO<*M}bJ7Z8%ar)2{D;0Xty<*)gWx&P|p5qgc32?DY@Ug*~58gVJh zL=w7PY%+semqm|VGP`CmF~J;C{Lj(~e-|iAMQtIoHL;mj7$SMfhY+*jQ-?hpltv;B zP8`Z2HB;89k(mOUulIV0a+i*Ox}mxSU`!@u8HO%XAWv)|$pNrdBNzJB0daIO)JBA@;YRwYS-sc9jIwIC6MszF9BYuj8)y@%fofQuEuM50p0oe*mL zOjh~sjl7f*xwOruF*VHpn6DO|+bvJ(WwN?8?!iZuyyfbejrt)o7Zp3>^24{heu%AB zrgGt|(w1c!Dgh!)yvfD5LQKBzDb=@*O9+XM49##z6{f9W+1lkoOEUUM<_!H@GW*tl z3_}3jFt#Q>UM8sT2GC9ApC2JFx9FLvaRM z;-aU4jz6(eWWpo|l=)|dj1q?I^jfB- zLN^KtW{9I}xtXY{uKdUiBbH3V)dF#{c5lu-_I#AzGHIW~S}EI~+l^eR+CZb_Qn6~@ zWNpExV-r)?xD+>}S`lo0MlgsgD8{pzBQY`jh#0d|<6`U(A8N*ImX=(MXra$0Tean@ zfV0e{&4$h*Kzbh)tRCCqQn4{&mQH+CI2>^A(&mTEJJJL~{Lws0~s}?$ZfvbAy~) zk5jZ=$A_+M?7Qp7+VIl|LAHobuVi(Ak9$Bhc=q*dtIf3_#14_MAwG5bN>vY_CC?&;Up zLqWo-3dHkn8kizP2OoZcK`#rW27va=k@6-!HUm!TQ)t%II;}=UNcvq(Fr|CHeQCdc zn%8-7abF`9{_99}+jHK`bjzR%H{tw@p_LlcKi zGXEf$f)}h!S`JM@I3;)j#mic^E|!;t&s%#wHdm6|Y#l+|F1PY2SY!*s2ds6s!@&ok z9w2D0KzqCE?fgM7c=Ns#k_^7QPd*A~SsyKjKN-LT&wDI1d2&D^RL72#=Pj8Jp~Y** z;v??(=b~Iwt6p=yv}}20=DC=AE@n%$*FE6MUt+ZxLoOo6@x3x4|K4B)P!=G)vZldm zSztiwOqq2|7W?u;gG!MX#SD$Kksr>7w>Uf5yzjRVtFfbp{X{QiL8)UIDFR2_bGQ6T zXk&If^b#{<)DCM#hIo4y8*4<3Y*q8X?im+XSeeyO5qfb@A2ude;ODTAa^IACbOx8| z5zfdDpc2mX@qLK+ru&0>eU}HFApo?R&02tWQ5e#lA(sb-l)8h*y02rrJx{K6BC?N4 z=pal<#gF9sI;PZ1hz{{0GN_A^82t~cNoBq)0ul!NaVCYYP&X50ycl4@CO;DS%Nry_ z)kH5L-8eu53CTI4uWa#_uJ2DWy-n7a;3fcJJ2YNL^?k^5xcCRGK4`djla=l4p(@rt zH3-i&eE2ZWA$MeG!1G|0dTmTkohUz`>(lPZC1wLcrl25f2w;PZ&@`m#qo-V8SbLI= zVYW%*)pF%q&b$-nCC@m)w%v3J*#R3vT9bG69xeR!A1V$=R4!pQuA6 z$8^BAF3QxsfkOJ64bo=*x+hjoqJ#c#GBWKe?x#D(`8ubhbNAs)$n=b3z;$a(L5RN_6uhUZ6J{GkNb}{5a_0Nb!cZ^7BZf{CmRoGnSIdt3Avdbr(jVU z6$EYVjBjlA&z~A!MMGR5t5}h0Hrl+$<$FWA^54Ynj}Oz$r+b_Xn_qNJtz(P~JG7VS zTOO9H+h2_Xe?G02rj(Y`cB@dV13jK@lyi2&!+qecue)y31s>L~{R0Um*nX8iY=sV9 z$m%>Dsmw!@f-X1jAH&;!f)DOjG0WF~g7RF<8=H+WNE9F(3y;DDQ*uz#$9_9QrqSTfJXtT@#YM@*raI^G%vPNsHq?jHJyR{k zaq9SPNLDiK^efn5A*7kf)tEMLeCscN+xU3I_xA~j__D7l&om(gF!#SvX1$a)1;|%i zWTntEm%2|2W2kg!so@YRFaLz<75{Fu#HuW6-ZlQN%g^yTEl&QkQ`^}9e*ACB_p{#a zQVUL`bwqiyZd-Uj3}+9=jS=nzQ9JCex>EWj9T|*2e#`c*=?0t()`82k`EGn(A02V-5(1bF4df;Np1nt;2WY3pF2%<$B=j z3TNHm-&W~DlNfWqt&(^bH72Fa^~=B2x>T4UYKjFqYdVlXzwAF^vK4%2BI+NT^+1XL z)#%`g9>rSci1W^ZCa6L?(g)T^p6pWyo?UId-1Hmt4OA0J6EP3Q(#Ky|4cD8w92zgsI7eJIK_u}{@P?r zK}wFlh09-(Ql9H@pmmHvo6l_XKL8p`xq?H<6J9_ox?C?mFtiK^I8#9tD4pwfQ|SFC z5HNd4^*ORIyXYVI{Q@u1fegBn&(VQJfEQr+2a0qip;RWrso(*P4)o0@_Bh5V>Qul; zlW$Y>O@E3pJA1C=pWP@#0(QCKJ)`VvqS*~WI;l_9f}&h^9LCTrIB32Kw4TeQsI#0U zj~GN?OX$J7tboy`$MxJ0t|9f@q=4ahO5CDZ+G*c6dL4dHfMlK9yteL-p4XNqI<{uzj5KxD9NDhsNELsbgDw0BiAy-t}kL-5x~O(F^Ik z?hy>R=@&F(UnPJ^Yl=A4IwmTxd=^Up0FnnN zCF2g&ZfW9jjL$nMZ01y&EhrLIsvXW(RLTaSJ^#nuj|_%|i54R%A9* zsx%wIt#X>NRg~m30(k88DJ?C6UD4_GykN1r%dZFr8)(+=%W|(j$A?iPcQ<@jp8L|U z%GZa4sgzaqZdVYUr0>_gT~W!2oZ6%2LvLG#2R0Ayi)04B zgcw+1le_>8k7JD}GXLqlkyvaFLpn@&hPe^IHU3&jtGT(`Sg0;VW`PBfwe0Qs#C_CI zj#s~l1;ugZPsx{ zliu^OzMSBHZ3(*$?q$(lO?v3{#OAsqumj0stl8o+Rirl{S&v9E*->fj<}{6cJb?b| zuWUkwP+roh(LWZOt5qgg8j8YEBy?|T`&D(xEuv>w3QmLdukFBx71}Kycn5Pl1Tp1} zS+}iz|1 zU1Bh`s`mNe_i^2G9+A z*=un;N_}W9LL|eNmmuW-(H-g-mBalQO{ufi;Mn`#|1QeDFuPqCwtFO*L4fHc|3W|E zh1dQAUeXvsbbmc_e;>+3rP&=AH9qyUNw^Np9_2 z_WDfyb0HnGb0b0tasoH|oJ`-V>E9(u>!w(&^PZJ0_9S3Vx!kwHt0h1}*55JTOk412 z&T4))+7-mOsOxZfKVY4BHM@yfXyaLbOc6~V+%i4Xh+j{>PxM}O4R48<-o3lCxaBg; zM-y%O4PyVrWX(A$I;Ye-cV|_2q1vA;=(MYDIw&r?)ch#HbEVDrTA9{=ZDb3Kj%S*; zkdpo^Sw3e9*%?&JA1FeTf4NV%Pib5#P!>@Z**||`IB>3M{b;uLX!|w^kN!p2XNZWO zBgjpROkFoqmb_+cW)l3%iSk+*E&g*wDsHk7rP+&l{Vl`Rw>B50yk|p`gp@ zK8C?UQMdU4M^QH$b@>jis6q=nmcc^WHsZNHoBt~DV&b_8+8bq2K+qr?RRg%J;2-+Z zin{!uS5)y6JD@YZ`t$Q}bqfB{Rkf(ej-t29y7n144QuPFC5MEn%a6xO`I~VPQoFje z#l;mETgMsgq#@*naj;Wbn3wFV50Pqpge>FjJg2g>7c5|WQ}UkegWHT8*>(c)o<+l!bSMM*Zw`%O) zjyW}~rN}f3uu}2GR5HDTo7$M+2prW%m;$u;X-Eh=H-EU!&j*|?5#ZPkGh+aHC0oA- zoh*ol*&5@<-fg;b7F*=_qiq2=8yN^f3Qickv6U(WrcXLoiYPZdJM(MtN}zc}i6068 zhR;kP-#PS>PUI8&oIfOv$oQ-R&nXaM466U=NyR^U@_)KRdA{~%?uoPI<6%lN;>%Xa|@3!Na(f$Ko)QfN({>4}NUjU^tv&RR+m=P7RTs}u!8z)8fk05mc&F}F< z`Yf_a?QiY-6S(%16xXE+Q>=Zh{zuqZy zOd%>r)H5g2Am`c{UxV^>M>WG6U^p2a1JnvZmlN5qDZ^*8hYr;W$*o6IbAhNRBkz-t z-`8eZtYL_dD!lJScW*f}_b)p4m-H{1R5CfvD!Qk(lgngiH~X7Pb6ldh2v9!k{&|A- zyTKL{E*vY$a-Y?dIf_&S!c@j5g+{>0E`w5~1*M6P_2+fR zMuJ-(CH|^%5i}I>!i)e3X)Sw|@3k2c!`l)w34TRwNX`6;JST%T*wo$P}GP~w3}jC$Qhr}?aT^hQolKqnJd}_chSC)E*tWvE=t0E zBR31R1>_BY%O@Y%;@I7_XX%$qdf5|&{{k-YWk>Eyet3vzI>aa zUpapVn$1@8l9X3s_Hx&M!YVXzuCj>t<(2$@!TI5wB3FJ9uhJ+`Ldm`;NntjnXjhSo zqBU2MOVB#?chT;v7YTHcD}-LZbQPN9%?vEseX^AO|8F=7E;IvwBSq2~yHVsSSCxKl z*uwwUSS-a!!&61P^~Fl|@P`j7w~AczEcy%j2UVGiwVY@)qfZ3(rTO7QyBp-`h!SB( zr#7kqPGg)-1-$k3)cNm=li}^3W=BgK!X{e?#{BV`OgBL<;bIX3!+)p7qJ2lb)d+In z>Pu6QF=yT4og#cb>Cb7@su+0k8h1||s3mc@)PTZ$&kruqD#csCy6ngscx5=+TK9GdS+(7fVKtf^dlYaE?YEGv7~ z{i{w^m{B8Sc>4_B$#;{!LUwEcPyL;gB{-e)B!Yy(B*DlcE|U_NNHDoZ>vMMiR?&qa zUn&>L?K`e*=GD5-$PML&56Bkog1QQqyW9D2Nh{|r{z?ah8@ER%iWP1k9`*yNWfvjj ze7HWh*}@HZkyO9xi_Mb1P7DRvvWTC&Y^gOPSo=YgngHQ1+bgOym88rF1gTewSaS`F z^;`mYHM-NDV>|giJ1Tj5@Txz5%+92zTW5To#u)P6E{KNxbbJaX7Rf|_=RqMNhfPgG zm(G70mH8#z|D@In=NGb0Yuhmh6J!hgL>uK%z%3gZvg(2>fs1(VgUPFB+@BS?<3iQ2 zpF;WTU61%Kgtk84k6tsAkqw711O=ClynO-v^0n? zv~&s5Azez2ASfjzHFOJtB1ngbQg;uYbDraj_LxsW__%SZqq|oMmS7{ z$sZv&c;FQip&K2ZbC2X~iSx+C!{!sR&e8`uXD>J?eAwdfwm6hE?;%4lCy%4Gr1l}5 zH2&42EqYt=7xx*5T1!@6r?EgL4lVAIINWOS>KSi|J=*$lniIwKDrUiED~m9Z^Z}_f z9WHUg>T58sfGFIjN~5=8LW~hVi{Uw%^X?e!xek0?ydN7^`k>UTWa+9OJG|#1i7K54 z&ii^!vg-aY^ithaJ^6xhBHijqKlTfIj!Hy~&FQTN-hAM6_3RJpu0zD03<4errEi(M zdA}W}-1}OD#boSPS@ICStRDx5#3JeQ=D~?4uXX$^6Vc~6EGF72;<^w94W3grTMvestT z@|Nop{HfPgJn}Znhx3+QSV}D7}P1wv7qMfaGs=FPWm+rBPXh@a+ zC1z46c1b0HIUdqJ|5wn9h3sHX2IYGUZ<|CS?kX(|!isq_Syt zy)=opRJNJ;pwak97CL4|^R72D{gYB&PP+Ho^e0sLaV8piS>%7)q$tXkyR+QP{8{{n z)r{XtQ`TtZ5q2sm7W%#G77GK4zgh~PR2}g521a%v+&CWtFHxZU^rr`}SI<#-vfE4Z z6LNH%HkQu*p>}E}cbZgxbyboy{XO5BPK&RDY`Y2l=&?cT>2HamFk+gN+>)nBz7c=2 z8`<+7U=!Mg%!~O&7eI{K0v1?^t*K^HUf7QJ#qRt$i3HdA2B*3ONDN!uo0hPB z(Y__{kY0FH=Il3?>t$vdsNuV7DP`2BPW0Zc{R`^Eh0~sv-5Zt=^-xbQymYk%aqsj# zRJKn=l~#+Edav2*I#~=BM*ZQ9Nn;1jUKBX{U-lkO__tN;Kab#;J^!Ycw6ra|Sv$0J z^er@eh?J?YE+yZitb?P+Fz?n(X=dk4X>w@K5T>CCbw5tTukYW3HO_g`vBuTi*VjXN zlJGqoNUw^6HTX^71g`wm^kH$MfpVMnMK;?ddH^|Xn^cz#&k?-LHQ(gkXMS;LQo7dD zatZEBjLNUZ6YzX7UGh>JB{&6tQ& z1oF|R)p`&3>BY{Z{I0{5n5HVY4Km$2P!DwHorOg!hs z=?2o2S1)VqJ3GIF?r;7|Sv~je)2;LhRWs#?{1IkYpJq4S_kco1ChWK7k1%gOAHk;G zcoBpyXM{7KUj973w!>ud7&@DQo8hEuT%?Zb3 zxsI5fRQe7VttxZ2s1zkP@m)ACGm9f;=y3#$`M2UzBrLtS2tM5d3!l&C;f2+`-%YX3 z@~RxAs|)6Wvuo84T2YFJbXWX4=_up9Cem+a3QAhBNwObf4~xiLG7|X{9;w4{R09 zpM*j(Tdi32w=OfJOJd#$v`em#x7uW{9g;Z?1=ES056^qW6{LSwp!eSzWMi8AN`dYf z^gukOP)HuN9=cC|DeQKK3up`CAIb?9m1+)AAxV^?mDs3c7}b+AL@69(A>2|e-qAa( ziphTH)FMh@I9ob3&uouk%USdNr11U@LvZloK$pSVR5Kpy-Y7R?lDMX2}J;;c3hsD|DM6f0iD{l}pa;BOD=NB;#I?ABP*e&u5w05!Fzu zsQ7NhHz+5YJC#e(Cq_whAd$APDUPM(3asEP5+i)ySumA=6deC6HeBgxdORI+@RAHH z)nh+4;Bl_;C8YUaC*0;vTADn4*}<^3l|T5&@4Ox9So4~BxgpN(M*tw z{N=9l)CYE8tS`)*mEBpqvGfx&)u|ewG<)f+lY#l23{B-vBxc(SH4o4RrIxgER#$cQ z!$@+uXh6ciUpzN6JM9Mc>*{!WSRcn#Cx5Vx{pp?pU8)MIE+O6bv0=84AOEmgOS5aZ z2>n-7>0H26m*W;q_jHaO6UdLLX z?_KSQux$=X9oz4xD%}-)V6SzAczP7;bSC<6M@?n5O7ri=Q_h9aEfJd)x%gjIFS@uU z)!BAH28Iy}0qZZ6_);ElX8oFIoRLv=uxP5!RTV84l7Gm8fVJH=^f_ga(B>TZr@VSvh8K9QzWKim|Fbqo;`O@3_`~0=4UJU(|R*Q1= zdVVPVXj-v6y~C22>}89i^cGEX05@S7yNcP;#{s&qjB*qrx}-J0+YYD_!Ru>4hz<1DFOwYN6xzjl1DZ9dOc_v)A?$xoi z5zy#OpvoeD3}ulctLk0*g)~R1GycACW{s~?-=dPy@EOkPY#hVo-D!YKWbVZhrzp@F z`mpXNODcSb*f0cdcd-%;4tG=YQ(~<;gqh;{+!#JEB~dCi^wx}swn!< zvVgU|fKk6rN)3@*F8TU>tIMa74 z`t~f#QeO-8K}XR-RMiW^dFNAOmBNEWGADPEoqJ_p*4qdlt{w~Ezus0^*@;X$+ znw_EwIBm*wr(B(n;2hTWpwG0C3V?R=+F( zrGM4jQd6Dqaif;m-%X%mX1p-v+j#3aN9dp+|BDImVrtx8s@f~WgRWy+)88M-lf*!F z?R=VbqMdz_>ZGJ}TR}n3cGcDgdWDs0{q7%qDU36(huhu%?0fY_-Ph{QB3`lDut8V0 zUWDMwq;W7?jw(kAR&HH>8|S7hTZDSoxL6L(UQ z`z~l_>Q_%!f76#@GSb>U$8a@KdISTroH5H!&4_@%NR?~<^2Dh6gver>pmFIW@S8}> zO1`tU-V6P23-^f|%kIB-<(xK=#re`wBfPTuZTDzZP^9PgeZEqTP>N-08o^y&*|9mKKEzgQRx>51@G?SOjETs{nf-DO(JhNOOumxr zxc*zM_+~v@5>X;;NPS>$x#01T%8VzFY~^)w^;1888`!TGfr!a!&ks$%;1@#?#T8?y zwMnfm_}}CCw`VPdth|AT?}M*SK3JU`TPb|_<u`9`&>+l zhdCz2g9(%WA&AKYv210u*M%z8}DZJ3eH`2FFvo`u>Q#3EUrR?nF5 zN^6vr3?sv6lC8|C@M`aWIhG=lF$xb$oNWoqu<2;wC3V-x+e zC&%lA^lx&nqcRvNHCT_&2m1f6PgsBGpL_bR9bwDAc7*2t+7Sx=Pr%9d|NNIk=2tf$ zW#Uac!dL6zR_z`RmZT4v81@3pFV-4<=_tyFoSpUnHMSkkugihjz|tQf=eMuLb68HV zKCe9Yzr3u4?vI}y9WTD~tq!wNyi_d6SG2e~HtGubb-Adc7}8F47idZNN-;c|)vBXl zT<2>2(r+^EQf(bba8Z2aemJee`lbE)U}HUmtl;C{b;Xe5=(oWkm4O|9PmYhHhoVEC z2Vb2xzx!hSU6r%t+V`sQRqFfeck0rY(?eUQKxbBtZ(_Q$9e>Y%D_)&!t)KfTUJrdI z`|#MWr^7F3u=wPI$B1MSkF_VfKV#ZF*MxCyGzs>2iQLyV!z&3J)wZ~jcL?Q$8A+PwwC={x zQGfJdR4v!PH1sbiHkz~Q-RitH*6qcW$S@S?y8nZV#FS8P>|^bQD_yziAj}nx-Z8hI zJgs~_ZFEl&|JL)Ld+oVdByQh_%*^LvpVAqPugH;!j3b^dwzS3)oUPGqi{pnIu)_qG zN(zsEy5aYXkI-e|jBGR<)T8p=jvkX6#jI{ykdJ$Pl1n-OfE^Mnjn;2{@{5b4#2qJ{ zMDMo){tIji8_CBXB2tSa{xL^z^-PL-=ia@sk{(}#N*{623l_T%KMmAJ!>LE#r2Tm( zO-&x&{Z5{G2dgi}LrHbRJN`Y2_(d*boP^Q66LOVHu{@$~2FY_2%59v{*NFaJ!HD7B zIwX7ax*C*PtQ~bhuk$n#A(Q^;>uAaLAA9(cx3j0H%e?ncTN6KQ;7semK)ZxAy|sna zdo_;1Xs6?;fw=_96zC&zp|fO8Uta*VNTdazkw8&oOw;=n8A}2l+ZL3*jbmE-y9Cvf zRx-TN;~qTyKwe|7DjL2D6&sliKek7}b}fDVm2=)o(-?~;;F{0l{3PC3wF{LVN<(n9 z1@jxD2j7?h)`?wLK5#oJ?#?|TIw`dEuCEBtvFR8F|GEB_~tSb zN#IgImh*{HQjMI|4$$slmWddvQsG1Qop;yuorx?odt(o6^m@8&ei@TzGiJ!pC^HUG z;N)HzW2Isq_QL>;6}YsV<(F& zjA6Q|q1BxG<6HC|>%2VW(_wU2+(-wDU9)9rB~fkj!-f=SSQ&$*_k@a)l!T(PBlBH+AJ3U5XdP0JlC{oi5}mZ8WHPi3YEg0jH#iL}|Hyh5{=(%j0%i47=FAZ~X!u=Q z;f%X2k5tVuH52FSMr#TMocPrM*V9V9<{#+3wJmbsMUx==Z;MVWqBR|{0-o~QvarRi zZ(i&;xj7AdE~q{&s#|FhUR52PU&TIimpVk4yHPNCK$)~2+Ky_dlN*(q<>B$x{12M8 z^v4nOAdjO4RjU{`{*afXsTQO;a-7lYD&lbasqPPl+sn>R=p%pF8l@U6Cn%4nQCN0* zFLoaffpy^9;*Md>gMaFn!p7IysPiduOA>nU1oR^v@66mK(e`3&1#qsTEVfZrLEa1* z%50i^I}WgS;z!+2Y-XORpP*Tvq%94f4-c|6*x~IF>3|XLa#6wBVA&jF5_lI!YT8M| zO;PxKvi9vz-1?54thSypaP@K!&CtDsc1yT$?6&$nj2(I!^j8_jSXNQvogCwa&}%0~ zk$YcGT+5@2YgxTSVW@Uj8y5bkG_JOFu3jQF*K?bte*{l2Rq_CRC4uMFxOgc%d*{jc7krFgIQ3tOZ^GURK3+*#A3e^U7IbQ{Svs_Vfx7f(wA!9Q4 zUB!w_IEfIuq^5uK^JSf2FD^4miTgq|kdc*4E!IzSvI^YJWQ5$@duCmUhZthUlWyugVK!ze{-jT}@Y^?K$$))W{1_iKD=Df+uzGTa zF4jipJavW+=suDXKC#8nWQb3&oVZv2IhUk)h4`dox)>AzDVG--Pa{_gm%1KjGcS>8 zw1Jo=wN{5U*D8$-$Dw-TpDJsp2fLY0rJ}!^{3zclTz;t25pTSu3HS>#gEz3#$Pi(2(s}mExDPJJ%=E9l<~chGE&gQ@-Z#gm~qv5 zMu*n+Qf^sEXTM6bU}1D2qfvY`rz#OT<6dqvFqDKv+w;_UtfsTbrt{kiMP$5Knzy}J zu@Fos9P*TnH#Td0VL3`ApBd~3*;UD_@-@~PV<7$4%M(laNkB!7@%u;T^~J1YG~*O) z-CdIn#cx?2D>P6y7ULA0UCf~tei1FN(pClr&tobXnC9CREA>W%TfWPl&il4F&_%dJ zxcsu;7Anv6=-y3egM=~}^0kOQI>$>VN-WLm<)glVTK~77e`%oxH`Po6JB!&5@B)3B z>XL5KLxCS4HJ#&&k_XEpn*x-(!Q4vQ%ll7IS3@pgwHfnBi?NsapIuD70zPyc8XLwr zGwvr2xm@)jc61rLoQ7Od53fGE2n#ft%yWE@`|k=2P0_aDuHD&k8UG3+FX68TZ{g{# z)m)nQNjDk=8weA3#RHERa?}ELE>`+`OwH{Wv3_Tq?SXNGOxEholj~fgpmhbR<%PehcTb zE9gHRt^)=jZTqH+Sm@KMV3NH}|CgVeoNxFOlAE6Kd;iZLpjz%1X@CKJ@$RBgU&~tk z>2{J%wa&xHJk4AGMj8E|gNuNs^5wTN`Lj2z`7xu6{^wunCvRbr%IAQYNbT=^Y)|Ia zYfAJk2p=52ZlW5SGQDG|PC71g9`o=}S*GaiZ%GNtSByiu%_}=KRQny7&MCnLalftR z*L&E8o}OP+1hxxfy_@AsfAgfOfrV^%{*m_Srhz~embg2g^8#shfke7i4w+{_YwM~_ ztYtE8!V33CE1}-iA{pLoXXvxsk16f@ynm!M&Q*gm^dpnFea?B6>IVr!UK70pWsxDC z7CmN|XfM)c_@uotl$6aI_!y{$UA{rXkkGrX4?4+m2MasvX-k-IA9JXm@33h_t9m5U z@DP=W1Qqxdv6%=ivc!C0BhX9g2~46-b0?Yg0c<%Ai=#h<$Iri z4ncEdxlji#D6(mQMhAW_URY*uYbw(rspPj?AsE*Q*U1$&VvP@#&sQ+{SxPvY_sU_1 z>lOE#6x$w@hQx5G&G0NN8xk1J{|7P3yjrdi1`TVHNRGb})fRuDlic zCeMX2Akh;+cPyUF9nc`xDIjNgw9KYOQ9@B79tyq9CL9(Hwa9&_zkDaz&eAk)z^nHm z(5uax|8^T=1EMSZBW(%d-^0s}K0x&?S5IPW7> zku$jHrq5#~^J{gK?~li8@{RCcI%$4I7q4M;2jP`Kmb1pTicS7K%|V${O5LkBxlE?g zAB+^c>reDWVy9_6kf%E+;S~_Ima-mk8|e>^j%7<=t0gQkNz-cjUb+CfMpb=b8`*q1 zHwbYzWAw+2o%+DHGeyVh8iI|_^BoIh$_Vq~UsPK!{R}$fppsQAjC?5~UI`JtYg$qC z555a!@avNOXs{n--pE*y;){gUQdijI#4F@~)Z0adn8<*d+4ys97%EUb7dvzlhiGU8G?zUJ524m@@}h^7uYdY^Rwg;4f7h)Q*GWJ+ z^|=`*D8lkNa-r!R`J`4L@qHAgu{5E)xZ0H5_p4#3_D>LYGk2jw)__CVq3thZ!}7G8 zQ0!2DCrgp$qAJtln^s?HJz{($S52f^J__p5>b-fPS%3GA*Bshq?%9@<(?XFP~^k_@HB~C8}@Jw zH68ub)v}}%0!%QxfXrI`gS%)uf~@~AEiD8WbNbljuTMSY8qU-7i(#LiDe^;58tK8P zy2#BbX!_>qzvKhMPA5)9Rt+_Wazwd4UVlM;Ua{7Bm7gn(7j@M8nyH zr6aSrd81s%@YuNirrBk-gcEFBge#j?mCHgDZ$8x8+b<0)7&}xre8<~vCW`21F2TAu z3qu_g(KCRv8vha-*D+yrc;g59K{|G)oAg%j>W(N5+m6+C$4itUh$SKy2)Q90*eh>g z`F6&Gl=wi{O%lj5HdgclL+me#n}X54SpioEV=c?JWq;#BqC_ll93Pa6!8Lb;0&kT7 z<+skbjuJTrcY)QNK7Pk|f>@L<@^M{s*V<$9$@`pFhfOHkmxaR8hx{I_gy8j=|{&xTpb2@{*I4KFn2Dbx%(E-15$j+Ag!(UxV9 zjTf<6FFy2MXS4Rb&TodvfWn$olYO1J(?Ez`8PFF5n*JMF)9lILxJ#a=97N!PTaT;^V?1?>3PWEIq(nNBK!z-{n6!kFP2Cv>k>Ti99SM9^Q*oef#3sc8R%gzVO7EjqyHJr87bL=Nj_9b0t4hcMj(dMP>EH=PH6Q)Y0 zA72?&(d+8{l6cH!v_HkD8BFvu4{_{cY1$xF*dkl_MWgEGKr<8zC?6Q&bnc(Lb7Rv= zaxC>&wqHBADN}~z-bp8wX_ZE%sAhx08dW_qbn4Q3vq8Imt9=$UYSZHSnzb2cWGH5w zu>=fU1ix^#b%bzhV7L;Dju>`3;*F!mL0fUU$-92qH86us7OO4+krZT^tKKvr+bNH` z`UDAip*S`)Jx?XAHx17?Bsi^oR|r`T2q(c>S~_*~IS#`Xz`Ld(Q(s4@NewN!Mi$m~rt`Ja#?8=QXQPHVuIr z!DJqs*JvQXHlIK+O;)&dr#ZuUwgoNydd3!`XJMLj9)Gb|6Q=$Ngt&_bj)xJTHww{4 z$_ayTeb5xj1?p;i5G4fBl13(|I^Eor!!diUu>e*(4fMrwvNj9os z^guJ0jQsH7`;w6^kv|I@0}h;2L}xkXGKcVv9^~eBb1V)GdaFQ!?~6T#9hADj{=Puc zgM=leCs95>M7{6ytuG(25~EOFDN5a^t_0)KvRU|X@QP7X;mo0G$2EBk`iGV#lOX4`w>P0+Us^gMaq6#Px4{ z^mnX?Uwp8}tvB$E7Z>!ExEQ^XJ@Cf;r(Sv|XX+L{TF^W8(TZvH0>{4+Tq1QG@L*Ao zY1b9Cwie$Ffp9mTzuwYINAA^)uiww0I;m}b^Yk(F4}r8u-`!PBx-0_%UY^sj0)!p` zd9HYvSPrx2PrXQ_L0Ow;XOKZGVt9@&>4ApXTSDpPhkqUQ>T$3VsIYP230NafXmPzO zm0@=mpD0;>n)5(yS%c`B$-0f-9-^DHZ%MrLO_E!5L#i^|Xdnmw;7$`66PWa}#iLa| z1~MsN8TyiuAn0{M#cmK3NS7_tAB0w( zz99UW?G5a}6>Fr*#pJ!T(JGc*~l9`-%(&amKIGeFTO&N8)_!STM4T&q% zR`%1v7w+X!8OfHqG5CLx0ID3{f}l^je?Kc~_UJTTgrNV+++c%-;WX*>BK4LB3tIAM z@S_CJTq#@)m^w4xhC=O9aY=j0GOJQ{roewjwV}TPQuoBr**(K~+ss z=}-$Zk8b9U1&eA)qZ;RRfcXkn^vcsf5x`X>NCm^g+{qi9$rpu<#^RmB1a0q;Pz6sk z_(_NvY@Lh@V%OfM;3G-w#?&BM0h9M&Et=NV$@k04G7_KZKRC(97HXf zK-=%AJBE-mWSOkcNH^~$)r>fIaTn+2C%U(xVsFxCm=nN8=miKpvEaP5Glmn zukE8>f339m1+Fh>x&WB=gpr$8&cBKgzul5CA|25E87t3vsJ8K5Zr`h5xz#(x_0t|j zSoTl!fy9T%aThsT-~jvTI^#Me%TJBIbOJ*=AcqRF6}1t0sHZHhRK8^ik0bL4q$UGH z2w$>eIG0mua()V8CTEA`&l4DmSWwvcP=umG{NDT!Dr|*=Se-aYc*fYq%s!H8dP7=S zJEr9V`66uh{aCW;l{;Z&qmZUrl7CY`@!?xVcb1s2;3afyFlSu&AC+j=N?X{UkP|RG zL+3fvm2p<)&wv^*PYBucRj`Cj%k*8y90Jgn7|n)sbR2i7w}mZUb!&^)`Ua{s3Y1hn zYNoXLh&>UpK6uz;y4~N?&RCM-1nh&lt@szv;B0JYxL* zJ+!enK_RiVx2FMUAhRA6d8C5t#3r;bzxzo-6ohqahCL zf@q?l&3#U98tHjC?jrO-82fKjvTJ(YKl@*)e0*>e{$L(Q4vG=ifb>Wc2x|bSwpNBp zX^SFPgwXZtjuU3WRG(~{zK@X`Mfl}?-3%%mTu0Yc%`ahbBvN+`N#NsFrIYbrNeU08 zd?-8V`}`rfBCl8!T*x-x+i2W{dCU-u+M}>OXN=LJmq%z@y!nVT;cN^^wEw&^?xiVF z3vwx2>)6}PHe8`lQdRA}=S!O+zz!7GW#l!M2QC{O69JKn*}(VT zOCG&mcud~3ZVMv0e-ulf_9tMC3imm;i#bx5YD-~_0@fhr_4gEVsdZ>PRHIsaETzo$ z-?5t>h#UQNE!lmHWOye-?-?eGG}>0b11p>%n6mfP=%l`jSHe;>8=&Nxx<(`o53in7 zi1GbV2sqPc2&1wg?BC#?XUYtB@Gweq;WEkrW?HG&X*{!n%PaD95>^5EiR~u@u>)?o zB+QSN-s!95aV9|SOfiI-UcqG_SJ4YA`>Hj3IC%w8XG#^% zfYl>Eg$e`SL|=&*0{e89L<+FvFeLGc%<=m%o(zd>^yyrv19GdcXX>V491x|Xg8m(} z_jE(|yd+_qBf#x3!rEw0#*rJ>%~OhYXshae`a$3DFU@4ByJaISuBMZS3&;xN=tak^ zT#8(Az&KcL&1hWP!tVA}Zm{5Scj;0i9$+*(8b_K$kUWrQfb4=1C5(XT#zk;}%ojF1aAHL5xMmv`g=R21DW{Ja%6e%`A1W4qtM!*uSG_qbzJOw!xJnd2I znpBA{wZ=zFsteuFe1jwGsimA9tZENem)7^bF%$Xg7}e(=F==B71!czX)6o<*JUkd* zn^S#g&t@p#Fc-AXbpl4bx|3auVtgP@qqId1I5*Yp&{r?ztmP^gsmw3P4BH%{<*4wYJdKxyx8AG3;Y1^n^I#x%Y{-FdUB~2gh1p3 zC=>^nNW9ZD7+c8)Khy2 zR+j`Ud?IuMfY=yF7S`-Ubk}3UXDhMHKn&dgIzX5Xh7SIA=l%qk0T`g>B{1h3t5#Qx zEh}YNoHF*w_#DclsRW~CZ|fy*xTpE^68nZ@#A-=%SX0nnRlpOlUKXcnn56jB{S$>~ z`X}0xHzt&MIaiV0SaX0#z&Nz)>Oa%w3&$tIPNEo)sNtA;uX_;w2|T04pcgt!3p2rL%4& zEYAHD{`Lt_a8O}UdE(gW%kyGFK+%_ohe7kn5Z}|le~V{FRmmhz$NP$-C4Zq|@K3p* zwh*vZvgwU2(&{3Z&6_XAcF)fXYC)mKNttm*(jd( z+Ca!!Gh_-PpI%93_0TMMSG_Dz#0rE+JwAXWt#Ci@^p{nNy}5pF6NxS(03&L;wfsm@ zL5T2Ws;Ga*xV$23_p7%T)sDngYtcV3=9 zL%;wJsac}1%-^>eF--CXr-YZ`1>aim)UR`of?dwg58mUgXnaVP$fs>?DK)i&5AM5{F+ z=TjcGz1=dBxJMf>3560$dw*U@$g^w!n3p$T6$0w)`!hu8dr<9PMrEc-zay08LerNl z3E;X4`H8cl}#rAI7;MIjgQlFm@K;n#wu4$=58d$Y49FJHoVb);wZc{ zno09Q0yD$*1NTn*?;f*O4Vg`^HpW802ga(UDBKDHAXt(mv$+ChG&hR@(otEGCHGc0 zeYZbeE@9dmFTU#Ev)&K%i2s>)te!MF^qcM{!jgfwjw;XUQJT<8q*(l6wuinooDu(I zDY=B|MvMMQh_DAlR^KyyqRyt_L3;OWSoouKO1lqzijDm9^jh@6u=KR1e{NiC(3)z! zQD^I(sk|lSDd@GxJk9vt4Z?5+zfvs&!XGD#0L=ia0h)mvsf;Su0s#7X&NPCD5*!f$ zaUff1!UkYZh3t+ZLfEn77E&WG74vMsNp*0>vj?7CwWnIPS#N|D6f~7#$_U33?yJ5K zOL6OC{wePl{!hz=iuE^m!K)1JTKk7UBKYH1HH{}xTppO7LY5#$kB-^uckmFzdyc#t z7eRkM0%BN2iPrR5%IQ1gCKH3x9UFqmHFRi%kFeAM+!LeoCgybN7r>oS=_*>6uHYXnGT&Mm8vj5smH4mIucBd{U=}? znf>)0U@#%pi0y{gp3Z`92?d~)@Zf*wn~o~6!y*T!aSAQ^Gb11@)Xtg<{x?x#>Si8z^##8vv2V3>m?9!of5ka}96I_nH z7Z(o%$nAH9sj z#=IP3&=o|M#WiEIy{~>c+LW7s4@qjyT_k0ok&xey8N^Z6o*=~VS|cNm9L{L054QFD zy9DMJ<;aGj`*xqf#Jr3I>xLd1sVY(FcIUH_;s{)VQPVD9bZf1De&?md7XBy)rSvpP zC2A7DinZ#YPkNZyv}XZiAEP3%kM1z`QsP0k-?A>$T%w$oR-P^e%esC5quJz-HM|(* zHlBeP{tIOS+R9_q8sD~d@iQEK7gJjXAZowuGxO zvC<;H5Vv+^Jpgojn|{*bm|bpgYk6`VS$ET%7tJ< zG~rj-j~>M*SKlBiPJl)4Tb$$fk5xsx#2&JSZ;Dw#i`i`DYOO(CV`aA75bT3V`mU z2v+Wo$OSea<&5Vj3c!_(wzWu4&ZVEAQHLqhFG2Ck6`x~7054z9mTecAoHlhskt5&q zC1cBpjF&A20tTPru`o2cHe=to_yG4v>(Js2gTkXA00(0zO`>&%h7XTxzn;}J!hF$J z{d2r|+g37_I;qCtyO`bmB9jSXjB(d=kq<`!))=6%*ZZEha1#g4eD0GUKkHHHp#u;? z5-vsp`t4KqZr2+ZA3SRR@nVn$0JF*@O$BDxvvig+`}Cv2vvlYH7tP^zSi{p!P0&LC zAH;+l3KScA#qfqL9H+^0wanzOoAN(PI~~3NhD>`r$oXlyO(r#Gb!M-Dae@gZTs{IQ zER}Gqk6KNFcWrY5f-&t)Gk}s=W_RRh2x+nCes2QK3&R}`zNwhj8^Kzy!bY>s23`1q zf!UBCCdZoun2GI;3&(DfVd$3NPIf2%1l%)0jKykpKs0}MCv*?z@z z{rdVU8E~;M%Gf~#h(F(*!JKX0@h#lMliUTUk>X*pc^t+pEi@F}nd)N!jL3f`Y&PKE zy8@vc&-uJpzLoc*EVIc{7-;d?PB4+5c4*P$3B(6^Z#mWSzbps(pO_;)&$-5(r1+V8 zSVqK4r=<%Z<5~m!s zQC29S)}i%i=%}02V$(vxmjZeslcRvpUMYRr&Ig;Cg^qmL?J zQ9^AXOKgavCo&sTK=Uaw@MqbbFBzRrZ|0O^PZ&rwxqX$;@MEla&+|Xv-O#i}rIYn) zoHrOjt9M;2+Ib;4b8GC)Mgbb-+&NrU4wEkW{g^}w@SHNgcJ2T*>q38LtRw^5Xjy`~ zp1@^5Pg$cJC$5Evp8j{)y7)^*2^`<5*seAm_Z1GgyMSeN^rJgNOYB14B#9ai-_iRZ zVzhoUxG3o|+-U215A+-)ig#e2dV3adDR8}74I~uAuQ6be(1lWa@D*|2iWkJ|Q|lnEes>ZNZ8@M$ z2u@n4n-<;=E_z9>+na%gzu1c#hl`zF(Z;K0XbJsMM)MXQoE@F@Zk_W&ZzbkAkqgg= zXOn0*)Uh)Vu4+JU_0=3P@6mc9C_Yj=4m}m6O_1RHLdo#_DZJ!(oOiTc?RCnosBRBzNP ziv!m&K1mgS-;eOs_r$8ejr2NW>HRyr*GVm|6o?>9K;mh2cmf}Yk|`R06|MT`scG(; z#H0AkT~TNId4hzJWeIJ{!J3q2QrdEa0Cx7qEd~U$wUHo$tEw0e^zQ^({RK(>pP_Zy zA6dod0mj`4$u&;sMBSDyN!LtF8cvmLc*0be@5xX~HsOF=E4+zbB^SLJr@}d~dH5SR z#5-#h>RQV=0>2_vxiuMGiB$_f$s8W)HPcE7cY~ddYy@fJCDwf@>7EBzw|(U%{QBx(Rz)eY4?CD4U>K~A*_nEqUoH| zH5oOohrVKqN88vCI}S~%e(Ej*l7r~7E?~8}kr2InXG>WL*OG2nLcotF`cQ0|_NYUo zWJHo7)Rrh*RW-esqP}iW+$lgkqve#BzfDQ;U=$?>TIif=|_pV=3l=^h!r8_O6&ux72F zIYSg@j&kJUlbTbmqS*8fyWbeNknuEIH(Aq$ShpoGJ86+J*vMfdSjL!q?nW zW%(%{nM&+ZV*~JC{`E5pV;Rpd1jFwXb$|Yo+9RbNFwe88h|Q<{YVQ!v!Grz{>zwN0 z)fI$qMC4~8c;D_*LVzd#H*wv0xhdl8tyG{!YPKcdHTQs#z_O<$8W9#c=M1;6RUz^A z&IX9!u&bFsB@0$y0|&TotYA&YpES{U(D3VIp33WIl_>l zucQm_51WTu@uW88l+gF_-tA8kNLEr^pn3%YL2=e$!xIcN33|G5#@}|S{Y-GyY$$1( zyE(G*#KVB2S(iU8$Qvb`pAlQ*n}?Xu*~=%~)x1l=qTMudpA%Th(5zg1F?ylf)q`r+ ziCoBxc9n32Z9gx5>1^Ms)T*i%PSWH;Y|YioufqTWwvL!0iq}N(;yU@y8J%W!=ig%O zMd2`~eSbcf8{95Z{r&P>_*i^)fWUB%fdZ0aUhDvWSxXekz#wrlKg;X#hqq)Y_5Wh+ zEyJSvzBpdGyBQ>hZeG8y00l&1P%!9}Mp6WkR6#{W5s|xR z`2Fv_&vWm6ci$fl=bYJlt?ycUe?QxkXroUqz|kXfFp567K!^8j?xSB(mPIW8Rrss)8VRt9klRpf_8SZ6%oy$5ojMf(E&n$hN@UQ~c_K&l8WC1&w63Uu*Gj!7GiSup`oNIOI)x|35`0j}}x7dX^KZ zbbsS1!dC5S(g8Q2@C^=n+TyIGiD}_=xrfLEYKuUt_41>ZPBYNP62W9@iMx10dFcc? zRN;0H5ChnN!VFm}`$z?Q_})=wSpbq&knT6f?vdD-xy@3SZ*Bp7iWnR96X#=rI~vP~ z&qmQRv5bNryKiluNXcJ_-|cx!xQ5qn2Fb5n#96<{C^CDwi0;%{@+ZX^Sk6=Wy z6OA7kQeG3L6tEOO$a@hSZau^|Oe~3*Ti|QaPVlO+73ZVCJG4>~#L!Z|`B{Ty;?7QYFF1k6q6=MiU#n6BOc8%LNt_ zeO=U{)pQeFXV+6i(zD2xwji?V)n2uY;zABF0aaSCS-zYD|IPy!E6$gddZx|3e?1*~ z|9;FBSu}b}YQu0|)KkyQ_Ux~}uAraIIm--5>VQNx3Wrm#*hhwc_06f=ST^ zR-fG*a_me@nVGyC>@f6&J82lIdH%W4Buys`Uh4y;lo_-y^`I?O^ODz5s&&+ifmH@R zg87n{cVPbWVI3i_XZr(uWeW#l6Wn)Rdz8fcrahOlcz4f&o)Q;OTs!ZYF^hk8e5sj$ z3@8-8I|U)nDE>HSJ53&xB-|_Ohxv@;Ip+H{lN(HL48|kOBMx!$ea=jQmZOIGjQ(JkK z+{T}0ZZ{+sxlC?|dLU#pG$-~VTS`F!9mI|I?O*i9%r9DM{q1(rehZt-6$i;?l?Z)D zAdjnkM187QmeU(;Up!7sHbBl(q?sv0%#EIOm@4JXdriO5wrmqw96{H`3JYY~LM!Pl z`75!PCgDbUz;TJEdH=vd=i_Hm;uLiX6x$fH*P`gGqj2ZwAm}4SFA=)!Z~?o^i`v*Z z3W9`i#Qma&s91@gWPw-ltH+VccV`nnd26xkR}_BO1cxfBLvIW4#Z7QZ-4?7$jK@8W zmog-Gu7z}QB)(;i9Aal%ZBwv=e@|VUp-5mHHKb$f46Wtli)u<0rJVzb&T^hCg=;Bt zh|jLKFG%-i{kvB`7@p~=F|D?Xz0#9|Um4=-)pi=PJuNm7&wn$}Cn)Vfj8NoG9kW_G znz`{(Ou9>3?eGoki{Wjn)kWRfhhT?}%aqnGof3CKuf(8-m4UdITD1j|AM9|3iMl(+ zm^~ksHb@Us?()8?9Diw`_Hm|&t%3m?r|nbiyY1WRYOu_fhLQ!U)Yk6CeF_H5^&ZBWT;nNcm7LeInn%~NRkPwP!Y9VV)LUdJeU^e7<)j_-=pPNU2pX)*IjE(V5 z#1KH81)Q228-x!$og5h1ItZYJ+-P&@7UoZ;&+cf2Ydl?TO;60x+5T5S+S{>18edpJ zVw=~E80bYyH-X%kfRerB64?{#K;L{DJo3P=|q)(}>IX7NOq#7R7BQ(Bk{ zs35}^gnOg-XGOersAy93<^MS=#Ca1?+X%9-15xS>La-Oc*+taa(Pr<8e%pskO@~f> z$CwWdBD@u8lx0I5aK2d{MQDSW0)(EtSP}f5TSqhX@v+{O!#iz)jfZ!iF15dcjPSF) zzdL0e$LrXA%b9QAy;OIDcKmg&h?U%j^pvdpG?s?=c%9hPSI8Xl`t zfPkj}`Y)~ZO)EM&R^r)#ae2^wf5enSzw7+x_&@FKd;^v>$_AqP3I@cC8GgMNZ%F{n zzHYr>*3!bx3Z364N=5%^HyBnRX!*))SHSjQJ{BoRr{Fc=1s?i8;R231fO)L~HD^V0 zcNO0QZns7n<;lO9Nv+=R2F4!MOpAz-Cy=Z#FH_Zi3;p0EL|80>0;2&%WkFN?G4I8(#=k7UN5|8?WPCd)$iNvd1r?gw0STMXS*fBb-?N*)H z1l1M} zHLU`?7&I&VOY8d&0rywX16mW=4zXy6ReaK%xK#R__>&fL5TY|jxx_uiESDDcC9RgO zPS-`fvnU^$_13^s;McfHHK%>q_>c|!)-CnSEX&hmG${|Ym!6jkA3V^cqfjeYYyZ{A z1kZhySCY{7XmM4EPxN%?$>!Kgf&Wr&bM3Mp1!gNgO{I=j9I`lDvjn@_6k^>;%Tl+> zE$-w5Gc9?60eJ|Ua1C*U{K+1YjcG>-YQ)+b`42X3kI%9T2&#Ff67<-p&k(Z?1FoU! zP(32>f(Qo}tC!AZsy$o}KHlyT7OzbcJ4JXw>b?cA(nrt--J*AxN4-`q+pv}ABY}TQ zweF>#l0xUowA4l@guS-^)KWn5+WRVWVDBrn9kPYR5NS&}uU6Cp3dfTV@27~Nd|=9B zWqgq_G5%`QfwWO^@b>x@Z$KI;vW1 zYEROT(famS*bko^HE11?<@!O4s2*))_>spf>z?hq z5|fkpHC6F8uQkNi{N3FJdg5ZNUKsUSsTQkq(Azi5Z#~mlNc2uZm%lM z`cdL3!o=jBb}93s<_n6Ey{X<8gi&XJFGiVINI!99r2kUO{|VK{E2MBWjlFD#DhJdD zedrRP>c*Puo+T?UEwpm^Bb>!oU=b2=+9`9fDsnKjt;ftC29-`Y!EIBl9W%$$XGOJ_ zOx{^r|A^D1vzU8oUVJ4jRy!1dXUID|=27@yHhzX`y9|_v-9(jTz8X`!^$I8GTvLl2Gs#moTI!>tCwc;xr?eBg z!cp)8cYbq|GB??*=q6Rt`VM(+4Yg3$a$%gxPTai)exc8xBcoPY*0X129iS@3W(xzce5E{n=WM=r4?z^fXZ#_6;iI3Ro}iC<4wvj zytf?5MQMKZ8TOk=6vhd2?O22~a_hW<&*)P&5++$^x037?uc;p=<@mg0M_w{YqC+KT zt|q9~x9I$4A7ApEwWIOj7Hm?KZE0t+`4f2xS}AP_{RT!eO23$#@IrNAD8^F7Y_6xv zWU@K}LY6vm*@Fm?v8pi;VPR_EuXGh5m=p7x4K?=wp^r{Ps4GwtBTOJ^JPvvsE_om> z0;S20OQIKtgS>QuYp=WeN-DpcMhXtqi2&PsQygPTaxSFqN=K8 zcf#5a!|Y$u8p?9uca{F+yC&lV&d7+M}QI`#{QLpU!~ue1tWm(OcP zlSeh774#=ZmXAv4FJmKjjONCT6I=03St_qLt_5y(!nv-i51#G_vZI%c?|hPNXe1bD z=WDX{MU^vRry)C=YXa(dNwfrUuwC>_VvGdSdV2}liy|$)`AiExt!3i!$(8F15CUYx-i%16x-xd*1Fa`qdFVsiJP}>G8G$Czo_y?J;Ks(=yLC(3XzDhP5 z=29!{)`ty=lYTcu$s8|UMzp`$^*ECpvB|V;C0(F*>Q@<^Y zj{#J?VzAa_Z;cQGsFyS+=I%X|M}TLngv#w?gebs&iE9Zw={=9oSkc~dxaT5yAqD6m zzImJtNfZ!Fmz?Nl@-#_>Pg*NvTWFYBj@br<_6J-NB>WiN>|Cgqw zkk{xJgPf}b)e6*i=l;B$1CJk^2HR*qRU;b8v#X})Xyk#w1)daFW>TkuE@BlkX1+^( z5t%i>l>@V>$~4h2CX;rN^17`brkl9Ml01iqPQ(R~M0cSln#A-rK5Nb^NZkQ=acf$M z{zP7`ZEMfA)wi@%(?mhfyl;LmK0e(0b{P=AR;i}o!}y`cvx><0J>u_Jzhfoi3i{pB zJb8nWhBrU=9f#NuAGSFn+h66iQ(f;77oMW}r;m>@S?vd=|B@?1YCKHNu#)7hBSblw z=gMnh(7`gL=muE?WcyT60*&n=iK{l_re& zOqU)c1^BMpo2%aAXTgot)@28xZE>%bbH}*iu$E?8vB9TPl3DKsJgt6`3I3`obpMldP=>%j^9p1$ zhvn+#%942MuBj)QtUv#O(h<(NTi;Wwy;N~nP}aleBfJTvM62UeAvRmmPjOX}Z1a1a zFMZ6rK|7R8Tu)?vw^!i(Th+X6ebq{n& zKFRwh8U>Y{{Ta@i`7vCOUIIz%KS&|V!w2s*7L(6f_FkxnPLvpIvT`K#VIuMnZ~PLo z3iBM$$V#9l$?rezT;wZAR~sEZ&8Gz29`vOkP}AuReRVMF?GHlheS1lsU`P2Lb4iMF zU(z#XiR+Ckbq+oeo5=Y1_$g=}P66SuEyGQRJT9k-(o>0dH2IIy`We0yAa3Q`EX9OPUTqKX%@)g znoTXjuU4zY>m2&0?_q9=Q*4116Id37slGZ*er^Gr@83BqV;a9P&@0xU+nge6rP`Y} zEdVK;k=NqIaPG1xCy$3~_&8Zr@`iZjpC&FjsV4F>z|H5d%}iMXi-5kJUkeVL6*D=< zA(Zo}em2qcEj6B7{8>Mc@;ut-!#NqM zVh!Lsodk0G)}k@&yY*@et{{jJ<66ASf7^|ziERq$KLx7TFE|?9{>YBX6|JMD zfM1$J>g!lj&FN>zP7U&bS{4@y)?SLcJLm~o=385ZIMB;cz+II{mbV7E_-I&b_fG?! z%Zks+8bjzWwA^_5>=^ly$vSCOP@HDaK$y#0?~m+-0THsYy2MS)za*+k1mm)|jUm1l ziV|aFk>}s?0=ghHBkZC?Q4RNwA(n)RfCSUlCm@uA(lSo86Qxj zNjA^Kpj_lQB_0*()TZ&v(G0`K=zu_eEP z>R6SECyK3PyBzG4i9zW@Nt^ANplyjwA!B!b)r(p#^(E1N{t=A`%Nej(#cZ?gmm&H< zu7chVzP~t~sAtn?WcuHrykgV!7}U$F%Bx4K70;y_*RCWq@$)nDuJHaunRIVd+HsXz zH0POX^x=w}lXflh88RElJLiRwy~txSk2#UW6@lU-y~7M^tC)|`-l4v%UYbl^eN}Fj zH3U$3@rM--#iwMu%`Hgu+jXWi`5OongSezFVLV;z??+=gOUc?x8nW0v81;e~F}V$L z*_GrjQ|&YF?!*!PgUs%XiKl9JU9R%ui_dUb4Q6<1@`GC=ENenu;>7K{{`i+I!pIm2 z2OIHLzI&?qRj{o+7^|oUYc;Gc#VS2nc!{YNbk#Gb!WzL<9s8Gci(zH1#SiCr*lVa? z5hT0uC)kt(rT%RKOE*B_fd)eA5oz5%sBLnltDRw`qTMK3`UcBnH`qu>#=#q^e%M{u z!veP&&`N9uAKPTPE8I@C7q0eW4`L_0)G0mj)j$bKIeH%vuV*_(mWpWKA{9%XTP(uK z**-q7{N-p&jc?P%6?l48o86bXI4iY42eY+%MP7b!f02YYN0x?5F$$Cje`Fnv#l){H zi$Lw>h-_5yABHPup;2x7+RH7i9ZOmE)rwF$u5>!%qFAS~YQaCUnkdj(-`PzSEpnh- z5ID0YRcaw-EXL50yBkRTtoMf~3>sU;N>9974QFwnpLtB4QsiiSf2cnCepBO~Z}Nwl z?xKz2s!ZSgmu)Nr$J3B)V|+ftm=;eRFh}umbM)Cgn5dA0US*f~fqgvlniBMdYaJ-~ zlI^_0d3=6&dJ?NqIdO_j*)yIMwbawI?5~!WMrq3a@Vv{-vCPO~#X6nJw%XouCq(+A z*Apr{Vb`Kd@xlDPNS~i$y@|gUb9!!*AeOD^>}H5;EN7}&w$x<0nCCuY*3?s_d-a}1 zYO4O~Fdt65rtxPbVRA4>%iyo4A1I-WkEyimdS9_rXjRNnvh*e%Av+JE4xqbC5ML-^ zhLt+BfnEgqNF(%e8rg+BNkAAOT|CH^4ujCY;t`jO2*;!moxeOG}AUW||l&2qFdbrIUE=n$^aNpaCgDNWT7&ZOTG+_>=y{z#HH<|d> zr9|uO^|l>|u9iB?)Z}2iZDjkino?`1S2&vtear+8*mNp2pRf$jofbv6TR0Fc*TPKu zUy{r?+nsN+t@VtM3KoYU5OH_*)EZMhe?!KpkR3O*3`U28r$0(9iZX`3pUIhfTpRFE ztFM}C<}nx$JC6l!RME#W9Rw~W^75am5p$k`m6^T8E$o8Qz0sEqmG=It&Lu$&pV#f6 zdOqg!nIBXySE!fcI{)bp6gBgt1m#yNuc*#&gn8l z>jxeSw#2EnmvBk~04!2!2F0ZmXRKN zFzcmo7N*S}xU@}gnW(woEuwGw8lAwTla?L*bcQH%Fg4R?kR7V`!!vmnt5PJ<%YJ`k zkq2P`sY7e(Z}Z~9i4_Yknk2~(AHXl%=xD;GDvVWTg%0kibCdILe z0Un0}jha*{sT=k-oA$WwZSLCE%tHICJdEDpOSW&1o8Gk-xLI`-pfe@h69_aRX)1Yh zsp8)rhy&aRQdzq z|JJxDFpnx6Y=*j-Vq56vv_~aoM&2=ciXm&5uAqo6G1nJvKKT0pw1zpru7(lXd63F@ zo_@Y!L|N=0^K3ta?tIFnVyJZJ8A9Jiqhd(k0qg?-vJvDA>8z6|YfZ+_QJ0r^+3Y@{ zmpIYffJ8ZXn66ao zU|>fMvHod|1BtFp6o{;?Xl!X}=t+zay>9sEy+eN!!#p#a!B6alj7`DZ{=8K363?3c z$;|NqEmLLgf&c1Ku%D+MO&wHDy&!2-t)*$OpO%!~(4Lw8NdDnkHPfZa8`*00PCAKD z2oM%v=_xPvDJV2^ortc5SJ0V%I*s6E8=5CRs%ddSET zH>)4Ni}28D-q42BHC>wEKn#itj z`Fnr=yehx`-1bxyy&dtB_TNi~?Z2=7u1n>elfF+oSeEFyhPb`P!Erm=l68;d@mHpw z(cG1lhsT!Q#DP!B2lEeU0(+T_8&l5jP4qAwxUn{#<();Y73~IkH{@}U{+X(NdR)97 zarTCLG1I`u`I!=PYiU!$s?QBuKl$PX-O8<}*_5odkH49RHs_z0e64@><)^~6Th}B0 zyp#HKB31q_tj*qz`|fP-_wcZ4-SpgIZQf0Zy1W%r1+^h;W9VGbVDWFt zNO88I?Dcu%?Q&WB^XAfm3GbJVVkxeHfRVw>rZYmC?3mlhA<7RZCTm_jKc*R?4d9wN zPCsxz9DB1Cs>s`)`+eQW$Mwr#^Vb103q(ZHU&icfyR_w-GILV-H0172Oy?di8r{GB zXPYF$=+ZS1pT#u3E$`=Fi&E68s@oMHHNAiT&j|V5RLvVk5e)c8kJBYtS?>PPx?xPR zz!Pycq99uEgZ9RWj%dLwBvepy(&W`!r_S_JbTq0*`C8`pc_}%|1eu^Jik&m%8XYZWcSYcIjZNkr{UK6{n3aC z^sT5@N5R~;A|mu4p=K}t{ym;po0VC-aXwYIdn5n+`F7r|@blxjDQ+p_AKd@G(TlGu zzr26Fy8Q3(=@&HXY5EZBzs(5A^{VuZ_5bIY7COVv|L|UH$j32U4M1E-cL_JDFE?ZZXRq zAOEZiUZ0vfo-Hq1Jnb&4zW1}!J%X|9{Li0mn?C~)msz#{hvc)O_if6yyH4fD*EzKw z@nu2vL5_Z2_|95;+w;OyN4AF1p?;=>0=lf$) znygQp;W2pOB2q1JpAWfT>drStDqPcj&sDm7M&N0I`1tenH=p7VpOyo&Hm8K`#5A{b zfB7r{u3B9ooz0P{DzWj0-`$%npNz6cB9+Moz~$H*`Nd169XvTL=hpLcN>ik5z?NmLMN9N`He)Upv5$Jv)Zm= zhl#W1HI&r!Slv7*yk#k&omN917-sV}5LZ;j8ferb#SIy>;p zEzjY3yB=e*@8P3{19`Yv_N@&Nx@?+yP$yzAa=K?X!9dq~Z;{`Jt%KbvbsbQwcS z4NN`tJ^n>{A65jIdqOjJYZ%Q|fNe;pOq|w-@)5xYZyxWqvK!#<*lHSVIv-GJr*z)y z+nSwkKr7^ip(OaGn5Deudgh+|bTnS{vc>^1iUS`2Fxf25!1Zm&xc;c8VJlQ~8=rS& zs5F}-6j;9}I^QoNn;Th}(^xXvy^`cJNQ7B-ydacV)=^>%6EZRr^ zk9FpSq3id{4(0}&2BM~aPllt1h(|5qn`Jon- zDM_>R6F#)6vf>urLN@zSTDwiK7&)#74s61ke)oUjyQJTbK)^w9EvC-*+Rw-@mE;iO z=n+r6xYP1QNe5DlKL3qmF&Ub#f%h~bE8~-K$3PH!+EH2L@_GYzzJvmjCW zlo_q|&Y_2j#YBGgu!@nnDiAnAzsMQvDw zPtXQ;5DZc7#aPNXc-;w{Tz&e9v^Vb?TD1O~qBro29nEq>UwdWdi5NBzWSq?Nb ze#nNmthYKGdGoS_KS9gT>oly1?B%7|cpZXKs-jm0r>Dqh7%&rh^H7JtSq%W+L1pBS z=fK35N(Ncw8`GYTJ8tYAtUr4qHuBM#9zpbv3uD;MDb|2FQ~uEZ7K4#DshWW<0SXHu zA{6MkQgI*^$?%_DyOZTL*aUyoQ$;n{s@+nCce!0u+EXwG4tBs_&3;`iBFWWe9~=Y2 znjSi$U=6J(j;fH^$eA?*$Wvj-m@9SDX;vJf_5OURiy_FK*&1pnZKMn^5@4P2#pUW^ z<`j6{+cl?AZR1ptEfG!UCMy=iA#{E?3u0W2F5VKp3<(j|k+XtV6b-C;=Hwyg72TIs zTz5|TuYU+iDf}q)4~I+;DV?mi{N&Z1PKGYdqq6K}(j;t7j<<+qCgADwgfX{k{aeu zL?Lym?iuPb-nsiy$K}%iIhNYMPe@%r!*%pxBh?Jpvb;B5t6)7H40O$}L_S`oZMsbA z{jt(}^{TeO=^KjRo(d+w5X0K;{FhDVE6s$y7WB-}*6gFYslV)bNn32t;2?!NOniHa z4iBzoVEHiAHCAP7O)TKGW*;7gAZ2{JhD(lLr`@e(bOf$RCcgb@o$BMd+jtiyHfz-- zt45~Ce+9^}tD2?PR^jPrEz|kSSijYyn%|@6cg{sVs$lIO@QfYXm+2Ob%yeDdeLYmO z8XqLn8MXTXfT8{Fwdjsje%ks#oR@>VMq%i?XK8I1zF1)JV3f9ClPs@ymdt&;<*zTclsHQ~G3w-&HDI&oyjiu4vV38BNBTshJ{Uc*7>XL*jNCko3=;yO28(p~ zch6ARoEq=d+U)#4UZDiLW^fGzKg;SRBq!ByO(wGbPV2i&8pZjr(Aafc;YgzlfuF{tE(M z%wOF)@i$c3wSPWT_sS=@1$?*!AyRm(I8V7(&}2-ofVeIKqTy=4pjU zssA`v@c|rA_2FULz5Y#R>_X}It%P9(HDb>Mw99N}PX^o_Y?+tOfI%08op z&%C@z?)_@$24N|BiQqZ_w1GGZOx$iE7$V(V4-zJAvYAiME8qvI^m(Ku-o^UexPz&I zy^CM{h6wG~Ii`!Tu%gr(m(cF_CUT!N%Be|~7Mpg~jSM;5ADQUKA}FUez8vdMj_7#R|3cR-dBms26+Mti7C9tWerBf z$3txnX++z54pfpGT_pQZP}&{0S@nb$lxHES!Gzd|jx{V!d~b6H-Y}GA1mVH&noAq6 zvM~EHTzP~z>n^&xst|hQ4y>41w*x`ym#wxC-8pcEsQ!Z}63|{!#G65EuXHfG4^=G1l%G@b5E!<*G2>XQUTAqYVY-YF40v z^`5}zTG4?Id(`T>-(@CLVIHwI%@=5hTmWt8q_B{PuCQaUERf%_OqfMQ+2=kQnO>c>sWdo7Ha$CZx0~tRd9vBxu1MCIyKvk zees+D4lY%PL7wTl13`}91;s>WIl~Q%A(IwEHdIni8?lhL#MieuxkGcyuu&I9j~AQu^$9J z$Rco&W1)7Om`yiEZN^#I=73f#jRmALYoeMbXEx;Q|6xczzvDog$!-u90Y^Axb%m+r zVnHfNdIh61VCy|K+?l2dCe0zn0T4(giN-Cc3IG$0;dv*-JvD^{@GT6UAyI%+B*;`1 z(ex%Az1FRYr7dDp)%`Ejrj4vd&5V^{+DxX|_1VjU1uVGGM=~$2PUn@rQ3a;2N)vs( z>ao!^(}(QSf8hhbhlgPZS{Vv$H(n})-(n8p+tB}XCSVkgow;cn`tGK3Uf)vjH%j-2 z(WujwU-2tWHEY$sNcuqW6QwsNcj)~&C>EyYapBW!^(@7N3`H`ng(BU^@CggA%1vXg ze8u^lIV)-t&_4cn|3fBh({f?};$-`LhNHqt`R7c#x9z7bC^t2h%N5_95e-AF{d24n zxPUegmX&I@4JPAzpHg0X)4sCLIf2PQBiLHTieq6-juC(2akwu4x;T94sAo<%_EW3v#g9I+Jq|UDdC;u`?0J1er3j&G z)grF6U$LMIZ1m8_kd5U%Bx(O>L{zKgGsNE>*Y*g})yH%^Ky@QAN+b_VU!WW%BDx~QEGSzJA-+=_N)Is%KYgo*fkFQ~_0tdj zK8WYMoV~B;{KUsxRay;Qhy1hktPt;iEw0ceaXPtQZ(DAgqMf zQ@(uxPf5li5j5U(kcI^jT3vWM5-S-2v z3YKzpvMF>EV3?K-KbATk-xu72J&!B?-!v-FW+7v`c^a(c58@6Mhj&taVB6r>@T7-}0pf!!a zL9t1u<)3bx-J(+^?wVS|O9Q1C05cxkn${vLI$tz7PgRX*;&SI-r4CzE*2WaWKVVP> zb>HOf-I4!(uIq#FSs z#`&EKZIheM8w;M}pHb^Sai&dc8~SeIwNVK?`aM>{D)!BOTpN}Dkv*4f*_qtIteal@ zsnOkIkfF+0(&@T*ad-eBQcX~ZQdlUc&il7kGF}>96LH0fmN=H-xC!81Kl4>hv$G<7 z(gHtnB$D}kJU0jMd~IBLq;aM1_+CXiohKE@`%QQsM7r=JF;5gS!U z1z}Naf+5!E=B0n;h8KMRtQ%g`y}?@L%DW`A|Il6PZGx5d!xy~$eRxCnnd_gpY>-bL zyfYfP{y#+b)+q`UrzpS(?@4jW)-TFVJ5_3xzQAa&)XJcKmF?dpfS@i=wTCL#CX;t5 zwP#8#9N_j9ms!0P?Hz0+SCl-v{8FAO0E!(JqDL_alxVGWRMfQ<(vt8YYHs2iw48a- zoGR-8YmGBFm*EeFf_T}`#i(Mp#O&y+k|+rIc|vmo`CayYk<-3~8lYj7U~6rFu;>Qp zEWjjso;}S`V~0jQSh+3y#*(O39b4QS&Dyr+W(0&qNTp?8AZ}cP*S?Vk2_#S_L=K|Tj)0#r~+)~crPch7)t`Sx#_2LRd?+tep zK(-rfpy;FOc23)7<*~dgkh~J!a$!n97bnMQEfkd>=AyE(f$m_^B_*Q4c6{ap#mZ*6 z=!3HSaKGBN9OLHTSk$$+T3El?k%z#PXi4XA7IrxpT|L3wxi%vkP+8Cv0 z?)Y>X*Tu6Pq@^HM<8g(eaM*@6Y}PEJ)bew{%2+n%yMT!oSSZF-97a_B(3;}okEq?XRQ-f=FYMO4IAZJOw{OU7sI&GVY z#KujK^^5BKei=wQTA7Xb4!kkF`+qvcF?8-AN1m?9%BkWiB98t8C5 zhvl(6I$iw{Qh@Dw0pb@noqegz{Rq^;9WvjVccZJ|eE2W`+aj)sYQ<7VnLJ6`nF#*7 zwD?Oc^9_g(x;syewaJyo;wIYr*>@ZW3E-36$lgRQzbd86@Bj@gz?ILtu?}7lNevWP zW(6Z8xTTZ`z{+L0ihTDgO#A=&lSG<qra|X<=~+y;Hs!=yV-g%8&-+x2YJkVUn-Zl1xaLE) ztqTo5@egA8v8o(f@JUjQ%qkF$bs6TC_WE3Nw&q#!!X8La)bjlu?VWAG{femrKv%wi zd!WDLH(6fxx#JiSq*5GWgkMI!7UH8z=1zviAL(~ZHZ;DopA!KLa-6)oKVqfmTJdV8 z|Lq=q3H8C?%9WWPgjnurpFvcm_oZon^{oFmH*ySFAY`$9F%ZFo;Q?5AO=*9=69vQyN|D!S%o10%R06%aKf88lS9eFlmZ>e84!$jsQK4D0!q8!uXWLxzhL zUjtO@P2v(kC99qhLO?CSL2?Y--KWi-81|@u+Bap2DFj(H+L+UT)5WwCMFFZR{r@%t zG_mbegip{Xha!CJsR(QDZKL$AFZ+~m&T&xfJx^~_$70J<3a6f3N2zBbKvrU|*mj>D ztD_)e;k!v28N}(#01q*1yo~zbnjnOQE~^_bPrF4pH39CJsx_nbcSBVq5h-fWl-MO= z_J4XdUozzGR|~GSu$G{}^J4qwBa;>T0B>|?^aob@XJBLH}?L+lyQ6|V-1zxVwo73h(JvLE2AB(E~Co@knkoK62Vg; zu+qdqxa{X5%DGIH5pcYVRP~AhgekUfUjCO91@dA9x8Gwal3bvO+-n|EnDc{k!4wz# zZeK%MF8z2eBuGo9pmRYEzFqPaFpd|@C!zVYhg2 zV81>OaBf&&6DY(tLkWHb*K*6J=x>>=(y$vA16c3iYz}pBq`Y0WE-poqkJNcF-C&w- ze03k$cD(w~H)7cL^Z*=#q7w7twa|q3*Mcv^$p5-bqF7bylpFbE&0Il>y_s%W!_?!L0)Pk$#K3C3fqVQvn(5MjjrN^GMn3liJe)bd6*Y{>| zM_vGSXei87(NrZ^$RNSjJW^q~-BE%_6F>EULONSi;mhbc&(if6H*EnGhXsW#VJPSszR|}e)px#o@UEYfez}tb(A%{VI zT6Le%nTwRi5yp7e?r*I0Eyf^^d@h$dLi|(E9?`|k(YzaPr9l<8Y`>)N<>qQZGJOAL zw*;fX7~tY^Pt#6-Q#i1A#8I!NNRc-bZ2rZsW@szy+H4(5=g#lMD6YJi#4bWsTa_Nq zq`r$lM*?&>_VGHql9v^nJT{~i(_UtJ5_~U01z_f}+Ht)6 z)CF*=#{w69ofqh-S1hg$bI!~TJ=%B;&{&*|epwbE4KkOEf_CL(atFa!n$IIr`C?3=Dhi$sMvad7b z20M+LvLh(b3UkTEhVW&G#X)}%yX0{H-)ojBCh6c(^Xoov?{8$&VWPYBgV3RS20bF) z4?UZ%Z#YD}&lb2C6>TS=3v1FNyukScKHISXM|__l$QdUW!${vLCdX9Mbf5?EOpWii@f z%5fKn$O{RJ%4^zf2TM8Qu2`-x3wsz!S;NEjQslHAD2~3(#v0Da&Eqo!>V;i2p4!~& zSHUMfxbu%uU#s4Ks93bZ0j3H%;{?-83H(w0?|_G&00Q(bR&kCWrtIjtQ~rL@1@*E0 z{nrr{nS!Z}x>)3lHgvcAdSB`-|989>FYqS7)8I8xSFG6#x!_?t*z4?NXLvE)te_*K5U)U0aYe4Mm=H~rRM@lQO?J?fxqxVz4Q zOY1o@HY%>K^;p&`{}nmd=jgOh!yar@q)nrr`7Z4m&mLF_!e)!_q<(O)6iL$lVQqUx z@`Y=EUnJ_1|3&4FkK8c(jeSGRtL8b-5X5UDuh2P{b|tj_7Jy%O?c0RYFB3zQi%&7}b3l7_2R%-4pCuZ!d6@V|8=>gijG_zSN zh6h{t7ri({hK13AxEZzfpsmn-YdY4C4aMjX8-n5g;RoMOcg3ypuX$H|Ec-v3cM>;) zYad2MNzJm?*lC}okKuh^x71U-SPaLPDJ~<}j(ul;c>`2V4)KI-k7n{;?b$@E?H$|- z`AVE$xlc3@&Q_*#{z`Zt{QJ*4=YNaAuBF>cRA3_!$rt4p;VEAm^_NIT$|-H&6JLM1 zC|NLFDw*kJygFQU8mD|r`>>GFGR(F%nqCYU+oeQp!<$3*y_{GthFj|^NQk8MOSB(6 z$9Y8me8}B4jbs@5I4U3}T@+{ZYIFBz=iiveU~jy1vL~Lj1Cd2fLu`?@bQcpF_nBnl zyIGmpjlP?(KT00`7X8mp<4n|4m~GUmdAp<#)e}pn_R_7#5OSJ9mfiDlNO1p_3vK^pw55Tx;LTy*H z%O2Iz(4_IahFtF6&#^T^iTm9SuK~T-8cjUw6f5Y#m{Row*%y0hmhgr$W5e9T?QbbE z?~H+)vXiC)5LEOQ_PFYn4My=R7?>vC-(~tF(@l7Do@l>T0AQD3XJeY+1wY0j?76Vm zC*Q8CV=_*d@y1M7C@d>zRo;NV4Y6?5wglZlhAh%U)&xdcVR2j&0LP`L4&}WPvLlsl zU*i5i4SJ_tZ#^-6TDu>uewoNd$dIP~kP2=>xiTZAXRY}l*Dpe_mB{(5X1prHkyAE(X&Qf0Akd=RFCfQvZ!fjjC zG@~NhC8w%gk?DIWQ})UvODDs7V8em|($-0)wE{IqjRhb+pu^MPoKB7ASG@Vpq!K9S zFU7}6lt=*Z4`nfci8{C(5q9{oz_ml0Ll2hPGbspVcS`dOX2>kRLWT(6(J5QZB#KZ$ z*v$d{rw{mp*&(jMRv(zfy?6sVkpb?Gl->JmLinSXFp)o#gwpTP?ItEecDpX60-8uO zc6WLD9H4{rryg?oF9ppnBwk+1@f^?u%&%U6TH!(hCd0{JO}YAG^sX8HVqz~knXY>Fl9xz^BZSuJB{L01>^46i!2dW}i;4R!mu1#(I8Zhq zD(1*pCO)EUhEQs%SK=g8-hV_X8>;iC$~l&^|dXTX69d-(iMKsp3*)9FA#6wfTfVxRr^6Oq|mh+1@PjP@F@uU z%l$+fF2xi*9~Xt@=U=Lo^=+U{?1oyz#L*Ctc@9y`jdwQyCs;f>Nsp<0+@%$;c z_BipbLduHUmkY=VktxEcaN=F*@E}VZz%9~aV~UKBpMbm%>>$bn@T7vTw3KJj_hQ@U zvlnhM`Kn_pqI*IB2n-mKMu9~p34=nEbEJ$_G(k>m9Q5y(6AdFfX$AXt8wNhZPfs zO#R}VIZNZ=^p$IzmiB8sWeG;$l(W_1mK9?$No>$NrB`|TWQOFt^XnG0o@;K*m5qiazvF@@dqK~XE_A08|hqN0S*ZZ??HEB10 z2EH%qQBkRVwgu8SUuZI|jLRQWX59vy z2|#=UQth+HeL?|%LBO;`yz@$Pha;bRtXT)2h;;NAA%X+?ydMVdBA^0*!a%ABRr)x9 zDZ_15)rYmg+ACh=j<65eB8_hP#ox%+qmdyxp29^ILvdCLllMo^k=j)8`D><>Z)y0HcTtP(0Y z4wi-kDQQtNbr>UkEXTSGl}zV9hKX)hUcJU>*ph4S^}&|jprV(6^+E4SHAw`LaUmHC zD$7^g+I{^RH`fCC#?ymr!7Ql{wCjJAZAeN=7t2BeMd`&w8v#rh>VL3YhGU4RyiHYd z%$#Vs4?J}kEMOW$Ke%kh|JUgO?@ zJ%IaY)feIDUhm@(IADCXr846flg51gk|cld=nJHat!9uO-eWvc<9@v>24px^)@&liIGq1MX0Nge87IEX=;}_pa-^mzYbDvk=k!n0qxzPJ5*s7V z?Q{9eTqYz-2}#ieQi4+(8v(My%qP-m>1d_a$@SJ`bNAbDR{780&gz3;E~+)l^xiv@ z*~vWGLU1fU&uy)79&Ii4MZ}k#e6tAeKpOUp3Ofg4i)nIe1-83xr)#YDpqeh09%?8c zUzdx2l&xGH9b7y0SBX^f2au`HYAfL~)6yE4y+_YOPKzsJPxe}$l^Ekq_Kyk-Vg^#e zzCIwWWYuZ4)Wr~{kPyp3ZlyRyaQ^fgeO3ute+^WgTWDZxPWmpZ#)vVr51IlQgjug zTcRm~UL0O-1onWAUX#+?kdy%A5rQb(om5oa6|zgvxJ({I^`#ORH8z~K9=NlXQIGsN zY#rC{3{8lj`L0PC11oz5NEM>j#VH!iTzaB$5xxfj8^-+T$z-_zrIj<Dg{T_e;#d%3OPuY}`P`~U?n{dzM+b))tcU+m5cFFrrRJa`9U)lRf$4sQr<*J*K z@g)IN-=qW`*}a3x*2EIJQgv)r=q4+!=N zcrzA3i(1m_rkgNz=?#RJGLh$6Ys1)Zb!>Zr6y0wfSFn~x(iBx`_R!4{v?kq*stXmg zp*e%INLZ;HA^hJ&FpT~*?{LA~z-ja5BVp^ot8S`jH}bu90a5wcsYqQJX=B%+=&urU z^aG?2KpBJxmTs&lMMcyPVL%8FuJ?1J{djAFVU<=g8}IIM=;Ers!oa zEGO zA>AepzP^t5iCvb6!pX?7ZK;Y~7BsodJ3*-_-yGOB?2lu=mBWM~qc3I^8Vc6 zqs%fZ-obEu2n_3dXloV)Di50O3(Gc-Z}pXqB$ZqWQLqK_92sppir`lt{<4md8f#p` zdn~4*Q_Y5$Tl!gYaGRuXQ1phBc(D@3QXV-&+@WR3r4DgaU8H5xgKY~yf>_KWXHFoojIwh|TC>9VO%D&JTbN`tyZo`Bi2Tx=p0+@# zm$Z>t4fXd_9-O&uQk37pND3-9=6k23NWx^=+*h~8U zm(e=69Qz$bX@}4##jWFh@<+BpJ6X;5Qi{q1Q8(PwicfWECUp_=lo*vj^5H7+CjiVy z->%L(%X=B}K0DZDg6&FBf-wsCwiklE`3@IX{U?Iq*4n`q5$g!PW9>S@g| zV}7kPX(&g5_3!wlxT3&A(-2IGC0O1Iqc!y5#~i?qnTP`aCF(Bk&%*9&C~sXD=8%P5 zZ8*1%vjFNXhImaUX*@WROG&Gw53{U-k2lDz?J-&F4I(V-7$(dOBJzxgK%xtR^k*aP zDoFJJpXPD8v8)I&+c~39fzGJ;NO9FYrx=~tSCWr59Lq=d}73O@S=t8~3VIuwQC z`$?5BQ(xKnV$y|^hr8ZMV($^rP*kShTUjvN3x$AnJT2Ww4p=pCyYDR$NT)SkJZjlK z==L+V_IpT{l#Q{7YFhMAGeU!8e4;{8aWga4_u>XeqkiV*P}SUt`IGtOTG0eCrc8*8 z#hCjbwxlm|%tegqEh31t?!YjP^kPwvdKTKXa{``2(Dywb^g=&EY=bBZGu9I7 z9Cd)uBOqTdzwxH{k+V1o%8t%1B4|Y?HDIzi(Kw;#I!t-c105z@C75MY{qC5)?`1lZ zi`c6f1hWVna@f1oa&_>P2c>mt2z&F(7AC5Sju8=%W7rf0uaVI#Q8@ytS^_|XMPb^F zOT=e^E~=!)kLt9k5^KEG3Xh~}powA>;A}8!m|>~-WdAdZG=)15Y3#sl6CvHbR=nC8 zh+pW6_v|-3Vmzzd*wFX7$2QTU%;|a;Ax+J;n{?0^M8*UujQ8CiG~;;MtycDu)U~AV!ss z^tDCsx}(`8l7?E;;=HlCEw-f?qQA|kLsbVG8gs&K)=rBE6vKZRO>>niz5XvIxY=p(q zyhPN8>(XWQN$DyxpRD;oOu(oI>SSMtmC##WOCemp0{pbW3&4QUEU7M!A-j7+6~kqg zUDjQQ)hM`^%S=gB4pqX?`Ej3H)}EJ41H)MV9ld;( zxSR%D3JaA>+rkt(`zV6SOy~>*x}&){-?SxRT0d(pUWH1^$y2p5Wq*@&8En}uL!* z!dJR0p z(R9~}`$gaYQkdRnsb^BPQ)`uRqjEg%M&okk^al;$D6>+Gk2$#Ip9%i7uA@Pf-UFmC zd?W5SP;Bq;&+9U04dsy9+t~IK#2DsKBQp7&^oy!`_u@z*rqpB7?D`H6Bgk4q;{dk-Fdu^e+LDz<2ujo+*>Z+#@nV^l{BY3qyK2^O zP}=7i4ee)sWNt#MIe35pe)IRat^lBgB`E096CU^th2WEPg%bU?i3yi12y)_tvX4HD zxdm#6MAQzs5L#Ga(70Q=D=|+fU9k*>>tfZ=r7yZ^q)z;A19QUnRHaZ}BQH`)^+F4) zCiRj;XmzuIeVbbVtPGsUP&FKSCNNmZhFSOXXDv~|OBv1DAOd6~Nc$0E8tKZNArFwr zDcoo+=Mb>?nqtN=U64@wJ@Q=h&rn$^mS}VE-yQZTtAhXF!T?FgI6{w2<5>dRE*FbL zmi7+k$68^D_X_kAOZC|a;KvdkCGr6AhoFe9nsNwB`-zLrbPm7^TWpka7|_Ii;e~<9 zN5gqiRw2Bw{aFykH9sV}PS(F39*YKtB|WlW8xQ1##m`_&fU>5iq~T>zb)~N(QCuN@ z3ZPNKK)kx3&_#G-q}jr?o$88dd3d7GHJn&A9bXMC1z8iM=grV&jDNteSk9v$ARdow zI+8|V90kwoSAglbJAT%XSyNGbWEn`~1OR;eI*O6d2`LHjh7Lc>Dn?i8} z-Sf^!6)B@;)`5-G&5wepTm|2Cwf&gddg&8)vmKe!MS}^T>xYqRv=NUQ1sq zEfdP@%s^}(=Ns4j&f2d)!Hgn)VS?7L*)10==wy4=+vOx<0(C?Xjv4v`BDCtT| zOt^vW*s+5w5g?ICN)iA|Lf&R`tQW*&F@$`b=MSI~^9ATyWL+y-gqSSUp?7;l(|}WH zSq!p*X7-tDp=TSwgpIh50wWHHb|{~c=q1GH`MA8?p_ z?;E?k!B+3ZW{e2#9<-#TFO$?_!MGVrCrVg=fc3tB!#Hh=iqLiYGf5}nr^vYs^k*I@27qGPbUn54mcV{5KxDvPYU2T zm|*4~<3tBvb9XWk%1BboMh=+MNP-5RKROpC694l6l)DGBYUumYisZOCo1EibJ;b>p zlYO-(!O&@9w!#ogV5r6jW?_I0Q6Rn`fMHuC7&#}jvnE@2+J`GX7PVibpg^=!$=HOlpJ`DG z=bTUulT7&xvE*}6dd%?7t^>!Sk;-E2agFfk3lZM!xmxV_bOIqK*BVh}d?4Xq;g83I0 zK=5Hm;L>h1AnuJms`;Jh+;%uJTyix($Bctbb;A;*ko?2UD}~nvXb|wgM<8u&gOFo1 z6F*$SF!gw&xjONs!tSYs#fnH*^S5)Mp!GeF;3jvD$G{Hn2 zmm0INP_d>Oi7i$q6p$37!t($CVvtxFvS{MTn_C9BAwh?%BY!j3ik}g$iTCIwrwST` z8zVhbpRl?!BP!~es7V6 zR*K(waxtp}VG!b~r~DT| zZ2of(6QOfn%uETmnO>e<{C#9kRTsQ2u!5j$13c$B%z~OE2JZ_(47c>NpSpxu<56NK zIEzE&de}`18mxXj%owg${o=c0;4ywalh#}LznpKR3coR#2|&VC0~`CVF}jo**`ybbv02pCd+ zU=^wOvCq1r2wGQmdZ*7yK;1(V1%QZEeYdjNp%|?u0ATG?^-ChM+xaj-Jt+2J6ivuWs`F`XLu-=$UBZ!_L~;C?L9~s zh&Xk#lOD6&I5B^mMWUC=DC?b9NZnmU&Hy3*_#}aPq|uwh#z!4E!lhm#O@u_Jh08Qh z?(85#oDo8-^)>5_cA#YOqxXx6oy1p9F=Rp#R&dsW*dk)p`-dzK2wZXYQk5F7O3HG&w`#MR%5z*cc|*ggn-0UgP@PipcUY@);xyZVduwur|; z`=o=zw++y_s_LZ@77$}3cvCHGs_J1bhYt-=L454zK_h_36$3lGP%Y5d27ipSEhu}75$}A>wYHcuM1$GHgT1~ zr9%x~@9wvu6_8tyvTim&PhcJ|yQ@|MeDC|i#Q(s=tc=tQ0))>Ngk>jmu!0lQ9vZDF zeUWVCNO8nOBp@&`NWBnTnn#Mo*jRmeigE;Fd~leS*?L~Ab=2}@0%axe-m?R*m>HKT zbQQm;*}rK5(8OqG?Y!<a(Cy zjybtU>`XI?q0*xnbhC1!IDa}TCuUORdGd4xM7|KCC0g%WSn7Kg&N(_-nV6G|M~4HP zeS#BHDlc2k>EGMHVuZ_xj}D)nU$&530Z$7o{Ep&!HS?S=&Uh?0pf%mwR)3_KWtK~} z&Y~B1i-B#;joH}seK_v^FbML8MW^Ma#+;#Roj3BV#(>$45D|i9vJFUKmeBm~^rCWj z*@&np12Fcl0TZQRPJC{Vpw`dD|Ck+0Q8qe&H^11g;mG;-Yj%Vgmp2LkD3u5;2(&T539o)*;4rk z5sa6Q$Sz?;!_!ffil;G0NoF~5o}cP4TmA&WT=NqEpgP~zcACmT$A(3$2i!O=4?nJR zGHGi+R8KG#w3H!`qa@?h+#3D-kCvwzo-pi0gt6Os6JyZ#i(^HnQMmIu^{0%3A=mcw zaKH|b&qr2^mo5xhvx zVDa1%=tXMa)rNCQyVrUi1QlVFOCL@INe7=TIhA%4AjouG@&7=@{)LQ?y*(-bm92;q zLJ~Zks@O>q+rHtWrF_IboZN z6H!yC+K|ZP?|jkuqF($r!UN!6qr|=wRioo19&dIV**|)>8L~uo?NP};NokzK1G1o% zzOqm_q(CBA;O`9#Wg+G}Cdz8x%u9!-S5?7AsA34*}cu+B- zPap$N%Yj5=$CU}kc7zVI!p?7hX9es+qz>U<}Eb~^K)b|UhhqMnSO&rb&S7ae`W(@Y;U|C=l(_2-9g#PgrCAO5FkbI*Ry z$5)H#`cG-Xvb4|J`H9h$G6z5J)t=-{c%*QdehhwHS{T(@u@S|!m493dv_CdAS+NOX z!Od#NUtdAjlrw86_20r=UD4TISTdnQcIN2N!Rm4PV!(_~^^Tr!S%;m;x*DZ1!&Xu+ z#vY~Ds+y%yOZvBooo!|0%}Onv;<(%3*Wrby*LX-$Z4yFcQZVlh3^(uId@~FeAzT${ zH6+oj8_FHR;?)2A6xy5mC0W04NKK=NjB?Y|Wxwl!;X2d*vzr9}Kv$Vzz2xW3%duE; zagIHJD~6H;@TcG3l+b<)Cz3y1{c-`T{fYNhrmdmy_Z^If zN`Bg6xGCvPS&+yNy#=6RDs4ku4D2iwlc?K@e_h@SJDHH;WQg2gC}Kt`!dQ@?8w!1| zhO$(E8n1^nNQKRwykzS69R97R>-lzlh-~Beetq&;$oP?m?!3o;>^Y4^5$0*M=i&C` zL3d`j_X*jFcb3gQacTCpv6d@vc>)id%xP#t&xBFxPTID0h)jxVu=D#e#$5R%jY+r?~pm$^L%gE-TidQCgSV+{I^`B z>v{LDVu07pYKVM=`_@EuORLSZ#?kD1#a zI~@9SW}fdR)dqZ?6uM|C{+VW}KbbjN?p>1AniCorw68o%ibjkb#28q&8MY5711eA- zd~|7}LS{mbB+jU2?_s@|0+dpaECWPaZ||jG!0D44KD{4*wl+T`M;}I@FZslOIam7{ zS9;{v*j*9#WwXh51fKveF~=bp87imz^|pf`HA$1XC8%$<@#ex6k7V(*iW2PqKw+ZbT)`DEocN&WP{^pzBlqOY|rVk+XSCeH;u=BhXt*3uT)K&{tK(H1{_8~Th6a-uqywJ#SkgvpA zVpA>)R=mX^jU-ZA?QP);v<1pIK8UrTRM|tyN3)Ttt>SGVcjp|x++wM0PYY_1f8Vlp z6@L(Jll;Np%QL8^>MyXHZMIEC>!D(N(F(Pd9Bvd(i&{2j4OZ20647Vs_jR*x`+ssa zO^JtLEN_R|^-v8ksm(0WCI@Sp)X?(AWeOc--;KC^-(gvk_Uz|wix&1IF0826@f@VpU6ilw zq(S<6aGgIY4iXLqwu#M9@C=RTh)+mEy?JFNki*DAHeWA<*qn;(3?uhk7>Z7Hdt`5WXq7&C$<-&JEl6?8zfZBcwD(Kp;d^CD7zN1C)Be8)b6Mb` z`}%g(d`2h(L_#ur{~g@^9qfPuL!0l7itV3N zz7;?ek=oj)e|-7Q^}6%Fu4KP!1_~(?U%^HdgSPg2TY)dY$1EnRFGa(uUv4U4A%#@w zitVpa^KzEl>iJT6!(aNxwac`ZuYhXo9f6-Ip6(ma1U}>;iKIs2_{%yIsVypQm7A>S zMeN&8(;6KjiNurCmG@?{R}$@HrBpCKs(o8G97w+S&?k+}#e{2`l?h8dJM)CJe7kP; zZh-o(#tybelK0g3E!T&>H#p`Qz3^>M_M+I@I2tg`)R_rU6C6s8-Uc!5NH-h%qQYqB zHyxxYk3%^rD}+7;OdhLYp~yLW5cCoA(!?tnxyU=^^Y?S zoicKI`w>6h;(8XUCnPeC9B~d6pcWB12r1v}#7oW5<_rx)i8l*TFZQRxsXkryqqK@FkEy>NJ2S-%ZR;7$*4Fp6$CIu`Af{bXJGfh|}GV z7AYez2GBxOA4FeN3oW@RAZqYf3BX9c$P32*eZ@8u;h#=|He-%+AhCpkb$r+AIF{|o zvVKqHY@Vf_>81L0gA-DPRw@BRX#d}Xdvvvisk+6B+WX&!>DnE2i=cYquDc)O4RS!$ zo)>-DXdAv~t1pMsy%u!-mrGSPr+fNA(?7=H9N+a?o$eW~U5D_IL2iR{BsCQBykP4m z@D*4^+vqa6f?QM31+GErcpT{xIK_9Y>$Ud!YkI#mUh1xCK&+Tfu+tMb_UG;1cx>=$ zcH-zfRQ=EK*gZZ=&p08LFg`vH3R`5orn2vM_7QlEcjU3@*|5AiCHxEl**~@dKkiM0 zL@;KZ?qXHHiu-!e{lxe@A@~_Dc0%%YwXaxY4@KKAnAi!4RA2BsCwHhxSLmBb|K_U4 z8_f7133GRw^DNFJVf#qQQ!NzK8YXNZ4hE6jFBLmpk8iXTM>iFREX8;!8J<7<;vh6T z=iZwXj;B@%P045dkbHhoJsij_eJ3wUSn^1|yE%BhCK~}h*IOr%5}qW#yeeegCK!fH zQ!m(aV45oD1$6;)GsTje?LX$_-znkl|18DdaitVQwkndIx(I%f2DH?dQa<$)mII)! zZ(TGmim0swsD$t$4Kj|4EOk1uUUCqf!r>?b(PMl6r>xsyA=I-u_#EAIw}od)m#@>f zUUh``-29q1qRUicBk;3w#(^2jT;q9o^@*0(TY0F3^XZ9`o`%-rqeT>h;e`-p0|#6! z(MFyz{Jo-DZFvNGa+h;-v$BSmdL`D|PMtDeI5d$5i<7iiHKgYaIuwyYA@{wG06p8) zW#JziZEs;i*}o zv#gdV?W4ub#kfO*|I#(j10C!FpUo|h0P-c>_CDW)q2-3yA_%a9FPeR5E20q_y{<)x3D_$ZkYgn{Y9XaN%LMNO$6P`Fm8bqx)$CcZ{V${_@S$&D2oJbD+6mTh;uB_c0yQMbin!Q8AY!JyeQQ5Y{J?rOt;AdMD=JF$L|qBBaMoRG+~CCDTU)ITNi&mJVtzs@ z&ZGvnj^7Aj4&_#U`n7CRPWel^^vF8B*@B6_tWOyAGHmKLnf(SL^z@$=sAQ-dcnwz_ zbGqw6@J3EPr*`mkXE&xh?C1=WGkZ=lzz<>I0%=*uQmxc9%FF55=be78tNhcgl#=HL zL&8miK>sLuo{TssSaWbM%9&Ej=S%?$^b{m~P@F`HAX9c@oAnZI~e57H=2noF_< zY1{mz8R=?4%Ks5B>g16B3LT%~4w@Fx4=Ur~`?$YvaJDe}JE3NKzhIwR=%_Qod?S1Q zJ?kCZ%KZmu+N5`VJryyvY}UJ2?LR)T2hsFH7pF)7zlU*#u#bjS6Nw$ye1niqShXy( znpi%ZXhgIus%q$(WvEpYSq5Nz&kYTervS^m&fBGyLw#NA0H4Hg;xhx zrxE0}cpYcKQo%<>x;}&v5y;*k>EYhPP?R2yv+3;#g*OLV#HvcaC{x>Q80+n0zgc;c z-> zYc#(iA>`5rB)3mb-XP_OvK{H>|7H$$yQt}TuM_|hvL#uh&sbe5!wUvv506--gXae6 z$=T%i5sDe>vdlh=@jR>T@@>Z7tsjs#nTUVhZk6l}RO2-#i9i?U&<=meIr7FE#o!P$rk`W#X4r zEb9@Tb#ZQ{>1hqUef;T9(L*kiYLSF}sHB?6qR2R-q{v7xjuanE9eh_6DMn2NOQt3f zHtunT^!xM!rsjnQH*+5rn|dx`>d}3aJW=9@nL9c4F$?vz8_jemxr^7+UzfmueC?W=73O}AY zWiacY*x4=8!rqil$WO+U( zReV9MMZ;O-VJ=I4gJf61=#EU2V# z>yd%RIB(^-%O7dggPscRK27MR#ervTUFq_^{QX++-m;5~(#k9Ql0#ObcC)9DI4_XH zaJg4l^_T{}3 zlVYY{$b1%PYd8ARWJtAKUg^1v1d2mLG zs->EA(tb!_mG~A*M4itT`TC%H{|gKUzDmI7;opM~_|4D1Rn%n)W2NtZJ^CTz&i!6p zQAam5P#lf1Q{uvIQgq~^M|%IOv395(`W(C#!OqEsf+a}&y6)?d*A}qYK&pS-lMtK! z7G?;ZQ6~JeRL8Bg?CRq$7ivq4^?XTLIAdg}@`4(TN|%x;5cNkh_9>B6RrUlMW2=?5 zR&O}@*_h{1UvEs0RyiZ%?ZY(`cRrZRXN_KtdA7zSYoU#R+ zkSMZcscXN_zTJ;ILt@QVnhF!*!Yb>OHck~vl2^aj8!2u+CX6>q-j<#n_kTB3vqRj} zIKhVD;3r8G!M+t8=fNT>PGp^BzdtFY9PtX=9DDub=GA*5YM~A0K zvO{2J6}nKXl=FVOh_C2w3vZUSidhshBF>}DKgToxHo`ysDQ)YUupA;&*UV0~o$xz{f`FoCm+BH3UlKumWJcV_i<*&^!Rju*_Lb+WQ6=@$2ut1)M^3{)9 z-%_DO1&ZU<`hw6_Tlajguk^SGp$yc4KehG&U2R?U?m34SmEm)A*@Y)-iG5&InjWI` zeyHW%M+Ndk9@jW$#JDh!l@4B5VsJpUemSMr7D>xbfpq~wua9k_?1stWoI;^Pv8!gN z_?t43*dFG%k3A~y@wCJ#9ldi^OjqzFDdTeuDJLUdGdLfYB?fD^!oG1_RSUE;T0#6- z3#0b6@&UKYaTpc9pqWr+lg4A>gebUaP$g4NboFWP0hP_{f`{UbT)c;d0x7w-SOsWh zkq=%pJf0;gbekf?7?H_ry&0ntBp5LKV@?&0jr@O-(29w>`F4(#(lXQ#eWUFB;Hb1bx~3Urn^gg)YO z6zp-eUKbhN_duWoEN=?quJ)>a8&{GCwN$!S7e<}|2 zG$VfYPn?)UxRgp#)*9jJ9{`O3J-3I7$0;$dRW!55bV4LR>eK!?qVE}DFt2@iV=bQ( z3kg~<)zQzh;1N1p*>ZQCUGHeT1uD<1nek{0A)lo21u;$y@Tia>jtnNP`w3G3MXmH) zIaHY!wjhiJ$BHrF-9Ew;6a>(g7jSbfx+#`}iQPxkWN%?iZXY(w<10|*jOBw@dg9ld7J z6DFJV3;Fu%5o&+8u&V2dJS*!RAwh334w}8>G4}dihKroLJ1I8!hPe9cOT<}S49qr# zHj=ES#`={Dq^%F6BMK+DCG>Q7=NmT$gZWx(Yk%q;6(1NXK9wiHJ z`)z_H=gjWKo|emrS(5pUEQR-!(g{gN_SWaT5!t$Tw7-q?jsSD5=?RN>EoYC59lf87 z{OB}UK&=>tO#7}@-qkcYsVW?kGvgux*+})#b@nMNwL@vOVGIwJ{3tW|5;cVL;WDYp z)Hiq{trxu%6F;hHgE6U&!Xm+I?&osQAZ&O6?I)A2Q99@j7~QtW&-;o7=u${V^nEJA z_RJiW@^61sS_LW7wltv9w#az;6EgV1>Be}(wMsRopXDuEmY=w{>X{88C<#@dNPL8m z(W-Y{l{TI~?6R$K(kzQwr{x!wyZWVUKWdd(p}wm{jeZY9qt6*_#?UW)ec3OqL;bMw zxHEVijk6N>T^qN`Wj;+`Yqk8LCFC)9s5Bf^vC3}tmz6?@o*3lZT!H=;E0>>2- zu!oRgbARhDXSTaV=~-}xBIlnWwwf{Q;dy5QnUV)1%iyaht3E-fePlRfThjizrtkeURhNqU3T;#EqjNVf)yfO zeRfWU+CSqSO33R0UtlXHRA!aTDUt4on7Bmk>k^C()av-@zIB<*FtrlgT%Ldvx`k`3 zO}m-tq7Jx!-xVAH>-r^_emAfD>29K{Ulu+@pb^?J2p#sTw|x7nzOgR9WGGQj4Y+~n zwTqIfEHnHBL11bs>6J36lW9ue!-f?;Wp)M6j8qC8W65kS$MEdsEncsBhk0=lVy{1P z7`zsz;&v%?dlljfP5Z?YhFXI>9+f!?XJ)~+3(C~mi<)1Y^x)TPD9K~q)_zH{09(HG6W zrClW@_B`t%6IxW_S7}+AkaI`*H5q?^JXB7>vo(bS`TEfYzc-JqML0RxYIRM6N>f={ zC|?)#ea4X)Cbl3fy2dqKM1w<(bn%nwJXc)6WwniKqeKi8dM;Kydx zQ&E`t0Mf_W)C&vQRqP9OhoGXw!F|XVwxHs)2v+!xNO^Mr0{{)v;j2z!vy0Vtn zmLo=Ds%0$)7mBvYBi8ovmghCP6Im7s8hh$vldc+!GPn_lE0=UlEPHoU zwpno={w>eqyaqd%B?|FBmCx_&ny9A`ANJyrn;1CMYxM0wOp%wb6eSsy)c4tx)KBMl z_ph{Su0wJyu>v5$4m}M2XSv~W>=dR6(#(^x=H+>?W?=<2#xz6M2_*X4!9~t^McU>t z`#LfBt4u+CQ0=%^Q#!PZnsVuAtoRSF7!?5h2Px+0F}+4hK>i z;tRug#8`rttE6k@xZo-753O{EcR4Hw--au(#+!U$41t!h5QF#hZ|{ZoH{N8VX22N# z^;ru87dh0w&noC=2CuzcHRZu<1kxxi-Ffq|rEaVwt)ggQVXOXUnB`V@>o0vHNNPMt zu~XvVDxZaD+yd040S1AnhSPU~Z7(W3`aN5{5QR?Ifb2ZBjB-@G@2ZRp8T;Fm_^`xs zI2GZ%t@q9f2fX&-cIl;hb-ekSmzw$KFlE^)_sLT?Jk8HU^DiH{5}0G?2v>D zL&(NFXadE$8q5-;TM8eMjMJMM_er-aCEtBjTATWIiTv#2goN`qUQ&Mo*u!Z$7+WS- ztSRu?SEt#D|4)rNW8$EDj?t9f=;ay=Vqd|eB0K$!uh+45v(f6Fm&WzTxQtspmt5%d zd|$jXlAzCrZ04M;k(TE5(!`?~7HQ5c$)9)@4nam=Xf=M&>KBe4U3BKq7zginI|RK+ z`u|w_%BZ-SZp|Q#H5S}GxVr`qF2UUiF2P-z;0}S{5HvLI65QP_xDz0FAUM;I_r7z# zxwF>%nLnqitM@tdI>qVQwI3;*K_}plwe-PGD|SPNo~+?_yS=U!s)@Ud`YMo4#%`+F zk(`kQA%aw4UF;d~4+VMiBYRO>!fz+kupCW=y$N6gBSx=Jw1p8jJ}X{HHYuy6 z8|Ou7fy^ay$|nouuCwQ|V!cohU%SP6*zvE`-(t#)_G#x`m5fqpAu30Ye7AdEDr=OKvcWZv zoU`eiChP|dTyo1N0oVby(;nrw*OzcTHGpg8b>D>8-#k2lXRN)R3~k1UzaOInGV{Lp zTpozC{LwRgbZW;wc#6T?@RJg`dm+q8cX9aN0*Wq_+OLTMUubfY7n?VHMV^&6UspATivhZ0@lH6N&ozx27DgJ1W!AhLEPYedmF0blQu646tW5>2%tr`mIZA|+}SP8N}y1B%Eavm70c1?8ZK0FbQDVM23r z*s!HUS(ClS-(!8wXeL-%W;4g2rQOCsQcNVQ%p-Gf$JpM1?|h*Syh?edeZW4lKjV}E zhX1lc0$5(S1i#FmNh46LFR^i7svrRKRn-YkEYSvpI=0b$*FJF`germnGa@xFOOW@g z^$V9s|5{l^S7NAcc8uNEvxP^QrHZNk`QH~?h^gfgCU75WB|srR#%+XlXSJ%vM|2FD zgm8Gb87r1afW%J8n)a~(G`?Ymh6;UyCK4doOHVNRb5B1gEMq5W$<$b4?7ZhHJmb8( zWXk9w)o+LAOPT-8&O`sn+sD(#`4hj#J;bew?cYzgOi#M?g4ov#Psdx|PaFVHB#A8T z?Y<5?PWQLdPxtqa7d!r|Ljvu$kKKEQ!undH!jG}T*`3{_hV9uu93CI`gjrawgwqVK z)1TUWA`KtT&qA|>A5Zt&3$y($W1lXTg#8}Sa3k*jgR*Cn^uugN!@%*$<5h#xf%fgrLSjzNm^|wZS^`uGShku8DV_EKKwVCVWp}mn z5~SY!V9(KxH7T8x45x8>^F{zV#=d9c>-o+*N=56&&l#HTdC|OBe7Mejd|!IiR$B%B zu&~JRUQTl#<>;mDmF}Rw*0L7+=y&WHv`S|-?-ZeuR8jmsho{JB4--y=7i>*fCHU}n zc>%!$bz0gb(ImXmPE z>H@o70PO>8ssUB5-NY#=8zH0fT2_-s%dhYm&yWw})|tWFp&gRQhBkp=Kd|>u4cdw= z(eVf$uRnjA$IjLp&imG~lT|{{39VI0j*6HWu~ba5sv2=8$`q=%FuI)P zGw@7L#^?pwv*YsCWbs_$oSG68Zc-Yk5L=A%pvsSz@HZ{^<+Mts^}}s-{e@{m(4I48 zTJQ-N(VV@Diva4ly9HRO`azlbtmZs@k&)>Q{TJ6$lI`U|z{6<{teF;^plQNU@SAek zX0B0Uapmvh5aI6Zh_u?fllewDUP0zwOzJaae~DfoeX8{4?yIjJiaEGfWlEVL zzxy7boe>Vk!z{{fUZ60(T8!=o)f+(QXZz6ykbj1jKzk+8!mIy)^McbW&EECo$V!F% zJkEbt6&5HMRmx~0onCQ@JO|!(q|58Q2)8JVxk6THmPn-+@%c2MR+L6B11X$KAv^j- zK$%6DSn`W)tF&Xz#e)ke;pcak5`c#uq1nqU+qhK4Q}PKW#IlDqJ2H}K^?3#M0xfZ) znv%#A_-xlXn;QaI+>uw$DBhVdr7DQV0L;%t|Km9~p-9cY_yVrpwk$40?!FaLE42%S z!bqsSL@7aG6n58L@}+qHuFBeKWL>BLB~t{q&6t?O65PNNi4Q4w`2=wdazCe*<8Oz% zP2rf#Dbiv2JJ~HGC7+WX-=Md`!#Oi)i9f2A@cCSMt{|7>ZMFPX`G2B#l0RX`4k0LB z{SK$h2g%4!7}dSNAAZs9%KWp}BT@c^*(b!Pu|#-<$E+;)PSSelUhrt4gSoREeQtIt zj*~L2LDl@T+jAWPJ!=xpS+Hf6uB}1Ve5uKX1MzY8amULm<3nZ;kPa&Zb?$P0F9bGd z+wUr4Vk1j~BBjC!l|5DC90EhS!0<>3z{feQG}TS8h*|DxZ-8H0ea#Pz4pYWJdbDst zO^bBJuph6Tx)rhooA&9Az(aEZn<=s6Yu7^Z#<$`R2Z@iB;D|07|YUK%G1z1_~`pe3s6kQ2dtaa5B z4-Eviy&3l{(?xOR!D@ZnM{mgStk=W%%q>E=-wecKPJPNT&59>Hw=&qUelD4G(P^dP z^qw~S{5V?k{{h4k{(y0{iP@v4p2Rmd%K-9_GTe1@cpKMo!7CpLr( zP_<6MP`nsn{BnG!$dj64rwBOstCjQ80Wqfn$0NfqwV@8mX3PS5oV{v=MCEMz5x9MTdNpwRVmi`h}b>*zyzeEz+hzXLhzib70^) z_A?m-yHaf_3qxl3kMeb*gxor}^E!?gcapSDYLWXG@fNNg_tzYt=Lc$e_xLV)PpWkA z?r5Tv)e;v;Cv|UpVfuqljCMcY2XhG8vV0rCjcVM5OMIjg@QHhhb7?HwGZz(#c2RmF zkQ;-QFmWO47$<1<`XafQ@>cXaqpR7Xn+jzO<=D~d(lH0^$^x!p4Oc0)C3Wl?vbFan z`;#rkfktgzEWCq>7if9V(mB{NyqDEOY`6F9VC>|?P@rNA3v6vlV z`*q4@j*!Wqj>qC&pW#y#s$$^b*`S{zJ?<+ekp^?ub4VZDPf0Hexr6q^oKZVC-vrHH z5hji@V7=a>PA?lKozqzCDa1bX}(qM(nRIai}gu;Uft384)PVc#w1KadQPPl_y*uXj?V zN+H=wNYuYy7fv*tz!PrIGu1+?v#PM*Asn2Zx_=XnrN)QodVrPCG(2j7RhV2DSG&&f z`=pDmR8=~<;2ei8Ur1nqNU7wTm!nUcN|tm{+|`k$ve;5x^LUF1RwQf}7KVsZsk5c& zcTV+ee?`L3&&!h2o`Sej+#>37%z%97E{+QUnz(^9zJ6XnKcxCr7TG1;**;sY8sBmn zqjR2dOcdSnU6(ASLWUflq-(3sjvYSggh~SGJ3H{vu=p#GkvyKKRk4`OE{Y<)e~cx5 zuRGv#quaD}C!=mqV1pgK2sf)8;Cv1mygB0wT;(x?M^@H}&OgBru=s=0sclj`@+Qa5 z|HAi~kD3-2SRf_9K!b+fN~T76bVvwG*4>XIDRvgu$r_AZZNC_%MwuyW=6FyXgd6v# zvdp2w;CEpX7#Bn?tp+9Z-Zjc%;ER|_Zza|6!&^z;mMG!n{sS zKj?H^)}h>3D0!9(M=4XuT~W6J5#NRN!2J8{%?-B9{ZBYJ|J4Q`fGhB(xzPj67UzQe zH9Aaj^@qSxe*9P&gbWnEXAaXbgZzVYsm5pY!L+%=_uZZh1aprwEIyDe@)$BGJY&?c z?jl4o%Pff+tN(6Drd5H6mmv#>EL*(?k32bBc61$xpf!`AmAn3 z79SBVbu}n+s&_QzyFy>3@?@?x)gF`XIDdzYH;ki_O*$_Okqx^N~VA`i^ z@=|&zd}s`?Pj>IN=CKFL@W=n&P+PVtHtA+bFme67vB4S4=oFnxOzznIO;m&2LpW4gohK;So6EsRI>E?{AvF&BI=k+*y z0@^%KMAPe4f6Ng!Cda^czIHqJ=-kEx7EJS(Ikolz4_^x^?CWC;y^I3&Yo&@#`e zh@e7D=-APs`0=+( zz04H(kE0Cq5sr=o@eD3ed^$)PU*1Wf5Fk*`Y;-2J6taB@*K9{2!@_Melt6c3>LcFv zA&dSBMdeB}ycS1_Jz7o;cT*a&Kanxayj4Wt_r1LYwMCv^Ox3Nt7nE^Fn_C{72*GCQ zkYd~&ocrTP!J+(~__R5SvI6=vjkxkro@kl!Zrm=3gqor`sq>;r+xCt~2DRY*QI@bB z7uu)jTGD|FBRm91ogJhQYY=xFV1^;wET-Z?E9}GK6s5q9xQ(?H`&)H^Gk7v1T;xz_?6Vm4bVa(}aFSjlQCGnpfab2#_85 z`KAyBO%Bqlo5Qru76k*Xl8r4k1V=qhsL^=BRIKrsrIWCJpxQ}%vj~NHDkg%FGJ{Jm zN=W86sllB*fExv#5f5B`D{WjN3W z;UEI<3b=t0y%#cYevKB&2}NRXnAvlMO;s{7>f(s_BRAr3m2{6yR!?2B`rnQgc7QPG zS0gfBp`xk2UNW<6Z$&NSjKWk9k$bfruw~}aIR*r+&wq)Sz=$g+O0XI1U_!5e%5-~{ zFKAf8@&K%#jf%6llT&62iBHvPJ7^OZwcryMsRV_ivD$)|-U&!8Q7#U9jQ^@p{ll-z zT)hQLOELqO`yE78I^SfMUwPoV%B9iBpxSGXNt9FI_4^yQd9C~m*ohGX;Q^GuoGMw3 zWlZJmWlY7C;A;or;obEj*-7m(0@a(WhnY){rXLnX{W`aA0R<8F(Cty02cs-x!QgG0 z>QZGcyS5pYS9|Xk&`HmhTs|3PsYu-cM&mWgByB|X@T?M46h;jZ>?FMK6j`G3gJl)m z>;;L|151|7_vTnIml{^a+flw|G9tjOxdPBljX+D{4`vwP%Wyo-jMZQ2F4TKN(Y>=?R4DsWBuBn5 zxCKaL3hD%=M(_K_Oit%Gb#yB$wPE4tFA@C@0nwB!Vf7LG7Z349!9 zRd1#qR%7brSh(QX$+J&L# zx`5tuMU5omrhR9!eamYy;#l=kg~t;^$u85)pMi?mIF(5F7)PrLz(29Ru%t^g!AqE! z#W8Tv=qJ+ymz2?U=6pdJ)LW_#&xdS$@XCRJ(}*xb2c;V{M!K!7lMgm$5fyhCP+8*3 zqRto;uQJ%e&nJsIn;Ca7dIgf7YD)m)#Qh!+%Y{Zn=rT)irhijRuoz}(oHO9hJRX27 zw4zFFr40?;SC^Suv)I8+JQQTPHz{cx`w^LXjqS6dQJjX`9^P-UH=3_%)n&Y1ij-Y! zT1Dak%Z)OD7anP*(&L5;o)I(yY6>dWw6SNTY<#^JWym`l8jvZ6xXEIKjWhGab`OT+ z9_EL}u(y>#g75>mL8ov~lY!`Gr*YLM9}5HHH=jC*v_TCb2tg*T)OrEy=BsToP-KE7 zK^vCGo+|1J?8wr@kVliYd@2bX%Rqq~)TR6+Zy>tAt(J8ao_}iFN8mheb7$AuC5Vp_ zVgHC%@d{$r(WJ~DAo>=-`TYuRKT7eE<(tP7 zh}|+2LP@c!$QfH}e|5SXL6xsq_l7QN+asr7N%U8?kj}t~t!nSZBGKU(3mJ6Nb`Vlk zkm&d(0CYnzo}d~bo2#zTRS1jfM8Nd4*d#D%)U^k37DvemI z$SGtK@L3mBUrZLACDQcpwaAfGcfxxAuG;UTsoeY?08n3e>_UH8$>3hH^*7jW~H2wBN&pcjg5RzZaIY1liYxT(RK|T$_v6t zy6g8%_Qnq+Lxmi(mAI{~V{Qvn=@3TT7J=E7Er|wW@xkq((ZmG?hY`4npsyDq74wMJ zW!DtoO&uZ{r?$cqzWwmg`W1|LVb$oc8pNR#f=iR*y$X6zZ3^x!4flm(?O6=| zYPa@d$Zwimy^82=fXG+nC*+G;^aGU%SZH@|(?>WUxKqC6A(kzxcc4j2A6;0^jCqZ$ zI!R7ZS`4-u$>}dn!;5dur8pmpaxy*)dQpjfMB2Pp3>>79-w7}DSfs}*Q&m$qs$l(e zJRNg}$nD?gTs6jh!V>!FXK_T|^dgQI*y=*vk6b9xBALWPhfAf|+juWku&*FJ6LWeo zpvJ5+hi-cD_f1jV;C5|&A&2K2lZ66@G{XhhJf0qdKE)Zgm_AuQj4DzbV0WAN1LxvC zo=ZVWpndT#gI}dc>YVn*r(PSwGT8S*D&)uK6#BVx?mTP>uYjUltZ(28+;8+A*69~s zj2d^HV>RKzkj0M1XXuROH`}U6L)aFv^6sjOzpNKZl+_Z#eqtH8Khw-U0^#Ie#p*Wf z2S|4km-&6olyg^({-B$;=7brR-(Wu-h;|4}i zS*Y>6hqYGfA4n#%o8y^7q4o{`X~}3!}N6K7W=Mg^YoB7x}Cu9*A|pRW4ejAJnce4#Dq2fxC1`SR>9KUn9fB6dtPO2Yyk=86jL?axm8dYQx!lRmdkdO&HG7YIr z3(c5hV!)w@bu9WGTsf@$!x`;th9cX9*51<8E96P2F*xF}KbLtAvdN*PXarSfj<$&R zTP7Xe4EH(uB!E*w{;eer=;EXai~&HDb9f$4bQwg2FJv5W%H{W{jmhrPG`f04k(#OvnGH! z7uDtYYJ{#4b3w&Qy|b|a#!p|xB`r}V#S8iBt2dQ3D4!^&7yGY+z2njKpFl|;;%)u0 zWxV*{B_>J=oqq=F;)I*D^>MJKBjZKQLP&j_0%@LeQ~6%F@?9LX8`sYzzNM_rj5Q8N z33oRKFLGKO9tBThtz=3C@nW}n|I+K;*`J54Ph*BW-!bvL^a|TbqG&g_>hfX#;H>#w zgbF`Arhn@Wuu3^CMoRL#z26WgnEaCGGKJW!B9h(a(l2hbbW4y|+i@7I$47#kre|_@ z1P;&jyGCapS<4Ec9f3~?+W0;xkJm>@%W|Jy(8PAg0L@VniC+6e484u#VM0rK?*+5@ z1KJ1o^zES3ZF+i#N6Kak_F5&Zmw#X8*rZ&W*X7qQ-sNomO>0P({{7*O?UXqW_xo$B z{!|;O!VjS7n<7J>m=0n8i);*w1ZEK2s92uOsg}MH#5R!=mI9O6D8;VS7qGD{k0RXBeBze&?(fSFV}y z*!`kGRP~49eZS8_vmb|lqSn|mXou=ra>$MQeE9NwsL<1I(0j)AmS>(1bI*s_vKWW9 zfO&}1z{Z7*iL2WqP&96hrtTBjKxSL(>@1l_l{zwr^)e2E!by{+Vh54YRu&S$iqZ{7 zicVyAa5BJ)D#L|k6GMqkkcE*F!*p6TL2$ka(CZUEIFMfx1j>Vj6_ukK7Dd0WO7)mW zNB>=d@(}{>bvlyS#{{g`rd_uhll}an89=5b<-5p+o{`LQh?q>~Q7kV0-cj}^6bLl6 z(toCww{BK~2WU!+XeQ`|ep1uS$yryYwWK<$5_>VRM%^ zif;2yNM25@1iIvVWzpXE=(W2$Qlb-ZInV&En?UjR6Dbf}4^OUjj}}Q*Q9s)5IE5V% z3(E-zC>lgrcpujwfOvi$Wb~r@eKqpzH304ZqU`Q?0q&l~x|AI+YI4}Czu2C^OYFd~ z7SW2=-NK)AshCs4wTIJ<2SdvnQFyXXP`i0HZ=&B{tqv39Y@N($ZVYypDfQP+c-TJW zA8GDy+1%coeUjItV1wb%p%CrO!BPx_Raby;#6s z@J{S6cqa-0?}m=5SSc(#P3|NVZ#LAeDsSZ5&(nwJ+V9UFx7A3J?C3B!j1#~y2=fm)xlJL{-Lt*{L2>}$M z7uyZNn?A0!w+p|v)xTAtoxFYeEu@lgxtP)XzIx}A_e1*QQENNm$h3dPeYMCOD#j2(sc~qQ^q`X z1uSXuEGcxNa=%RgkE^PMrPTX1XiqeZtLu&II3SPVa#h5MMF)&Go9Gh*>n+V66HA?y5^Io1 z!0)RUuwk?6vAq2zC~|aajboG}h>3#7K{mB9O%@cf=~&b%7A(e_NF|-Y2gb-JDY^!d zeR(*<&JI|yDrFys*H|VkZ|(-ZR*^E&GiE&*@GD1EQkcu zCJJ2itm4c&cu_Kc^Lkc?zNlyaHrf716hw>*hKOj zQLl@>L%f#hIJ@otE#3iiR|vH$7^cF-Et=tga_CS-^oXm274&W?Es~y2_({`>N70Yv zAkh3+-wpiX6W2FPlcN<8;3N1tJ+fhq}vduSMf<-==YseC$~v)oC7_x*rL!$_M?cLkaU1Mj_JaMhxQ?+9yf~e?js6OouP<| z$BV2t|3%hIN%tb_l}37z^(KHIWBOOt`_ic7qXoEb?&scKT(pp*e+0VE=homo?~stcxe`7=Pg-qo!h%t93xWS_ z7TYaCMi0^}l=h6(n7ViI4i&{F<(mwy20B=mxEp+40Fo^KD#gQoZI3$~fS88Bo*`DW z{+0#-@osP>B1Uw%2t-5^WMSA}$V zt1p6!0CBru6m;V)Za~;MCohqNt|6Jnhg>FJ=hu#98cN)}*x7&|VP4ha0--nGK)v{n z!ws|uPSqvA_k2Hg)XjJaoT}3uIk(N=Ij*?taiJ3vxfXM-fGKP*RyC}qFP;q}crR8p z9PgfW{(VuCkpFc3hDPgun$d{c&prSBMXG_ssoM0eXlT&7W7~{W*iKnWg`7t2`E&y* z6-pX8gXsDAnSsFs%`I0dE~o14+2>j3zuJEvtAC6JOji@yAk>k#L}Io3oK1g$Z`Y!| zX!L%{XAV)p1ZwvL2=Lg#|Mppk)c^BoP=qiHlw;q~zn{Ks_E0`E4B@yosV7zQ={qE4{T-7?^5?R=U=r0A=3J@kCYZBB*Kmutes-V3x9KpL&74!2%ETZ?Ih;e4oXQ z+J$0;r!~^$J)qvuVVAQ`PHS1VO^36(Mt#G1>C2~ettfgr!q_5g|Gp&BObAFy^sI@O zzELFT@VthMmk5F2JvAB1vn>2s2=Cyzh+ktQnhO~*z~2!A{v9#s zml3NGc^NUriGN4zr5Vlke?~zj_R~|-JB#zv4nCDSocr22UB$wP))%Xn|LO1Yj~xE5 zAxz&phUd#iM{B%1uY_eHhA&rpQN267sM|5~1OHLEKWp6Ib#`{UOL}MG!~WChZydn+ zEP-ziKl(T9ZJWpGso^J%Yo~frLCA1DtLq`d^^cgo#?u3TZ)5fn^> zk#XR_L(C(KBxA6M5%qEv55;fPjS=C?BN zmT~-czNcgNN%@fB!v`GPOq5by%U>YY#yakt7^F#Sd)V+Ci}n-bvV4@Ni3!=yc}Z%2 zFuul2$W}i2l6vEQoOFL6{hM;X_Jxm)W(saa#KXF9iys4NC>S$%Ii8>`c_e0XIQ^R z9je*ZH|7T3J{2`395z68%ns>vvk1H+X(7(gAjn#Vs;(l! zr8OE&77G4xG7ivqGYU#BzW4cFZizEfM35;5TRCj2%jA$}cVmCewes40^QOdR{Eczf z_GK5D#KOV1RQ;q#Lob(^sF<|dnwg%S_@%pJv}}43x4UmO`^b@5G7ofCE-kI&hI!&y zZ=F)on317-qPu7A!FsGJFi~)L(N}Gm_3-xV8t8&2_y~RmYh0}O-`^5Q`oiEtSaMR{ z@a)elx+^rZXHMM_`GA%|lkd^_(4{4dXdfi^p7aAnL2~mvA;3%c9TIyoO})}@oQ&Kh z8&2q$BSJ1{*-pTDYU2#@12Vw(VpXk5YW#V5F~_6y{4>OlKd<2eJ*AnK_;E0jjVwt+ zv+*=))^(T--u`oWN=$kck@FgU$>eA>*l`sfvY_kNYcMZ2$@Qu9!6X@Vc&EBWy73hPSKj+WP7Rn`g3bFU zz-aasa$b!OCqSc=^yR{~!vB01M*X+{w0_mlDN8{%BUQHSUn@?BzN`yIQL1KxJFc^G zoW;7Dc3aX8j#DDX}4Q^)p?w)k#7 zUznoDZ_Pq);Tn-HPvBM_;;DiPS57YxOBAKRle@cPL^i{hn^G~@`7>1v-zz;o`8{Ou z&zG|PHDcH3ewg~xhrTiv#B;hYC9u=R40@{7c7H*yX;i4B;)qvD{j4O z$3N>EV8f@08>q>@ajsB%5NzNGYEUw*U?XKEJAThA^)X?&w>07AOCk zisO}7Wu-(-G?LOlTbLj0O7209hnC)SKNQ9t#~NjPf-w3N77xX>dm7`Pr&D(OZHUcM zUKF7x}-rag5ToCYv^o5={q`~hP3G=F|N>S+?$6)ku_>$#YhxPNj7<`@yP$`!gJw6X}^pS+?jr zX+1I{Cvwi7S}v4o)&U$Av%4KDJZdap4aU!Cj3FTK_OJ2^S1bRQ>Bq0>n6?H-;w)T&=IM~X& z${)}OzATA42yTUO$~E1QbHep(zaD^@}3m)Q3F=Vc9I z_-T4)c{{Niaqb0vCi_Fxp+=fb`@uUp-j(1e!&m79wP`&y%AY)4Ep z<+&`8Yh{?5hvYyMVb<@30zB&un|1)hT$gN|2q-?q-W z4pdljJl(xe`E}$ET4F+~?D4-q5$?TU4$*euna3|ignsZ}FG?1oJhp%#y67P=ICK0WD=wi_GKAuk(VfZL8+|ZvJzP!&aWS8Hp~@7!g$g z7Lt=vT6q+xn%064SlwmiC&{15e(Uwmazb_14ppv^X@v+4l;bTpi5V2$@(hd*zMZP~ z0f3Tu5r2Q2q>Fy_ZIA+{Zn=@$d-pb@W~qc9ZevETFvmvZac=~E;JIof)N~x(2o%rzs+#r zU-fU${V;&UV8XTa$lIGAfg3Y!XUiLOBe}vCG^~H_FDmslP`?poEWJ6X&ezjEtK-1O zA>a>bAq~>(G%U6F&L`h z-R-)FEkK-kfwO7@X&2XZ4~dy4{47M9LP$!*KHzqDGS15`d+2FWsB zS~Fl+7}H6@NFnTI&lNCC5X8#PS`N^zTzz5taXNB$^hPh`qamPdZjS5GncrVH#|f;9 z^_lVAdm`aEbryy*xwC%%ChX3-LbdyC=vvYdNdZ%GUP4ji7*spc*zN-3d?5dN03;tz zZxJg%j$$Hd0hZ+3w+_OszD)ZWkos95#eNEExhldUqV~+*x?SS^XU@uy8m>8E<%zLx zc$^AYItxP>!SgPANQH#k4t5+$IlaFMF5qwAlJeOu>~6vcgx#WWm2bEt$|r!dc>=GG zbg}9HiUEUtE`dOwSDhZwXz2W06vJy2t|HRwxo=+;HvG&9Fgl=)ZG5%<2=`8?W28d^ z+sZ@1oup$eD^ncG>q9~5!Wx%?5V?7!S;7_>t`TI5KwKkP7$_QnL1X}r8cUXRY(g~Q z@IR0k3_tIs{aPW67!;-h^~B<_G` zKs)>R!8kjz5T9n`u}beOx!i7skJglZGw7S+cEj5JhEn?zHJi+rz%)Kk-UEz67mjV% zT1KRhdIJgvUGBVpGzGZX_q(!|=`=xomySn9!r-@6F)oZQm3cs}uGSN?mbEcJLrFDS zp2|gqwM`a*$@u>5uq$DI9;=_z#z0GF{ZvS@%$}wJbM+|KVfkyMkA=Bs*ZtZ6!CoIa zXvOLSaw5AGYHZl6asJNpuk*fL;)mLu%xnvDq`_uJpWJ?f@oc@n4Dtc1Gf#fKNqx=i z@w%a{&l1UXt1G%@{vNzsGvAT-bs^LF^gzB!H*5(#)3JTMs1B5#SbV5Yhraw$`le`n zF)x*Hc`@KauzxW>ef__~d7EAJvIx2b(Dld(9?0y^MP>tdQt9DZ zUW6_IU#jrF(`3BXG!2k4gt%SzCBcyRvenmN^q?6$mrS5NG^`Iy%xrG0VNZ#8T8K`6 zmb~t82DRWVrL_>K;%R88kr_Y|-t%Rg?T?gf3YYy#C0w(u#U?$&&zW>#@rz~bf4uzl ziY?@SnI`_@1#fc_7gCAAlfHbjw22}i?LUEsPpl}V*U$1UGv`->x*mg^!Pc&K6*%=9MIKN4Bo;@xuAG@kl}`;>sYZ}=Cn3{ zgi9OG3^a;a?0e@7_hNge$qQrU;)Iv9$*$wv7RwQ)$?F+sJVl(()EBHueT)oiL9v0n z`>sBt^B#A>_LputWSV zxur6xeIdEkx261w@}(9h*A=H3*C|a%YR0d03rWV_|M?p$vR|^5PE(6Tg6iUenvvry z^4SfQwrPdW(fFkNTu;aq^*U4vQmajFm=F13Lw?`v+~LnQpHwp!SOPS>M1>@wB=dqf zZ;HS_=x&BcX!SK%4%WI+;S&$F2MH3>bX4A%EwVh5=1!=ASP{|kDCy@7(r*>AIXrT5 zT|Qru9XE0uGAzcah|!@eOWH&R;1|FITA2O%tRe;l3vU=zsIy4Nz|Efl7Hf;)Gj{!M z5oszCd@7jT8k)oCibPvb>->;5O%sNjD(sFMm^=o6*ajMlf9KoBj^ZfgsUTa(eH^MP zG}xGZCYl6mR;uKyr5EY`4jahc!NvfO>d8}vF)rEt4l{8gczEp!jF2OCMVj+>esql$ z2yRMJ)bBbd(HbedY#8<@L|XLxv?D?@rlUw93imEG8Ky!5&t$Y%3d7oU*mlwv6c_wf zl%rhy^rVBvQ6SqGagH+vtA|o02d^R=D<2{9_auZ5c7ZexfDA|_5?%Pg0t#dnUln<4(WZ3JAeTc(fmg|T$+t`&JfA?R*CKA&O zbg|XFlUvDDVg=H$+-p}BtFjvBi&V8a=R@dyZJKdsqII|_ANzh$zx|qmKJ+9-uM*}B z=)R9vt-j<+*J!>(imWg>k=|f$;1E%NN~woEf{3JoK7L7Mxh)dzp|Qj#_{xT~)0B6- z`AKdWHy=9Faq)F$AM*{Z^gG0{?2fEsG&O!_7pRnndnlfWA7;A<;jeZdJLs@U zP2V<5RSCzHoLKkM8Uq%h(az*D5eg;bh>>hz$i8QY`+zpo2E(Pk#qtOj`hunk1=MpR z<-DWZg#;y=pQt3o-4CEY1JcPps38<%w3d4*oj(1A zNq!F?X*Q%q)*oXmrK@6t>2OFkcLL+wU-lG9Q@^Lst}tQYkOrFvtTX~zrAY%-?f3x1 zg|W1AB~wyyQ#Y*iu=Pbv!5%l0m{U@Eyujk$uOqGI6zoN{#Tq*ja$SiV0aUZ~rnk~8 z{VqWNKC69P4Z;8$`l!(LFzPQn#G80}ek~oco`3>1cQHCQPw_W5`C?S*7BLW!bst-0 z{%z)a_HX*W$)-kD2F4ck+R{)I;19B35f${X2j59-=|ob8Jv(bu8FXdUy3n}i9Eny6o`lBeu9g!V+QW0@w=ENu}I>IrAp3d&rF)V^BFe(kmz zkloBV%c`gz)!N<{z|e;=6!-a8jPpCKH*y@`Od9a6hMD*CM z@OO&GJdBiX;9C10QXS%QB-j(NwDuQxN`p#-hE?ZJ$TJ(?NFuWePXeD>j%>2^#y#RJ zCe&tEr1%#;zeASIww{2Zi#Pl58fLomBgvp31zf}#QpGx+($rW;H`G0dFT;@aEw#)! z5Mgut8o7t&Vy|{dC!5v&yF;5aE%ig2gGzEr-+u$3`DkZL|oy$h#lEC2`eqKjH z_CYD7vk@mXkz_%Jm4H=}k72sX+kq~z@{U$q3Gsyji#JZph__bqOUd<`cPwUGTryN& z6C}ZpGxzsd++@FbPvivTDDl5x&x}?ov*r7HzGI>VaL;ZL>Fm!jF}6jlSncfyNWV>s z*8%Rn4VCjU6%BV4#c|^ABVDld?~qH1`(>)*Z1`xTF>XtUOYzrj>^sX$VCf9c@>;8(Pmngwmm6pF%NPvZOv4wFSp=?Df=j&^pz^eI72xL5vW|9(M6R+I z@KdD~D4XJP0`dm70*{LiiU+(c)3|Ys2x~_uJN0^kfUXD05Dq)%65GO*DA3U+`k`W*3NWTY)@ej_=+d^(R3*?p;DFbT~TkIAm z>)#HZ9JneP1euv%r2q^Tw@T}dV)ar~9*n2?Th~(tnEndjWJrTeUvWDm@EKi?ZRn2? z!HUx}?}>awdTCdS0y9%&S&}*-vWi~%i^L}8i z#|)@#)NLQ)e_Bj2sUhwv!(TKy?-5c-V|>BA4=BO zrL#ZJK_<2>kTAC3y@1m+Psm9!*A%C3l5xC<>V(42Dh9-l*kK89@8_F`t>i1;7~{hn zIU(kJxiIHg}aw;32TqBz{tmUv@j_~zuTF{4czNJ0PBm&j@T5BqB91o z?~GhT^4;i|(e&MxSaDmuM!ix$D?^&SZUm=;o;1Xd7-_jMw1)1&=e0Voo<8BHHS&Tl zG>NHH)W|9W%~l15MP#a}`K|rD5Em{D(+vG#ei|+6u4^to-YV4rnk-4<#F|VhU2kV3 zY5!bn#$=vIoEMURB3(c0O5q+4f22#^#i|&X3^su~JC8gcmNn`#^FH1>6M~V5?5n&t zy^h6`>1dLc0!$Vt%iHrUGmnG#A@{~(;9A$tg~u5d<{j%;OHl+Fp>Jj*mfq1gw0#U2 zpgQW}2(zxj(xT{N!4^BwSC}%Fz(lYl^of4L0q!o-3`J@YMEJ1U*_W5`W!H7utsZDY zvP~$GA*j~)CXK@jXGjb%R{G%~dP5t{#7=?nC#zacN(CbJXE_gme|n=Akz4j$lcUGC zj?6Qu2Q^eWY(i!W8snB{h)?CevXC#jR?%lzQE-SJ~adl$G> z!_0$&)Xyy+zaAXExDvLB!a?IM@q^)KC<`bh?#I>JkUFJEL7#i4i1LqTD#=aZi(|MQgYxfulB48RibWE*o4+?g;lr-Z zakab<-V)`e;yz?4kd^$Y)tf@xj?dpYJb=g^g_vXWAr9z6_DdWcx^YsVq9IEC@n!OI z`dh7ScDN4CvUfU5G#2@2XTap}d#`*I2n$s8E6mSMmW5zLqp3RcEW4k$v~xBTaLR+I za9iUhN`HWHS3m(NWEHKr;)r~g_L^O8p@JhON8~}jy?Q@mX34D+wL7)(UvC`b*ddyP zJw)OyY&EyqSLPgxB0AmzroXqaf2$w_2{^eWYQ=F*cS7#?8IUna2HGmd2!H*Q&yIqZ z!tMsJv6+}zm#FjRLCms1x4rV3-tCL3ep_hAqT;;XA3+`_^R@xyquC`pCLk zK%6e>{SDK2!UTl7X+HJYQg?53&v8JtCKE+EIcx3;=Yzn-jO|{%s^DIpUBD#6TNxU! zdfv(5K64p{yMyJ|Fzq`T9N>hA0@jpYN^inh?x8I$mwFxlYCcWK9LMDa+Yhz#3s|qR zHV0u7>5hNo=d<>3)>7}_j_Ag9vC`2+!Kup((`6YauU5k?JG(%v1i5}Txq;ScV(~;m z-dS?V6f}e1iTf{K$Wje93jMw~&Vpz$sZ9c0dW<|?btsj{QqYzh>m_Y}RxL*5`CnX} zbySpH*zOe>kPcz!lx7C$4k@KUdIltv4(V39k%pm5N@B<%B&EBC5>RAF0g;x5Gx(nG zTkD*EShHL#7qg$e?|tuQ|E}u-X$tN;PXpgF3!|$WRYFI+l$-LK@QUw)jWz9pPD^%M z;C8~vh+EnnuT$lcI3vucA|7EToudkZQ?cfB zcQ62JhTPV}E|3w3N%>>+?Tfe` zw9mL)e74x?nbH7KNAibJdi#oYJq7eKRMmDFbo#8fsx06xq47ivZ3bTPmrtY1vVN?x za@MCjLgXfD6Kn*cN+=B?mJfqe2=u*IugMq0MBBn;%^WWQ4O?b?yv8%?i}K4=T*gPm z)S~^SRXXEjrBdDcla%%f!!k%cr!}@xeMFl~wRrr3*;9nYY1|Cn?e#ya1RM`?4lsAM zAx^2Q&4MwK_Ojdqc}ba!zlnvh+~nnwzUW5E3)snWTO~P;UxOb%JuJn2`g$=tBq?Cd z-E4fZ%|?}bbS$K9i!FN({zBquknZ>4l9md!c00~+oR`as?BBk!YgHN)B~XfNXN;@R zYDFrfZ@0OH)axfCbr45oj4!Iw?Qph0+QKr%nZ>5Rz%>mUxDh$&^}a(((T7ZouJ5W> zj2l@<$tX&fjZ7~Oanr8H9kM}UCnHLXKUj5f(Au}|9EIrRqDRbOJ=jNqyxJ_0fJ7Uo zIl{`uGB)8u;9{0yUHg}f-x?Obx*J5k{4n=F%^YbIpqS%zwd-85@W+ZGZxk^avh*6o zNSc~)wC4H#k7hc=r`C?e^Q2b1b)Wc3=;Q=jW?v%Ps9e?d3FmwIprBX;-5XH>P5bB% zyNeA_fA^oijr%U53r$<}%)k zo;D=)VpPtSJn6}eGLw1TqGPfWH)QX1z0xA_ko zS@e?^UL9fj!dEl;!V+7+uStL*8?oGCA8{r%ewi~|wQ0a@+##;GgmEAeKS^Zjb+dC~ zHC`EW;pYj}aOve7CpN?C1Dxzn;YI7m17h;x0f;kEYv=6J#Vn8aHl>G55h+7|t!{wf zo;#(4du@a$oO(%(2E7uFfHAzi>5Xi{L%g^Rk-A!KE7`;4%q(+dkHh|t0;3ks(A(5C z28D7b?}zZM0o>oe3?GKBvn|vnB&ly8i?P9KVuNu&i-PqqK*?o=JQN4j(s(oW-`g}rJjsQX-%iM5?E0aemtuaq8l~bu zWXM9L%fEKZzg^Ci*JxV0k-S`m-%@qxs=^Y)ycGArXu;t3D6;H!Ec9BsvD>gqwDf9( ziRor6_D6SJr;kBrzIP_X85{pMrW~TC{veF>tG1pJ(41g%6vKNSZTF8pS>4>9tR$d@ z^QGN{9{|S$-a5lRd%nM1H6og^#_RMN^lT-vJljjge%256K2U8A+kPR{nHSr- zwgwm@X7iA3C|L#UQzO@aCq|JJ?p!7|ixc5B<0Y)SZGXTvTxIRT&nD|vE%Helr!)d@u%^+T!H3v`x-zTiE>0vW4<-h zBlMMH$4x3%+3jK9R`%Q?wV~BIu|7{v#w(O;?zt-+gDf zxUOUAkN=Bal5P~F{y~!m4%7bTf|5-(Z)zM{@e_EWN+kP}g4paU>DGZ3!V`=tTRfu9 zKfVB3qj>?4pCW~^s4kf&_a?gLI@k4`>S%?^#OlEio^+;=WS0O(2IWlujR8m}jtPN; zzcqYM-s;r^$CYiif^eq>r zy;nb?C`-(n%hX6IE-w~+jt(LpBHl31{&mm}S^TP!o4sPY9O${cTHzf{`@(QxO~DAu z%rftLwSp_2D$Y1(M8IdGX6g^^7e6m{t(q_*h#qN=e0A@Jyl>mYIOu3{>b&U5^S@J{ zG3BuQlWaDQdUmf;c?6iURsr75v=-8qXADk>7G14O?AYLS?}i~e>w=gkNq;zPrd!Hx zb5Z?rikBz*^=PJxI9HQAF(2|7r{jSyYl_8fazD!7hXmuS>m}r^)L+&2j#S;6^%pFj z%4XeSBx>PbT}(K)@`xQN($K=Cse*4qxCdK)@CSEZ*^ws`GN{Y_DXYOi|BoBJocSAT zs$N`<49qsKci!(u*?-w;GkY4Qr1nrpCDVKLG8{4|)AI26OAzGy;mX&f7|3A{=fnDJ zSeF?oW_{5k_P)z#Dl0ILID=J~x|&gXpDtC_PoKKJ21qx?;&yZgH=e4B! zXtyGlGB%h9Q|irJoww2N!A2@`ZTa!k6ghyo4kqF&W^7?rKCAkH+(xjzu^B%Aid5c; z|CtyGy-*LEjS87N`g9xwY*L=A!=G8rJe5`DpwaOCi2-(0}r%d}u4H(-k9PV#XsNDg|ky^=b}2W8&i3e?l1*BG@V;`}v^`aO-c_R*2W&9MX?u86I%J3mnyCi&q!&VNQW#}$yc{-iHG2yv^4fi%MAVs937P&B!=<@-? zoJzC^7)1^-;)MUm=w-wlNK9Gpu)zc^ z1HTTKBgWnx*CAuwg=)>cKmR-jQVrS|P2FT~Xj(jMbmWV8(L7KKrl?z`25h^*oHIEw z6M{i)`oT|qE`5Oh{!=(isw_++*dJ<|vCIh@YPo|+f}e3+)*g-n;NB4ym)XVeMrP=r z2cz=UuTA-6tjC^-GuVV7v5dAr?a`NW`9iZ2pWT7W16qfmHXw;qzd#7AIqkqA5V*W6 zi%Tf$1^H7$%Ux?Nr+O(Mk5Qk3-|h(#&VJ!j@>V_|Z-fYX$QCPT^<}$=T*^lMo4qzG z^xtzq7V&XQzIEyk4?bKmYoViQvH8;DP~E|+ntuI_*g%QZd`_X$D-xo*Ir)c*P9=>r zM+>)^kq%LbK|pgn9ZEdeO&#u*D!OIxt(eVuVtRay!1cXe; zp#Zx{q}LBuahufnZhNp|T0Z&T{Xo^Gw7K4J^RX4H0-DlOS{3EbTse!)(W)q)*PLxc zYG^k$?=Ya@*G~gJVRP4?j?G~j?5G?icEpL`wI3kBFiihbz)Sb6NhA+L{je%ZsMCPk zD;DBM8WCNldleU4Qz=eO+w$N+Q}TiII!AwHi;B0<_98FMm^I&qX^pNldM)Gdjj(Z} z;Aro8;)pj%U8-+^)>Gxk1(SAjePNfu5pGn2%$!FQL{=Kl(6{RK7~e?d44*3B!`@77 zD#;&91xyv9R(im0O{j77fE`^?O_D5gUPIfj(1cFtMIYIjVVv^5zBIZ}pC1*L*xon~sD>lC6 z>OozX;lUqbj3F;~R}M|Rsq4}ShR%%#%i-Y^YE(24AXL+f@i%URnWpGD9!yRPXW}Ls zSXS%R%()EhM)DfP%fb{g1^t35ue0@igxY4jQonRC+Vj2Y`vW~RGj1_hoa~l&8r9N; zeF0U4Z>7#2d`%JH$(+u_+Bwq2n?MhE5X?lcrP>Wz;J#bJ7Rwt>@_Z!PTZ8+1g}`L+ z?`&X-LC;mQ%EaXMOg{NIKg`@+^c3q)1avN-hZAF}hYkG1sUiR5;GjlV=t)C<6{CrG z4=CMm?2#ywP)S1rM`Ye>cj=iV?jh;*&#Xwm;TzIRHYJ!iRW+F4Y=@wsP6w*>inuy; z2fG3G5-wqLv361QT9PLtYKv!3R&lK-Bl>#o)yN^EJPZ%o062tF_PmDJ))I zzq6s3&6?+CR|jfB^bG#RgYX0sGVcRCqmrB-O1QE_IFQx{sxI=4gcnjZy_J7eg-?`v zda$DT68I^-IER?0__hf>@6s)Oj?y>qHRI|h%@8%1r=412G;ri+>a}5z&ul}q!opaM^Iuq7@ypFLhsre%HH3vLvt9ktmElLWvc5Vwu6-kEzBKW zU3U3nh|=+M_H>Wk(Xp?UN3Py|f8H|rbyxaGkJNr>xHBZJ!+a_SYshvUJ@Wb@DO?E( zw_Yjocs2N@e?J@PWr*Gg23savqMier-SxIv5*vm$-(FNU^ItnuFEettD&TRYpit7E zs@Vru=f+hLo;ed-IZw0eL2x6e{R^e;E8c=;BWgQU6!@F-EuZH&@^)W|6-7ajLxj++ zGg{31;4O~hBp51t&^TPDl3nlJr0ktDK?CFNsd#DNW124%PjFGyYyNq-(PMHo+I_N) z-V~>{02}2xT3#839be8Py))@FTx;lG01s~RHGRrf= zU1usQm-*W#4)c?fL8Tr0YSjCQ3=`6h6TI*7`&X8>1iBD3bei=c$Dghx=Wr!tz~yCh%L68lWc(@bq2nq%p~|42s6agi_P&^;N^*;Hrv2BHh=!sFE# zHf$}BnYT+TABW!W*HG(nVgm4Zn(d{K71a=H!Aut}kb45^Fh^r!skuTz9(E8foBcXy zPBHJ~mF~{OMe9!LI7003*W_fhiWIXJ7b<9I^&{*vJoJp6r7BVUYykbvwUF;LVnO-7 zlQmPVg1#em#7@;fPn&PD;^HR*wdcy{swOfy%)>>#k$Qk!l#Y_8x3S@@M|VlJ?xgfz zr&wT)hWzb0ljRA8Bpx0INbv$)-8WQ-__w;Be5`$@E*SCD>Ijz^K2B3ZM;ZEXx(}cO z>~!OQckt>!Bkz_;HargeMW)T=cDreo-^dS~>3SrvEil93mWXeYcFK_iVoe(nl`UM} z+u2VrRQ1EwyaEr&9GlUVA(Az@wxHfs&og?|%7auiNZ8`geGa`Qh`C$Qwr9FsOFUud z)<^r```4-*3Au0<>TwGW*-`65-r(=OmApm6eU*#+-b~HiSiyigLj9<9o4LOG`70MA zs2b&hq;vIC)+^Vksh2hL*+75 zg;NE^opgT`&2(oC2K(VFB9oa#IcP?BH5w6{ zvdT)2{Ay-=-)!KNp89&5ab#qrp~LT_gV&x9?C^7$4p?L;!u>+@RkW3`kFP)hdEARJ zOfUq3O@wuA%f0+=FuNDCadR*HIA;7YbO>l$s-HQeRQ%}R^npPoQg&d8pa;F@z~9?i zLJF{n&TMg{f`Fst_16@2yP@*d$zY`RgIaOb$KA4cqTKHiVfRxjs?3^CKL?H243W-4 zMq)G~AisHNn_?mT#F6q3Q0^__su8YDhGik!X`-X^oT=!kHQd4zB~LRW{&oBbh*EY| z7UxrW$ZY5i2-Vx;v?Zqxr)^uy>|>`j!5ks2FG{1xA*^eLs6R7isJ{*p!GZL1e4&O# za^BW*Sd9JM!LQT3<*WFFYqD(p>;wS8y8wN$4}uAlVSrJZe-TOA10W{h?3Al$*@`;c z+>5uxPim920|Hmoo^n=;+59_-z!ksO(r+W)!*YD#kNXHYGmy^J%g?!G3iM0>GO6-^ z3wMb%M0ExDGGB+OkgEp60_gRKj} z`yCz6!uMd@*OiG`xBO<0{a4qqH_*S)z&gKJ{;A+(`D935`C#iqxLIV_d(4RMEmBeP z+g2Bw1U~rXjjkM*?Ku{Y!(?Vh{9`My@Gb%3NE5nId;ksH;gf9Ru=&v=Ph3;oQl~7Wtf@?pNgG=q znhbR2iEf~?a0Isn;O4}_Qu!;wP`lil>~QpE5ds$iH%r^vDrrXy`%%eLvO3CXb!$LT z=(cYzXSgs#Ka53^;)Aeq#Epd79xYvqJN+`rOImSu&Yj`!mY3qALwSeVr=G(ucDxg9|3n3!+Qu#$zDa^Bq;O zo1%1v!6sMrNuh)UYa27>#f*~4<9UcrvEpLHF20V2Kq zKHCu=dZ8E^0_A>tog+(oe(u8p5npAl&-fmxt{A;0zDlug+vWGFM!o8ODU-+9wzqY5 zcQ7`8m;P1AF&+J$;x|ek6I+l0a}P`M7RHf9j5nhae&oA^jo1h~f9U9e8zJFD;<@Us`~5&HttaoCxgwIWFqBoRVoDjF;)S+LrM@ z->x1xyI#8@x^=qj-0y6^y*&DH?a0e@@$B}kw8QJpj80|H*1fq%LJDe%WjXtTdJr5&l-o(F#z&?T4aFO&!TwL3^}v;jHPcLMhAD#W&>i83Yafn zP4e&9xw~#xLRCZb+ zqc!ZjH=korHXRcew=pc(_=k9_95KE>G$tsy?9JnUkTpi&KYx*x|AGO|?}GvUnI>SK z|MM65`k%kZFLNDm? zrb4m#FsY36>WkustdV*2Zbhywr@g0<+^H{iGTo0@1jiz=U3zmaH;iA&6Fc~gNVbr* z>e`~uER(#YTsbaHBGykNW`)YG56T6k&`yE|>oyYuNj=T^Sy z^dGr{fi(fQ!<|>>f39ev_GK@Qu9hgg)<&vjZYC!7WlV3kVt4O$u88siZrlrm?#^X+ z0&?#@qTjL#3N zrSC3p*ES*oKHS~PR2_*Xry6y2-ArHPwI}W0<`J*nR2YbyZ!rWsAD%gHV0L>`6>xFA zy_M&GQ#!)nyA{;a>E%BZ!t%3ua|t^wrwv!=FY3ouUWaMgs~;-6%tBAoN_Q#({Pvx? zuh2;k$FUQe)yUVTA1*JZTZu$6FU188F6#pe_2&mrTN8QL10UeMcq;rQPFK_L^o{+_ z+q7!*cO4-@hj`p=6YU51+)bM<-<)jk2i&@yD{01q8-le|b4_I<&xURfcNK2(1BmvI z*h@dx7t9cuWR87nf1jY7< z{xMXTXOZMKAJ=@EPP;{byv$ypqQPWl-#ZUX_S~7Qf7l!5Im}f-R1H$-i1T?hv6l-T ze`r)J6+iEGEVW*L=A4f>-*UCLpuIt?sYZ$j*ed1d0q`) zz#9JU7`JhFE|%iP=>LQ7vaoF#zk7u^v!bZ}y2d|ftdvd*G%`i@gE`o~U`4h7`ikSk z0lS@IUlLeepTS2f-FHaQmoDF#)b0shJBXt$isbt$>-wlLT88o=AkfYxAdG2_B@CbN z6lVvv_2~*T$eyAEt(st}rpY^i#@&e)7*307KOVWkLEpgxs3LbAzB3EaB>iQ{V_e;d z&^*9l43%GhY5kl7RCy?UfU}4*R%vWnZ(D4Xhh|HN^UUx;_SM55x>^1I%6KELg6wJ7 zL8h)Rr3ce8*aI%nQGP>lPlUy{aiqJj)!Q#H>z$9}N@k<$vV(D+0-{fIo&vzL$L(^T z8166x#wpu7ic$@TX}dF4bn~GOj9(|Hxqz^FzJgbexnDFaHPVg}HQ#A1mIS zM4V%S(QU|!0!go#*V)cdSHZgZPn646e$PzBYBGMSiXtUrDa72;G}Ao8=E@Okc`)jK z8%WdV!0#5S@W3r#CB=2u9u3$1CSooJ?G=WSn~6yGYR+8?TzC(p=9gfoh&-qp5$k2A z%q?g>rQTUP?n6q;+D_-F#7-TH2GQh8C-sUl&8VSMY!C6e*=E{GvREE%A6i?+uj&lv zqA1#4zNlMw<0W%>AuBO}zbSFgi%=*;zna{fV_E7(gT37+&6SnU%^mZykJ!I;o(J+? zY;+d5mBx=crr3oWae$<8M;^4g;Q1y>W0Wqs;)<&=pwq{T;~BF9%|1TxI*Urk_Ne$4R`Guld?LDD?+f(!zTG_XNQs+lYTbK_vT3F6M|XevCBo<+ zXrl~EJz*{E<(p*t**WfCfyQWbJR0bpf5{M``p6*L zInj6vdXh(dXzbqkBfG7259)zGlXS{4)phV8pK_AvbK(H>#cY93k|$y1$u7B}1?3i@ ziMh$ig!dmZ1aXG@Qx;MHN?_w4!fau}d90iC2qAsBkz>=G0FDZ6yYB!Z+X}* zwTjL!P?K&)+mc%pZO3_bWUB`IQ%^S*@BI@csFNyxwn!|O5t8XFC3q|~euW%IDq-0H zJOn_x_}iwrukciY)K5ht^jkde(A>V)qR$0kQoL2sT=16(RMN7c)|=M}VcBtq*uqN; z2imMGSLyl4G@0~Sfu;U1rT-+G@8t}ozW8LVT--QF!ov|>xXNNdHe#S?ejRrKXLBU3 zk5kaQxrySezSW{JxSc9?wFo1_jSXj!P}0|-4u((iD*zTm!^Swdg9Fc3)-FRv1b*FnfkQ8y9Ssg)FlTB7f+H zJ2~FWEK>BRqcP|a3_uzDS06*DQ5i*Vm{laW%}MGRXmHAySSu12R84Ii53 z<>yj!Q#PI8_{YGu3uo=C9Q^e5s6I|gY@l7+xg<=8`wD?4j*Y4>{ms?MqZ+BpRgq$7 zZ*_rCDV(e(Wd_DizSlK?0ajHqX62Zf+;P9`;<+-pqDhs`+=nF0KNH>imwOrq+n4hX zYsSNVldieXdnVXVq95g;7AW9ho5c7Rw!kqL0yK(c=D(l+@Y^v>jt}(qjnn+s)H=@b zg%VL~PQcoKrJp)iqE-jIEYJ4KEkSuO)?w0>?N{bo(bI1LH)|>ZY6_LphK9Ni;^B}f z_a*HjpEbEfbZFv=uvE5s##YSdZq&{q<24~?yJW^F3!iy{+#H&M69;4rv+POre{pQC zpelRgCDW@YcOEG%Z}JimOR_qMhTf_Tnnx<;9;CKkReEY&#$ zDWb_a18M`95)c=(1Wc@f=0)zx0}q^JA3V~=pagS)b5w-zW_mgxxd7TP?Ly`?Z+fzbe`i(Zh>a524*M`T8KOd~F7j%QCvZv~`fLzX2Lc31* z6TZau_HSe(3eZXf_0Rdn-Q4F(y|ZgISBUYj9(ll@k1{)@u1aNEUE6)>B7zYoKsmsL zE?lzpBYC>B<5fap;<(?$N7&Zfs|Mi!PivP#frHx}U4x0D^;kT;{}}s20)FD=Z*#At z^65G0_I)`XSP^Ms0om zY3e1QFb9|JIgx)C&oxGJ75LaezVmsl$rHXn6;EEu0H9xsX&}P%o7kI#?qD@Nmg1`% zp{8w4N2j2-N`EDqPVwA0sRK$EOMR`L`4}akdjM(8UzFAPb%g+qtSKzy;u2LMLv?E>X5HVrIFMI!}<(r zTeEkLq%h|^TQ*y^zSx#=&FQHBbr|r*%6XmGizjsau|uTCeVuAf;L|9*Th9JJ)HQis z&GkGOb{Uu4n`?UfFoq~Svg z)-+RY&{hS@(9w9K$g2#@*ivsEu4Lq9#T%?rjL`3{D-%vWu+Tq^Io!!53QOWn(h>5x znR^b;>rN1jO}tAPrB)J6Ijdp}j*mW?94d=X)%Qi22=k~y7Yw%ID9^!L*{@P{L1A4M z0bI)OQ54VRmw3XL?!j<9l1D*)R;y?RhZ!7Jdq$?omagV9!NwZKR8kMOct)J zY%cg&1*&m~Uw;AIVh=;ooIN1DdW@ofL~CqLYZq;Pl)<*E!Tiqxqq)9&+7~y z^H_-FQPESZo<|S-p?ajU5Ra{LmeOn$k7je9Pd*V@jg(K&h1ql>@Z`hq3{_GAG(vtMW_h)gMS=gH+75O(U)=7PoPj7zpAq`@>3xZI&Q5l_leu;#PHA$xVFsffd` z7bEzg-(o-bJ)VYaNw(Zl+*Y3ihq2LA+s47TGSG61fMsEqcAdQLDhI+@|Ddk3Yut0e zTC8ES@f7dh=9lJ&6(Ez0>UPh<(DH@>`%$A9^|vFM!<&6u&upw&2do6q!jXA_=XSSp zSJzXOuz~23y}?1O$;Ke_SeC6nUX5}aSK-JGVd5Lkt%Wat$ltCPF-{3IGsY_Lq8CbN zbOB9Mq{Qj9AgM&h(gJmk+EeqKQM?VwzQpmS|B)QEx-N6YT4Y+?8vF&=XS{&EjbskE zMhorQ;^p(L6LI}0Dp%Vksu}|$>(-U&sH7aLrO^;Hv;L$$J8R>{Yn&T>hz6?(x|& zB-f_)Y#EP`pR716cXfNY`!^shvwafDZme=_RTjYHseW@6fheAk{XK2e5buJ^ljUe- z2_M&yYTMQnbXs^b6evj$O35_}&Eed%4+a!7Js3or59I!A{ECE+x4MmIvjX(=qTQ^< z&9%R0>0f%_@04W=z!dBlz@sTOS)St6qdi}4f?XG$NERTb7SdZ??M3{}!RaqG#DF-P z=y+mpz!S0rho41{LK7dOy2m+Lw(m|-cu8UOnXmAme8~=On@+NnQ8Idz+sIAYlyxsT=;%2r?zWU~4y>{bc?StGdO z@RWtEH%plLZlS1szJ}?Qk&((8EPd7uIi8$-_bF_qTcB5%cl`?&%Y@lxTE!FrQWv}kBI5+B%d01~7;`1U&ouS2 zd0URbh`NQ+#<~TurUx*P(uDoW7Ex1xD5!C`?hoyhD_`~;Vt%(h@J(bVZwygk;xrmI zsjTF7)$C>BdB|u)5Y@_`mM?pt@hPn4oujfRpCi({8Yu$PW5=@|_MH zrV!3shF2~`{>*zmfxapKr(9dU&Y|42zi_CBFc`As;Uuf0yw~94^T$l~$YDH#qLIgN z(;?u8!-#nf#^9PjBzbwgc@Ekz-d}b}w|T`WS?uexA|r~wb3@aM%!QV&7l0mK)ZVj1 zQ=6=pt}$oCnC+VBAPbS7`L^`iToqFG?$Tm?`R|J7sC?eHkm}2;H6$sqBbuw?6!~v4Wx1 z);?$Z%_9F7cu4W@X&Lr2?+M1EjYMzT8OHr#_z!Qa_U3|5r;&~SEF>zfF^CRb*4T!& z(pdTY5vB=NnXL&2c%k9eS%2{gQC}3Fgg7o)2D&Q?Zt%hI1tD+u7@HXDeXw4Mdvyh& zv(3Kgg>nvFc2GIu6WwN*b@cN+&RWg8ajcuz_XS31njB13qHuC$hkEmc|8%~|aK$f3 zeMi@GFW6~y)O*~mr*kOj5KICl`HBnE=Hqc{0AbCPWc<*8bsYjN$g=%IVXs z9H(0zrW;|v;q(>}14pE^PTfK2GTnyOm_l4X(+}(b{g+0>CeKoCQwhj|V88qY20(Z! z21CyS{AfQtXUHcEKX_jri8&^xegK%{E?_y%6C^wvi6L$#k{+hstuA0hV79AmFl z#dtCvWyX&gcdR0I7i!bm?68;mk+4rgKTzhUJ>a}^O`#~@7H2*nURWOMDvT92*xFi}@uK_2_71#f{o z+REZKK8(Le>S29LZ(cTzJx-=}ttukpdYr|a4`8#Q{ecO#4W{6tK&)aUb515{hzDH9 zDa;HzaaC&kLs*Fr&FaB0%Y5K`B@t9^!5Mi4w6Wz5wiU9U%-ox$ve~{40WM$Of2YbC z9TN4KkhwR8;oo0vO+L;+is?e_=)l|^XaE-L$S^CvkBM?21rK`Icq(Y)^H{$gc1><0 z*3~R)=7@FAePTmL!SXHSHW@X%@9+>M3I_Ru4Fpg&TqqWI_wFME{scDIcqJYKs>9-3 zE=DqwtS?HCB2oMHONETbFET4-=jd>V^kYUdps0TCRJ8qZaiKigxU#RO;+J#g(3;tY zK9Kl7l0+M~-6GfW&4~Tn6v~|)lSy3ATu~4<^~PSRN1At}^tHAm<@e#w>oX;sJOFEH$gbw!qlx*& zX^B6@#SvCM!B7SSf++69RRj~E@T)4}*Cg`*Qu_Hw8G6vFJf4f&*iK5lh@cR2@XSg} zeOFWwMbIOE)891s{KZoApDj=TGhU5jjF@1zRGj0N1oC+frM&vkX*H%mG7D4#Decho zME${WjK*UKGY&-hS4f}GW4F(-uSq^WwkTuNi?>20>e#X;6btwsnC5A>O(dm+S@)kj zhBUpoN;BfTtC}eQmbqf{Ni4jSRSlyyUdr#ICvxVo>62iY-U2x`kWaZ{o6-C3T}EGs z+54k!#JN+1zlmFK>&-Nw(bHww;`#&W@MvpPL*y^MTq;Cwdno%O#DNKsw)5kb5!Z+f z!zigIP-GqzX?n56uZFj^D8XR?I-1KXWBFIoF;lVq-cWbSAZTPy@c z*p6#+-eiX_d;W_b3t0m=EsPO7=X$SysuCqlWHgRflf(08v>SemI_Xt^#KF4r76&;qIRhAHR_@@AdjcrCG zVPbTn^mEmtkK<*#I>RF|C^~~C{>-Z9yW*L@JjYcu{VD=M#o9mAhc!-*1P9;5Y3bX! zYZ>L>K-$o>s8NCnW9jFbZ(R$zoY!bHUKVJ|IzGPz(onKdU$m(hAs&3g48@gMs(3bd zbm$^%k($=c&Ap<#XfD=ASa7hGy9K&qWfcrcbswD^hh*D!i`nl;b7R4Wq28O>GJh(B zSi-srHn5a^z`;EWid&Ds0n(b-5f$`#LN@9*&F^e+AA+eAzS}rIPSm*734wL&!lq@> z6qt!UXA360CIH*Gr5(X0jBk%6g7-rdzNMuir?)-RuV;)qv8#gmV7Buxd4L=76$2zUwIU$jHN*pDtIi|VQsFM5%M zTFy?bOadM>=Y*ayz2vi+)$6WMM*GOvNXKyA1s5NoNg9nvh%f}lX>5l&RFo_k--Nne zG)DjbasyHFCMG!5H?>u&qqs+quSmB&yEw>PQ5+;1hjA`tcx8K;kO0!hS5Zgok6hK3 z)Zb+cOT{O>sNA^cA#+r^G@tY)FKMLiUbj1t%-uE8zrr{Z-79Y?H-PH2I!p{O((p7U1 zXj5IHM*M;*!H>ThwN!218tlakDvx|&3<^|;K$(09f(>Pm-0}(kjsMm;ZCQAt`2Oyd zLHTsFlTf~;xAuz>B}Elm2#?{4v&aNAGtDg`weO(Y2CNGz78=uj5X-V-Z4F1$P>;ZA zDMg2-70%mL@)cr{mM!#R_CpClUE@%cReP6yBga_(H>RYHkN}w$-Mh7VF0i8)0ZFRT z>A`vZt8eO*++e-uk4e-;eZ?pm?M6^xO3Vr^Ih>t!!&wPjT?A>e_(mB^&ycjUi~)`!KR*ztGW(+PbJ zO8sbx$5E)wZ1|g~1x}}sD=sXLQ3wRbA zTZrKbRB=beD4KJRiQ@8{kH%g@wj;xV^uBMyKh=#|x(G>DSB7JkWV=jd&k1U!l|sGm z2OM!yj{J}_jazIaB4iuN_wfN#zt`;_uL88#LgcqmFAwPXhHI2!!JT2X_=P=XBF-{1 zb+5X zrZ{LEHdP&(K?6t5ck=?1LFCEqAsT)6@)2z4Vo|I2cP@jlB1CpTEHcXb5f3afZy&uW zMqL-`E6TPt$GrWE31zMXTMA3vHn54iOL7u8bua9`>eGIRY3qGZEz4)2H`gP;_aV}T zh$!tl%N7L0BhF+|>kiD-HyZ9f6i)=br>(n$gN4N0woUVpN50;+V{o^8_Q96%`sRt1 zFB;+BpA}jrrvr?UaS&nJv1U4&Xje<`&y^9_0pfMUeW1NiXU@~ke-z$G@&L%%+ci53 zt1yi9^Ih2Rf=w!L-#t5wmp@A^G}_mjXrGS0zDi0m-W5Bb=L`VXwJp=mN54$^sQqU} zt@e^ZSHCyJSsOt7u^0tk|CW;LLyeZ4Ts$ACEdMs2U+h>1@x?9vVV6yW1?5`yru;LD z<|kv2&wZrYFiKl1r*H=#ydArb={bGSe**57>&p;%b!zJ_M;~WH2;oEqDRB-B+x0$N zeWU(|oYDfXw^BNK&>Mm5+=n<-%(}0|6)Nme zm3qPqBi4PX_u33B$O0=meH5SQjO2rqEU?Mb8)n>wW@dTzF)2}rZSOg z8NR}R`#21Gu@cJbHNr!H21OyT({q(b0;$JoZ}ly~lHR%=hp!_1b>hZa)0(y*R!yy|D2ReV7d!Xl*@SZB);-q43)Q8cR z(FDazwDoLzRYzp+8+h98VhE6$ltO4y&cH=YVo@e zu`pu{Q>fyzgw)r&jKkchQjXREXv7F_OzYn~v)1r%eO4iuEJ>>uE%Awpm_eN=;2BpK z>dT3f*?XIG1e@-!@9FNMooK8aKo)BW)98|rps{L=f(&E5E=%v{-xUTp)uVHMOt7DY zZEL?>7(Y?j7!MZY*V|!iD`h1VcTeKa5t{ir)K@vo?=790z`9e9h#%8Ip#GuXCfk?r z35Xo~G$iw))P;5aV@|z7oAkXVVAK_z7-)rtJOu!+xg?q+>K?uIo!rs_SttWs{u~*9o(+ap_Q-uI3w2;=w5<)UPr0 znR`=@W(WN+;y-hQ2H>y$#zpX47R7(k4jBKbSM6Rh7WFbBTwiyGBuE>FNDXL>P;g{z zlVNg+o%tf80o9<2Et475_}9rAxz4(V*r1hB=xNhMh@2{riR;ruSGD#aea}~A9Mz#| zbAOd-kDkk`W}v&I=;Gvq2XSn*4n0ZRd!hPaw6UKnLel1Nia(K#&9dhv-`sL&5Qoo;OK&?j8kiG%buLNcJ2wmTUefZ-%0pcrkvU2ao$MBM3AhrS%op_@mS5GvFa{z zVPa5%M~w>0%X*%BMIs5b$W}#g897V_#k$i^wE<0(5};{UxW8`A-%tnzT|~rtSin0^ z0s^Mg&@&O%gp7}=DH_z*%99xMDj&Ck^)0Eciz?r!rHHs&+1aQB{_Epljkd3`TkVvM zt?NsAbCjBtjOaC*dsS2jOg|X42kPI-R(!%Y$&wbjvswVPJCRG8{~fP1;Vn5YRazaU z9-e@B=p7(A*`_+OUkZ;bLb7mj$L?9;!6ddH?>LaG#mRp^7=CTa7^`#9i$K%$vcS+6 zHVO9}DGsX#4y52c&^w@Gmv4k6)rJjVh2&GE8eXBJf$gln(RZY3dxRsr==izlQO>aS z{~_%y1L6vrMorw^-QC?Kg9Wz`oIrr!?k)qtJ-9<4xVyWA;O-jS-C+mv?svbv_x`xM z`)9h(>F#rw?xwq{rn(-`iA-{neAv^w3JT%BF$y(|@E2NA1JE0wgAhHXN3yV5UtlV( zSAO1p*9tfP`gn6vN!Hh_#oisN%2o4wDh8{4{)&bf>D4i!x$w=HkF zzLV6rY`wH#-K4w}1F>HPY@=ZUiomK3XsyLJ+m>VegCOkIs3f-sUu&SAvE31)M@uA- zW)+(V5i+AChK8cusbwEtTMC+5it@tYRR>ILVzP;aZ%*(6!-lRKjn=~)K_7@9;0mT@ zv;^W?`ecdaJ9@viD;6Gli!m@e#zWw%A@L zo?{Z(IHS-^Or~yY-E)fR8?o@%cu3wxZIv11korASDItFzElF+(a-xmNzlhet{(N>J z;+exbs6A--nY6SAzr`)zG@?HSp!Ks;q%!rDPu-RLi__HmMMXZ`v~A& z6|rRAjJlRC*seL6wv?oEk^UVY1X{zj@8QcX819OQs!8>N?YeVPLnn?H)QUARYM(9( zfyt~6lNdDP^7FG#M%xOL)vBnO3fM=Qd$1mAabvZ(;q*&m4LIjYCDq5#y) zfts`TqpidQIsWS3NuJasSCP%2#hLz6^8KAXmLZbm{ljZJXrx@1AU<(=?3Fm#r_EKZ zit>d`M8-{u?9U^dO;u4@4b492-z4U)i{&@Q8RI;eR8H9afY~c=3ATR+)_f-! zSijFfs6}sVNCFN?NPvFcGYJaCG9I!P$t*25o!vfu1Y)>MBHV(+z7sVP;#vl(ji$XC8G)mjidCG1c2MlgOJHcF|kX= z*A1l?nR}+MNL6y$VYOtf9UTB2Ll)!D>fMO&VPBn30Ahde2Qam{-WNpyt?WM5HPeUW zZ~Tv@louU`8u$iMK+ck0#q z_4s-&>2$|7K;^gZIqBus!-tV41w;M{9lZwrH;|z2`^Oti@IC0LUPH5uMCD|oi+#?{ zj-4I@Wn&lBva`=&+a0jDhN?jRzXTzFs%8@TRs4Zd2J;SBwYnI#1YtlNE_?nU;H>5w z@sI3BuV+*)*_+?+%b=3i3>p>}kbm%0P3{rjUR&_8M{=nI_YXYp=UJYc)E}6frM9};-y|gfr0F-VjW#vj{tiK1d z3)nz|vng$)ba`y?|Bq}z0g|V78u-`i6;#sI*YAmBPGDkq$#{!0Zfe5u_mECcD-R6e zx!bb^OA=dE&p+z*x1%fWcLy*z^qO7*KOF~2VGNRG^oU9V#@AmdEFd84Q&g3zWwsSR zS>$t_lLa-7$gC!M-1QR`Ov4n?BlS8zY%iiW45SxKCmvoPh3TJkyg0xf%;_A+VCbC$ zQkekQXc>?{CElH*^D-dQ0iLdsDHEp3Xv;t97ICT0<^7<|F_#hwp*)#p-3=oE8Z7DT zOytk!%a$;6`v>uV-j8T$<4>C}Dwo*~*2wSL;d~ylWzl)P_ycmr_kP-Vu2_418pG_@ zmpOR7)_G~La%VUL0RsnI*Vd%HP>0?~g^h&z|BY0r`>L_#d-rr$fLL}v@M?e0>2|ku zc+2@X(((GVyWRKl@e^E=u{Y}?5+6u8qZ@1g~f>VueYUynPn~@xhU_REf&j)L(k7;Kw z6{jy=TQ%3*f1eMg#^t#DQ`m_V&=6T!E`nu10>ho}QEBeasm04h#s3|fC9fN3{=UsMX=$11=+M$Sr>hN>8 zD9oxp99Enh!#$y>H3^l2GLE@A@B&~Qj$8fQGIInWCW4I6p9n?!ulv&M0p)cn>iYJ3~Zje&GiSCS4xP{rHd-P$%b88|#vNP<;t`3{f| zb*6raV>rt{^f+&7O3-`z5E+dDrQ#E#=J1(sIq~t*;=u+j_BE0GWppY&2$bmEh5h6W zrAI%WUD=i3VJoYE)QWbV)<;pX>pq5F9#{#CZ%Zf}O622*1%Wo-BPYiTzlT@;_72yP zT9^Wb>wc%Q{mcHZwLny^7FA9V7O(mZi+6PI{b79g|2N%`wz-9*-2v@9AY_0AU*d^u z6Bb=GtE;Lg51{tR?ubBHj>lQ;`^CikIR2S4^Atx~Ok;&ievFUMbSM zl4_pGHW}65@703tk-oLelJ9jSq1=2<0Gkv2MM|++$%tJUON9AS(GK=`Ktjmwun-Gt zH$g&3)1FXgPHAXvp7H)_@ZP~Zb;ry0&8rXAy>&Q`_0BUU*|MwC4nMy^o5njJm;ulO z>sYExKtf2uV|FKO?$#0!ESSF$`FT3DJbD0I7G$Y-99b`i#KN)~^<+PElSBGP?~C$V zE~e@dZ7Mb>U9CYwgE^kT2Fpwy&5F{VhWIl8sR`I!L+v5XS=13+(&A4;h@?n!JODE$ z41596>GnD{j`Ix$b*WfR9S2J25o9v1fGSw;xlZo7UwdXRfk6NIup#BT!F^4(gu@o>HASmScka`<@o3I~Rx%wCl*`J8R8u^sUdt z>c9B=r)uG@cg-qzwB9Q9bb=s?(Eg?-^LK4=G>|hE1SZe+je@#U;oabNQnkX%-xDCQ z4!0X&vfVGNcE^$cj(0V7)@PQGb1)q@K@GT@QcGiZ+&y3*m{0}&0oaEK{hx2)=(pzH zSweT*?@&me$p1p8yk{+IzpA6r_G}{8OwXA0Fug^#e@{SwoC**ir-SFN?$f0u%TNWp z-317k*;7H^I0;CjZqdCRtI%>4#G|cYD{uByxKI?}MZ50(Xrram794XVp3QFEA%wW4 zgv~-tg}B&U;9LnE{h14q5~|b}AH$02u0;FSSB7df+f;qX7b#<7?D2cd1u3!!(+@gVLM|i7E@et!LYyr-(b}LuraiUAFWf$i%loi1 zTv;`iz@~bK+V==sNm~Sb{i>fKf%q@ACw1#p1dTj)%$K^n^L0`64PockKFiA?)i_7m z-#ba(<`G|uI%Va@FOfBt+z_6BXZ~~fzCetY&%hnUY8V}*+NU?-t)w;+82#(dc~F1u z%?9WZE}^w+2)uKE0@Ip}^kY|D)#UjVD=BdFBnSmr2IRmEk5AxhPh)@06)9zCUvCSJwF z=qB~muw0kI|4R$mt9{R0eC#w=gsSsZ>Z=m%1JO7-O=6mzrB3L!a47e9e-K329y3Om zFmn}=kn#jngwY}bI#HkF6q_PDjg?(-UmW|g#p^)HI#}3WYX) z+3HHX$)eeH`oSM3(UvT|BHSbYE7X!b9Jp!*gR8F~mfnP!V2cQ2+WP1b z?c*VF6L)Ae{Ko4asMAY=72E#W#7*>Z7JrdO24w;oqnD41l?uB@4|5x}3YH0H{S9CB zHBVSMLd?#Fhc|NzOgAy<#vo^*g57EP=fW)$r8}05ulaYSW;I1$pPN)KREp1m>iDM1 z(d@;sKy}sCwchga0|$(@LrsU2E6;6%{O2xmp~#$UTP{>KItcYy=q)bZPFX+or*n+I zpzw+0@&O5TiX-Rl{-4!7&2R>xb7K3rcg2gT5z3;*6wjzrzx@})52*;NZv61vYV-RS zsJsjPc;0z(Rju2Q6#OOoY-+kcx|;Q&KH(yjQ8+(W{fA(a0CsK8#i}%5VY=C2NoM)} zVS>AD`C;kBjh{w?rz+bGO-$@3_A432Rr;%jNLj8Y2Qf<2eRp7tN36;Hua0ip065Og z!&~&&eKFgp&kt&SMbpa4LS*5DP5P;0;y6k(TK)-QlDi$tU4wj?LOTBEl@}j6bq6u( zV)_FACXz3-wIFviTzxFyGiH7}b_st3T=)$OoEYjGC>e)yX+`i|!&hH%=)~8g{!=9@ z#}ApWSUCiv3C)u6S}O1*^sLa|CtM8Uhx#ff6n@0<7PhBQdy^)$WtHsXc+%zkxoi*D z&y})@`-cI!OYiqqT zs^!7$Qk0#ACAvfmm1$!~T{rx!t}~Nk%7vM%>j+(KBWAF01@+ zKnL9ho_=(FtYJskKfD_gPbOS2?HT!vyK0&bcpgM z@8xO=(v%@K<5W4FJ55}juA^F|aAl&Oi~C=ykeX-cxIKzc^U)3$455jVEE~d)!u!}F zyU>2@HM9L?10Hht#))D)vlAetIU8T2aRwtn!5?i`K5LbDak`2FS(19{2=GUjHtwDV zZ(PPaK=BxCaXM5O=stN7)Fac6`dj~GTSvCFO?Nqb=Hn`nNq#3&zx><3JN|L;SpV&1 zh;Mt@JDEC<@4Z*^`0^)u`{*g%6pauHv2609- zegq*fIKZCp4caE9?!jiWD^=g%ISab(tZ(qM4zNG8eS8DNS+m&$*oU){L@eQMAhoCe zBgfIQ+YlK>q@l_GbXp9>=j9g`n|loJFAiHM>Gv6kNK<$zb@fAy*yX=G=w1H`^=Ny+#cx{Dy7 zM3u4P39%3G5Z8-Qav7Jb&z}rzv{xP@XPl%P(EBJTkeuE(f_8R)%K&}Po}~b+K_3#h zce%m+ohJQ%BsfIqZ>dQN#OqU5^#2dp0tCESpXzP{hwUn5*R8mJ(14gSmv`n?-+YBBw-o1ckq?Lm_@9e4%`P)=CGW2k2Is8}?uivmT6M z%il`VPYq+5i3kGWX$Cz%klyt9jyNw(pv3<7uHta2#VVLZa&%K3b;m}x!oQHzdX&%5b%)?vcBL$E-6eyF1aqhz{xWQHWPF>q0E z{nEKca*E<<51~Ce>}XkJ+QMk+{ELv@ zK4=mPfR)5=3GRH~tnyozFC%|>VUjHYYcZJWRc8WG#{^l~Rw!o$7_Zz;KhV!`m>cb| z9V>ki)ZTR>b&SCrG=({vyv)NK*Ogrf#)Qk1te1w9aafl|g#5zmMy^v43L(LPFpl(h zO4}Crk>LQ!t8x^;R@0svi#yI}he$&ITOyvgapj&2P7Jb-EopI5`W|nX; z5dcVF6zs*9zKB>D&6|=A0I-;Dl2=4xdJ&vnY?o!wyU#2c8#alkIV=ARzD& zp<(q3);GSIA%Kjw=keK{1C$flo!VD#oI60*QSOOu*EWs)0t>Ixw-KN?o^G__aq{wZ z8nS}k$dFJ*ut+aDUE(R6?pHN182OMpJw70{(lC^*p!VFTn_<{JfoUfhz1(^{pC65# ziYeGKY^A+-;)*83KrXt!GY1J;;30482R9j4kRqeiLY56q)!J9|Lwh!7;BP zFp?7y$b%Ec2W>wmZSh-QArs4?g%6P8@{3J^11*GMp)5``Pt=%7RV?J=b zBdBU{J!c|U!P`eLX@&w zhWO_7W&9mHZ9uYW@;gg?(e?Wp@#(ua$|-3J&MDLa!LKwJ2*%65;#(pl269szJGfZ9 zt_#FK{X0BMmY16}(Wm{}wO+k|1jWoD?q!eN3b&e!{w9}@*5ySd7bsh34Aph({F#aP z7f|pIJ-c!J^}8(wN)m-WW9e7vedHItCka$?&niDh!eGZ1sp0~e8tS+QdZgVy*K$a> zhWPZQD+)+GO69|e)Rlh{Mn#$((ns^)jx5RriflsJrf~`<)TAJ`f_$+Kv4b_xgi`u& zf<)okO^LtUqvZ%~xL=nl4CqAF zJhrNx>%z!$#fo;2bL~vJho5wl0-BJiQ}+MBh?e6dXISak%7=Day}>xyBNyH&RSY%n zlqx3^PEbsSFHOxh{J*%gOZ8fEZhUGB{UOI8d+#tf@#g zJC4st{_nvGz}SpH*tWWQycZ4}N*zQSEp|}cg8q6KnWxbM9E~z5ztbEqHK6AeNR=`- z56L1F7X~5SuF~^FOz?IlG=9eHCKvj*K3*scI;QQ4xTl$M2;PLIX(~P!jioV&UcK|N zi0|KbeiW~F^?yM4zXX6NkM}#U?{}(<-u1bEFt^p-mLe3(2yDD1TWftj_0Fu4lQ4>| zH(=HwD{FlivB@Z?FVK@xYCzw(*L>%V=!J;&2JCw83LK_)#+QBOyZp6m`d%3}wFU$$ zrI5R@%QgZX8|YDhibm#*9oBoa^CYmLVCoG@qsuRsY@gnsi6(`P7`yK7F-g;uO02?a zb*K;B99`x)3(4_zEWVC@Z}kc{_Ec6%#qH znM=;+6UvBjJfleg709~TKL6;%6UH@F~6e)u}|Kmy8k3@f8GFs@6Y>}rCjBB zXA(BN$I#dWwE;bk$V-W+m1%puDa8IPtm|`p3nj{z-x~W)LV7SDe?F>JZrIa~(_Lnld($lJRbjF#LIYZAAbcNeO8@2aju}Nug z_5D=XL0D5tX~dX9YUerN-M*Z&P?XvPralSd{Ud6t>NR>~rAi-Qh7ot{yUHu(Ypb{= zOk5`To9OvxK9{wZ1-soGyD%$)*BwmM{JsaID$@ZsgYqCR{l8~(v2mA_)Utar4 zN5r=8cgvy9R7f=bXAvrYiGlSU=ELfWl**qLPiLPhT=s#{tq{cpNfr1-d@@goNAW7g2G|kl4^a{`mO^Z2>!P$86}uGgT-o^e{dP`v0nXSoy_LF zHN%7ufUbi$#Z)*{wt|40bj^cU4gGllqkyc&;wG*TS<7cm&|1YJ?XY#rJ- zs51(V!St{;0||Dm2s~qz->a|Vim?E?k55j}#vy4QbT>erpHS--$bZTQV-`Q0dyDtD ziKc)fCjbJ}FWjDsV;fygf8~`tBQCnhbsB`FtE4h+xCPc3fTOc9Eyv|GMG<9W-4xtQ z#DV>ke+mT+@ilbws|1D_&T}1(l}y+wU8+zP9`j= zPi2CI*Dr+8?IzbsmT`0UTPV)!7rS~m?)0oX)laRjJ6fp{iG$3ncyXy=nYHYZh@+rc ziU=%J6<5{J9simVUD?ho6N;>PJ5-b88uOTG!)bg21v#zI;Gd^Tf`XkpO!GjX@=1@5 z&{hmu%j^@hOntIdK({fW9bph>&Q`RO7&HR}LHPIt!|!xD1}SJ8K6WPdM;>z{2<#`& zM8$^bPefus8j!TUn?}XXMbKZaNf*M!ZiAUk90F0BfPtsxl`rtfgkznHva{4-oCLS6i=v#3?N$4Es zh#a{|12$A_E3XyTJZ3J|2}Q!bD69uGsd4H{>knU<0Ha{gDtemZu=M<K-XU_M0Ue{gzFB!Xz;KpkArq5X&%$~4_rgV{$6v8P z3&#ZyxyJQ4N5!27g8;6{+3(QBT=ap60yAH)dZEU3acA6g38!XE>OEbo4IF0RcP;tn zTdZ!aF=XdY5d^f|3qMdDd1`lsEMSh-3aFC}7_~Kuu#!k7NxAls!!9HfG8zo?_91*@ zCm~!Fsqin!kFAfw4wQA)c~~Hh4S`zL>|#@4V&&ucH6=~4e|sB<*iFU7kZ9ZvN4vm; zaJ$_eGN~r-%4|Tz(>WlxWDwvKxn7y$5;vmk(U|fb%G<~kT4{fx&~UiHARt!KDkh>_ zbVRBl)5It!{z96Al#`9{zVK-oXlq13cmulfjDWV<6CCM(B^XE8*tSy66s%t3{Si+m zyu~{YgWqrJ4#S~lS%>Z@e}i}{fuGI4U^=cZ6a#M54vnv7o*TIbsJHSqp4TlINe=9e z$ol)+vd$l#@Zr+}RfKxlYKM9nl!{o;XrC!bAhJYkd*6n#DW<0WnT%ECZTuu2{wSU3 z91a21Zm*<2rC$&7$%<(lniW|HljxdjWA+HIUjTVE-QV9-ZTGy%cJF1Nc zhAT)yww*LpS!E=mhy_0~c(ez^KZXH*fiNj+>=Fp`*>&^`muL|js0*SzgSi|^8dl0V zrqCbOChX=OdSQgOVBkuS)sE6Fm-ZGC%nHkSr1`!m(yQGEw0GLo_yP!R5%4u*l|Fal zcopkt5H!*>D}Ps~5dA9^nipyx{I;o+xrqMl$JTQ`;*ZxV=)P(lr#(8%6t_WJWjU^? zzQm|a$&$;h%O)G%NL45NMG;%LUj5aev_S&Vrr|{`1#phMr?x*k0TG*T2AGsU?_vfG zoJYQVoDn7EpP!#>d^IYwXLdMk%$!Rjwnk4oJ#jcbd>`+;$&lqxI-&4tRTg|1h9k-o zq%ZbvijLqC4jT_;W-)ubJ|lw<0Scf_+@#N*>OuFyT~!|l1s+bqaRtoZdrsp;;BItP z#qKLDb_k#bOv;OEIA>bsbcHVSxBG?LGC7xkrDuPfu;86{X8BDn*cXhPFmF(5B(MzS zMDZ9htCLE3`9#M_Hrzl4sj}^b;cQV=0*#{c!$>;wpzmr*NcjVd=QI5^Eof=El&SPI zj!}(=2yTB6+CZw>7nnjz(a;kQD(vk%uW?H2gaA?02^eZ*f(juqj3ibZ8kT$pMyXD^ zA^N=#namEF2jCZYnnjq#%PwWaH=76ZlSXLSEqpl%d<`e1g5+#p@2T23wq4d66y_%F z95Bi(KUo<5i@2}5g=8iLv2<#=5Yl{b{8E|;4)cof z2P)oCJdN394cWk1Nj1Fklf#dQmFx_D@R}DGO~mTi6P#)^c1Dx^P8gxbnNra*_ziZ< z3Mqe^j@zrP`+WywOg}e{3w@C_M3-;GZ_9epxN%(N&+`rze+mbLp*AerX%8L$VfA_3 zUGs|7xZ@f+vyU>wH8}4na6P@*9);~5>DEkr<`k)Io;vY*%En;fG}&7iP%{6T4o*}1 zgET9s^oOgE;!q1GWzY$|>nz3DInQ5quu*63?fsvq)Lw0u3WguEK)-!1!o%t(8`S$+ zFi&=8M4=xVQg4%Ey6){1RgS?$_iyI15})9EKG}5)?!JamWIO{=j-Lm7Tc7=|Oo5+7 zC`lcBfm6JoKy8~PqPQpRV#3PLU?*19ypv2ad%uw#ZOwm+{+w1ky{i>O6is$9?JC%5 zu6fF+TOTE0DQSn*o^#Ky$W%H_aI(SQ#q}sXTl-nDr^Z1+n}DCSI4%6m=_-7SMUj&C()5k>WgARh3Ij%k8p9wBvE(^tb*ItL>_HGiB(md&aTE0m{7&|jR_wac^Bg^qpS zj^~S;u{mp><3bK!%U5|NorWLxzTdZNt(}*4#_OA{9cSwW>Y8HW))O!5XROm`H7q;J zZFE?c)ElhJF-ovwv6=+x_o0bVNnCCKu>mNjNL;MA6c|=I`j44bLx{5DgKI+j(D|FWtBA7Hv1znz*<1)s z?iaE3N#`)EW=WJv%-hh|U>II;UzEXbtQ9=~{oev`r-9^21+D8b&z5IEZPmX;`K{|@ zmFlO^u;{aPQPfw!PTCbR?@HrF#k2b<6H+sU&n=z!;c4}XeE*Whj(=D0iH**3t-!{e z36ssDz{btL+5;B($cU-Lct~4pl|+2=jiz7Nhm93<45Q; zJT{iwQ~ptYBu$5e!Z2A3Z!FfAO5T&q05~`|G|Z-SsKx z6r5pnG@2j&x4K8OQ{dgKCz82>+MCN8scz@thqIzDcQ-v6r0t%MQyQnQ-e)Tf9%H^w zaWAQXX11dDEvUBb_?A|)OJ82SpGqRHe=RA!9!@PO&AkfLLwntfv?z&sx8fkY+_BE- z2Q-VeKYO?O-jCht2b0!|UWnd>zqY(!i$0$BxSoo7-JbgfXZhR|_mXT5G{J3kyv zDcOqZ`rh3(*uJ_?trM0;V7Cl3)QGku)gJlE#@C3n^eoM-`QqN)&-y4 zoZ7bgT!F1PxVrP=_}GM@G-UCA&RTMHd)@!}Tz|WYQ;^I5R3#dfzq981k~C&3 z`g{ZW_8s?kOAVm!wXwu2>h<*eSg}Iu;bu`K%ztx?w`C(#1Zc*HS{E}K_+g=z6M-^5 z>Y%%Yf?InZJV7v#z)|E+#<6r>gH!})M`;HVC8^OY{Vas7=e&hvFX|evzz40;{l=or zk2&5rgZ#80|Mg}66Pl&SNdOC@X@~8{<~UzuD&T~C%TkOn1Ai8OlY{(kuxEC^0yC~-fi*=v&ERM&4mZo94Z!#5w-F6#YI~@t-ldy&w;YOrHBOX zJ8L0hoPJfBeFVidNy3ZCWwaMFglvWT%QMfGC6uCeU}xUb1Pk%8Ge}}EI}sUwXh+9E zCJ09yhJ5>h91^f+$1Iz{M2|^*@E0gWsVpVil&3LaR_j}i?I7bnTvuWXn84a;OeV+r zyy=baEOvPpV~nXe?&LBEmOu@YpR$S_4DK^N>P~W8;h)cz%4yDvf^9V1&8O?-hn3$X4HiI7u$Es4|O}!Qe2RSd|m)0=I)Nv@megU6!g@ zUugn2fY&X81K;oSP4!Y_t6ewrVgcFR-=!_Kab|946n*H*Kzsf=5bHa-72ynh=Ju!- z&Ht>v=UTB24BlL<1S6(nfAf6Y6uTqQhCBIfHQcR+&Hl@@nBlAbPfMyLGRQPy_Hk{1 z>*&lz4EaQS-lOV4u!79D%j-cht=RM)5RlJLAjKv_8ly9OFebOtDJ>4`M_&lQFkb!d zOVJO={_*@+a+`x3y?k{WYA_8I85hZ*X&8$+3SC)IK8EvgWqd9(>gc%&4`CJBp`+D! zr@A*u4jyml$m##7!Mm3dW56BkV9G$gIsou?_hg6-*iZ3G25)UoY^Hj1vrVl|_f8-QNd8LMCT zzP8Zh!Ef}`&tLCbbSIcD0|#g zQqegJhlJv$Q8xm5{$D8YtuVOkRNUCY=0*g#!zbL0P%8OBdF74SJshN8y>;R+2rEw# zOj%FKZiQD&29Mxj3e#*LW%7lH7$i~@uDEl5H7oP11|n3C|3ZkfVqbvy{+nl=2gy`y zfOJ0)K@ExvC+FwqIDQqi2!v%jL(MN0a}DhN2rnQr0Rp6v91M2QRxXd%K`?^K_X^um zwCP&-kKqm)i)-$E3&Xab@@J4_dWjVK^q{wM{RvBK3k}kVF#0_Z8Sqp~`Q^xY%p@06 zNoDBhwqt5XcVmutsq)pS=K5>HT2{SLWXy<-_Tsqk{tT54P@J0;M5^1ndc#rf#ohB{ zgK_3Do!V9wvM}uh#HRA`xtQuH@cs$&`VgjNb~u*dBuAK zG*b0+J2gSGM_eoPQOt2$H$@{HYkrnly7OBo2K>c7B?nhRq8&IJvvlR-6*^+~uWxqb zR_ixJB_~5psT7?j{1O?JW3FE`)-MHPEIdDj`t`6E(ZDGP+>%8K>W=T|h{246kCSof zi5ipXxE44Z&Tdud`QGqxm+VYX$e60z84hZZH^(f4I)k8bJfD2`g&fk}<|_9T1gU=*D!y_J+o|?K*2quz{B4Ai%=L#F3^fih-lOuxO*6|1$dPX2hK=NBcPXm?6m`9-_dcfsrv zbbS>+rb^4Xvg1UAf9M}Y$ejGC(wA)81w$D=Plm7On;K~&JkpOuza48VVq(B^p>p2@ zz1Gjgb0lhFaD#_9Kkk|u`O2M*-Th}FTgcV8!;lQSaMpryC;15X zB$VTWWA#{ehZ`9rNL}J6oRlD(2(Vgs7_4kkPh*GwjxYNA$D9QuCeKwtwG3VvPn&NU z4}5MQQbzcg5FB9$g?D&Q&L~Y~`}&W0gH!r^c8+uvpK@j{>Cw<|h>u{wgv%{sROIs? z(6oHoPwfC{wqUU0#wvq97Bn54?Z|YJ2%u<{!Ghy7m&pEnLL&SkQL4~5sbHBGk1;Lg z%_(voK8mvj7HI>rV`STE$3y%PgPI78^tGYyY`XejSSJ(T|L#PFOh4 zKUY6d!#JU=XB}hLnyy-S0qgC*Ve$&(S;gY0q{u1`WWG~tDCe?MaLPd_lkgdg#E`+< zLIEw2Ujav;E5_!7c}Kb$#qay>IZE?$!okaAt$K^5-*&0ZI7O2rECGPz9_}(8c}WVD zu5Ue{f1gmCWt-X;I3p-s0O5@50o!2W$D=9|pIBWchT^DBjoZ-S~!6&cDbxDw>0k zR6_l?O!s;Gp=EbbxztR4AgRDo^@lWX+cZ2!WymalhmN?%(Hx(4j>fFEcJkKD*4h0L z)ax88$C{>OIQ?S6_0pRciIlYZaw+5WZRohuHmy8=<1dx=+^_jw7&DgdkuhII4tb_$ z@oTAqe;B&a&svL1rVs@TAwzla3_u_DAq3cH|LuY9a;h?#sbd!&C37KVfrFurnrYt< z%S#x~A~fR951yv#8C3xcD^jWyN=6&4zX=$8X0a?{7;HuTeini|umUTjP5v zlI=J>MDHt(c3T~#A5ntua09SNZJIV} zpa!OWui|<%^)hoVSh>FgQ0tlukJm(Jf$-Xp@@BN6BFI{_r*7%-uBK8*E1ynECD*sC z2aUke4=skKK9b7bdF-b3Mu}WU@7H0Xw6k?ty-sWAeHSFAob8hx5~n#CANlk(3!Nmg zPf}lCne*mu!zXs^6%d+=(kL(BV zL3a!Zq{qjdUz)udBUMn{f(q( z4Ifd2$#PqM9%^XG^*xm46!x*C_u|gR&6nrJ=f&%(0^6h9te2bp4DQGKWAw)nTi;#% z?snfx(KYYa{l^W{o0pMfV2%MzqpH}C2|{mM(#^+UA8cQXBksGionT*|+n%)_9rOCD zt&dJ0F~%+hmLRJd2yjl}ZgP=+Grx4~ZnV{a4t@DNZXah=EIlo%e0!OFNy)6Yp}l-X z_+!3wP(PCMrn;9}J6kxCQ+F}%T9+*GD3#Cp>8;f3)|c5V=~AmRQqVhqik}=->IIsb zu!tI#Ys*G$MfRb$<8kwDaQFmJ=MWdyPswv94xlUQ58Br4Sk2FXfl=^#1gvyaoXB64 zRilI!;&J1%V>{d@5oP-%+i>G$DR`AU0OnW@mwzd(S+|hCjI(^q)E+`)0HeC2YRiT= zT1vkHM*M+cErb?lQF8Ahf1F!~uFD(zB3%Ad=gI_E%~KJQ)18%UHiG z7!esyEq3we!~tp1c7Fd7-+zhETe4E$Xztv7KB5UythpU#hiD@43*Zwe|_ zUmw?7Y_Bt4@2H=-;U`voE?h}hy*Ov*OB>o=9`7^=4;31+UgioKvdUkbtq|{y(p)Gy2?Y1)r|^-kof{qKG~pDZM^y zlZw9f)E!(c6lAUXG`^gj+OEA^@_2D*;xoEL+nV~iu$nKtVkOkfmnrmgzv=Bs>jt?8 zFaPIi`?auiw~hBArZ>9kKB3H$>8qP1>c;WfnwPM8$lqsDk@nZSofXa7oH0^EQ^z9Lf;--BqQoHxwF_T`dN$MZ?glC8+g&2hA?@7>J^Dyi@5i{-65Nv4+Z z$0#VWc>%rv`v)OCSk{|dxt#0r2YY2q*_<<)ea0!bH%W8s1-wcX$qEl)xxA;Y?)UIHaq&5xyf`yE6$$Jm^xzJPVq*5YbqT@TSNhtm2S0MVC-?f3^y844j}R+q z=r06`E)vNuy}K*UtmS*AF4&m;SQUn{6}(t`LTUvc1*i3ntfZD5M}32rILV3*T1cRi zmrCGgr;k%gi)hy0OhU!~o8*0V6{q^WS&9DFBz;9dUxBaWboa))zE}uL#y6tMGk!(xbEG7Fk+*YNd{^j~}1iI-i5$n3T+}t9#uqsmsbWOs2SU{R>sb;=t0#c*)zCL@p zX1;tPvEdnA-S+Z6Kf9KQUA^V&Pw|e#EmbTmEVP}EjxP4`aMh-y1v?mM!P*m-L& zvwP@POM9<57b3H5eA|>n>eh8g!0Gyj^Y*U?-T{b<9^nC$eT?ZGER@F$Wp%Uic0j3%slHEmgi$LI~CF8b+pqCG;yoYhp1MS+?s1H1Q{86zpd z4*r5R{H`_M+&cOXbw)|4)5P8DlVr#6FUZI7 z-%eHWLlG}!IXm~qXlk=?Ja>lEyZumL|LkqRohLVN3+fX!VHVYBiQm_z0|{{~2m7f5 ztd;k~kr!eY)W7W3$%q=ye?Qrw0GZ z3bf>`y{9V*3_Epk!={{j3m?z!Iv0>UcmmpW>+h{#gwp`HcJ^-tZ}&1svMUZZMoq@w zTY<{-e-~toWar!prU6i63qYS^iAA_(DFUF()iS7wNB!KT6{z)jy=WT1*Zuto?+pZa=G=8!zf z{Lu^gVOo77BX!6752x5>5ibmZ3nUX3da9$42?{;R3K!Ms@q{~1h6QZ97om9Bf`h$R zg*B0iB)ecb-}$lUoHQcUG`}itIYFSwUd1`H{R*=QWBg(5oO-t&$WZ`V3@!x{_-Xa# ziiY_+)_yk;Z&Vl}52g1!J8QIq)&XKpiN=P+B1(@h-s&w;86?M=NmVrr^}AHdC=E#j zX04=@%}E@)1_w9t>R5g6#$^?(v?`GCS32+5vuk4ykKuVoL$*duiV-eMM;yJv{wac76M_2`a4t3&c9QQd!H544xU z-lFh(De^kIEK4tLpd+>;f|Ts=JL&s23e$QBlX{O$SvIl5TLb~or4F8TYMbiX0tXjA zalYt*_aQqKHsphrh6Elj=r^=Eap1LX?f9VqYA9b-pG8c}?(#1+cuD$%^N#`mU_@*B zZh=1|w%`*jl{9=HjR?E7F$PAf5-}>qk&KoyCV7IsFclY{T3`pvwpKyI?9cqtCBnJ( zE->bV2QSA%DzhWUT$1gMEkE$p60A7 z)(wVcn*LmMh?<{-Cgw~6I^Ub!-!0jXZ4HBVTy^X2I+>?4uodSNjTX)tN1ou@Aza)- zhpeJyI!BB|{`>P4-=B|%P-|o#DwJ_yUmG$XV@q4S*)Y_#L{~n8`5E3yam>f66L`#Ka7Y zAuxbS#ON4>YPwe?%&bH-gZ~$2Zy6Q!_l1vAGsG}-=Kw=@NXHBv(x7ySfHVpcl0zd% zw;)~84bmVWt%QU~DUFnXg5dp(-{1YOyY78=-^`0y!_l?RIeS0*+0Whw<^^~9%*7%y zJ+u8VFLpX~XuWMR9&0e#;^yQ3oh2obU;h2j@Faj*GK?{VS@yRi<=Dj2SFg#qBMxRb z?&4(4NrMBEu-{znO_fiG`XoL-W)`YZiZs<9{pF(@FTL?u`&-q6TuN=7Hid4#53$i5 zMKjTcpnlGxqSa>|W-@w*Q2$+q@So&U)&eRg*rTe5*3`;=-9zPVN2W{S_g>uO#hE(XbrH z;JQ}#X_fag$FI?qz~SJXYWEdc;a@V2Kr+$)L{5MB-}euBW7BGM3EHctqo!DYo>do%9z8}V5t6T^8XPqaHU zY{vZ;;GBs61-Oq)Xz270CB-4Z0Mga)tooua%#+U1UaU_l$i3C zfO-Lfy;n(P$VS_E@OwPE;5GpI_w#-%^dDF1VWO96eXsr2(DuJ?VE?``ius4cuml_d zkDIqM_pI;C9)B5Ai=^(s-iB!gtOox}aQ0w5&XXVyb={p^WW$R#?+G&dzd{d37-w*@ zccz~YlG|7Kbs<%r`UMM8=ov0;>9)w?>CUup*++#zVViZBh9pP*5K=TL6+)9|`B(zx z=S7u8a5~gq`eEY@v2o=a@`?xJlUUCMUw4-6f3JbWgSM z7yA>c$|8zRsZ82uCASkt@jVw%ks^@{dc|{g#n#xz*|9Ds`E41mM|A$;Lq3Iy{FVoZ zJvd)G^D;MIlA5wOK4j4A-hg7_FpS~lm-Rw9DyiYjuc3D7(c+4r4LW2YO1wpzlmtn|^S>d6 z>nrjlDRmZOp{a5qu8-Qp(arf^&d-yV=_s?t8}n2_av0al!IT8?voEQsa56cxs93~y#4djm z@dznCVa>^E7LFxAYg!5V$Y_?k>({&BpL%n=cE?2NvNMDB!daP%K3+;-f-YdrebkiB zvub7uMn@xyPh zip?gB;zRi@@ew6eMC@NP$bjOeoS5bDc`VfEAu9{zVtIU#7IiKnyzKm7va_|Z;M>L~ zXb(5bs1fA%eK^G2xq>+_5j*1Lu^D!73!q8nfP;zMgIfKQ*Y#e!U$qA&4*t^ zP=9M>)QIX*N5L&zN5QgHk}u-eX3YuklP^>u!fgyTvorK8SQB-;MsZ|%kG`GSI|nYY zuxcbWDmtM_O%z_Wg^T!?>b0<;I1~=#A!UThxWOIQTSL)!#rBJ=SVUp9!B&VQHecBT|cE7Uk1aACpQe#8rp@usj5!jCyXzcD}tuR3Q^(* z60i?P^htziDQb;LvIguIA^`a}rcy@b|4oY!_g5kTTYjkD5)l%<0DWXkPo+Hkytb zeYE|g3Z}eNSmJxa9X972dojM&}Jb#P*$lKxOq=)(Yr{j*B<;Kv$c8nriyW-u47kf z>Ge;x8v)7OxUxA-`+A9(mi7Nl6o6=cCD(SGfP?)4^MQSlYK)4CgZ8Gb+tM2|VLEI3N zg42ENK(r`JxCNHqCC9NCT`9KsrMGr{itMM`{RB>y07adms!(E|*06|%VR1ISiYT1t zvaLo?c{rEY5E~07gF1r7BMe6`Yp>;kB`l4_c4^88vq-l4L6&i7fmz;bE2Rk@v>$~p z4iUI@PU8@IA}im}Pp97UM8bR|zs$2bf68-$@yK>SPq)m4mOFJ!sl zsA9_g`{&&&cg2y}#sGFl_vf#O6M2M!hdWxoO)wGKamu?!6NAHvb+GT8Q>V-`^E!MS zOP|B@1%($2w9?KsvdFO4LSp;7o%kNuajh#^XULMe_R$>V7;3tPQa@Y7V{&jz_ruoC zkH)@aHRUhRPwhgq$; z3tnZilVig7#b9gFd_6Y0xgO+3;ekvsj^%V_`780o=LziHCjD8pCCgi2>gJ0wJtQT! zkXQwDn3`mEYTUra=?Ou7A0!}eVwm9l6UTTfPK8jbow0gPw?nU+6pF7ktY4n*^B?2O zBnRD)6{H%i^r5n#kgi5i{1|DNbw1&F4pzPH#A}96#eHa4E$De9JsSJSj{aNHShofQ zJG}cxX2fg5JT<0AObD+iU<(-q>r1xQsDNK?3DC2(}*2#=lM(FtwS*N16L2!$)9b(je}aS=Bz6yxS;{@x)} z(mhJ$n2(Qw4BpP@G}x-2)Wt%1l6oR&J7|73BTw?L*au@I5WE3yIMBf0y$km$DhKWE~rSQ>zIWk`ColQBU_YeC$C_qdE5r5yLc5SnxT3Rs1A z*H%Q*S8$Fj!^g@_J|4YN&5AyPek?=-Ezon4K;U}SKalG6jEoX!H5eG+R}%eXL=~m| zP}SJFY_iAUCJJ;TJrM~qoy-~efK>g0sxG)`$NOZOIFc7QY~q0l8hcq#fUG7ac5HCma1*qS34Isab+w1gSZtv zzf-WR30}0my2tI5*FWbf6|G-XL7vBWLTWnVRlk_NFw z{X=6~eSHkm_exVX$Uf8zM>n+a5F1)hV1YHxkJOcTO)GWL<*84qslg$-?T9Cei|+dJ z@j`YpCFLtRVkt$mADxemW$pBuI#QhY-&TG4<9Pw9VSBB1MFL|3AE^;OKG!A?`~BTO@YO8v_G_NqK(}*;CZsw-`X>&Zp%n&WS!CDhh2HaC^Y&T z6T3!G?(N{D%v(HCz_j*RJR)lZ7Zp>;I0Cad9Wri{y{JpVk#72qq zb|%Tzw3{{}8}@_nj6UtN`Es-G%pN7uvOiG0Vpd7UlciiVssducNl*}e!p3&OXJ{3o zTY+JbX`m5Axia9Z=Xn+}PCI&%ub%ql{m;bkTH8D$7?O z*8>51k(hdI0v`*fRQrTp6eSq@!f6A^g8$zU50R=ZNM-}aZh@P>$0ZTp5tNLfDO$UGF8 zfVCxB1Ty9A>-&aWOm1`|v0O~v{MW}I1vhr+vr%>>;UBHBMRMzNNkc5Ntc9v1=eC!# zqCnk$e@7Y(*U0cic*Ih-`W9A<9{0TyX)W8Ix^dSqa0p)QFbhF4*L8YQW?$%}U4AzT zIz-ptk+`)Mf>Q4joA12oRdxmWR#HmD(q=Hvkl5;1buwW@q4;z*S4eEn4reWcKTL|& zjYmSRNqL6GwnlcBtBEDCE;yN*RS9&2oFiK7DrWjd-c0W2Dk~)0=AOtn5z1kzE`LJO zdwkMEuZb$vi$!Eri96ZxlvsFBoi4YznXN&jurCBZUAjcIuxxM@&9p7cywl|vD zL286KlMl98(pA9D@t3f5yipPb;XMEIA87ODHfl^A$oi{4z4Z%&#SxAZwF&Egav|rg zg-GNod+Q%AwIqMwIO@5)-)!a^!7?0~Zye?FcoCv@$o|~)Mut=4)42+baw%T;h2)`U zeG*J#|9&}L5}bd$+?k{vgWgG&4NgOqaSHF8so*hWkKp7RMgDCTTLR3S@9*#~)3q2b zeAocz!&#Ok?0hB5S$j@*+{P=>H!I|ZMd%=}O=cv)-hY~t0zVZAH#+O-u9CQ+Za(EO zftB)o@>D*U7VKN#*FaIZOv_S2+Rq_MShZ;?tTD??H-&ivSb7@8y$dN6fK+VJhe1jt z@tCb+A^k{B2N^VYLC6^5EkF*c)iC9GjTY`Ev}crwZxXZ3-L z+HwwdxR!yd8V?0kMNt{oLKHV&%q=xlg@_le!-tK?UquO1TSetlfdWCxO$B-NiJYg4 zV9K9|4dNs%ul>J;23FaEW8bnrX(e5t_eAQIe{ecW+67np+X0c~w;ni~kkUvzyd;9P zQWt2gBch!9A*x>R(yeOzDI4khy>N<^V;L<*CqccYgJf~lp+h~S5enfw&(Mst+jpFhTJb zU_F{q`Il1gvIXWR~4N-FKgR`H76_B+7^jVlxk~_;@iR97B3)DUik2fMC zm(FnDa0iq=O4dA|B8tLL9cwUy^Kv>v?R%#vq|g(StyvJH8Y)v@|Ca;4c2_H9M9X#_ zo6Iu^DnKz3Q^NvTk^CfrvSqJ@x6PCKPl}A_zX#sR!&`m&`nVs7o%!?&Aq(Y<(@U2v z=(8&$SE;F;gqGX<`bgNY_YOLOiXI5%MKWTY(^Iyh_+pP609mtZ<(R^Jqv=`J}GI+48rb??>$B*GVnqd_lh5sU`^P?ZKqNN%|xdF2~VxCxpG0 zBb^xTz@85`T-UH$2WE_)YtZDwn{H?&AP`&jiM1)?=r-V(2Qha52H{j12;iJJC?^Qx_tuts|(+)d&q%j+3IJQp6KK8BxBR zsVka%L>3XQWyszUyn-@(xDdDKvDNZy|L`DaxvnS28rhbGQ9IC;2=ROz5PQO=l+bXK4dx&_G~kVfR`2jF zx0D^FQ{byn$MPN>GcasZ(>Xnk{*+WDX+bLe{@03BF^+&3pd*Wf>e*>G+J>7l({U9k zJz0uR!LZu3r0jyNI#djr$|vh@W_d~Uyl|@TcjxnNvp;q=8u-4;bT&pOd`vH)>>A+L z9j%>&oTPpcH74w|4vU0fsv)A3VdK&}sTD5?Y0jaeHmlYRIBtngZpp-4gq1(L_xi=&<%scAcFr*SkHGwhQ-_a9Ga>X zzcVxkxef6u6E+LJv7!-9n`~RhVtXIcAJBHO@&Hp9x4SwpcdoS6n8oSv!qQ@ zdDQ9Ru*Z!u%dExCIC9p`lb+8i=|qR`enm8OI=YunCylnFM6<}az=F9>hMVCv?#B)+ zIdpZ|;{*E>p;ytbS?5+@nMP%1mhDd{zl^=0qjh4*gx^mNTLGXuq4(boxWGd$(TJK= z_isc&k>r4mH=o|Zys}FKdvIk9OgDG<6<%?q-T3!~tP>vS1x=0tD3&naig+MEYZUhK z(F_KTI8C_fPY!TVG?@Z zJQWS;Nx-)`QE1k+!O$e@Y9>CoY_{2T-g^njPom#5^>yE8uK{?HAYEwF(odSVR8M8b zU`jKBKM&zH5m%EetA{|HjUE>E%&>}ok!{%~BCKK31@bx*{^s0HZmegw3%+%!Xeamf zva)LCE!H5Xt&o*DSc(bx4yEH7yu)4c`<|nK^A0r^gKjnFeHO2%AS;Ko*Ua4Thzlws zlKWC$0S-f)#+5v`7cQbUd`GI`@)=Lyg%N~+Vo~_*q}6py_7s`>z&JC@MBBGS8bJwGOw-=4d~H#nLXFtx zx!HkiDfNOcDjV0Avm(%<^+*uCS&ejO0Sd*x&KWb4wm2P#@jVF1+5GBF=OPZp#Ggab zN*s{$PxX&4V0!BJrLStl)l#RlGIBe(Qf+A57v7k|@bpZcf68_-by6e)b-KMsYv#I& zFI=I*Em+CGr=9yQqV7eWxWCs@7b&M^J~^XOv!w!W?Z=TfMU92z-eo~ zLdfZEhKz@XW$yX|TTcgHqG#p%SuYfihG5nl&QCxrD2Y7kDMO4^%=ntHsDkEG#|2t2 z^Y1mmV;4Aj3gsL1l>Exk>{^SeknpQ zAn=gAEbZN&e8Uj+@4s3iWSA93*?p|`&~n5S#P`{e!n{AfWbO01w!93!=4rlQWEb1o zsvAHmCjTP;fus1VFrx0S@h0lc(>!kG<3HyS;ezn+c3N*;0-~=)EQQj4jCQ}%+M1E# z$(R)%Gq`5wD4n#ul)bCt`ZO=NJM%T<&&Ud~WQb~WhWGP)i_+Vr-jLJzH|G-8E{V)_ zr5see8D)Y8=CB9xLe?Qy^WtfIfO2M}8ChQH{PZO>(`qfX<^X5*;Uc4@m*{zWuJk)s z+2hsRldER${40r&Uz_I&l(O`T6^>~;U!omXHdLT#7yR#E(iIcc7;gj312a!fv)U_u zzsY%_Kfd!MBp-O0$8Fr=Krw{k?!`|^MyCBizv7$ugL2)qMHPkHqgff3%k!13i`=!l zsot2NzuO@_H+QpFpKnS^h-(Lrk7jd!okmM2{5gNee8&W+Wxn<__0ImUF|c>CF0W*v z?b1cwO$8b7RLaeKwN=Sb8vJPWsB-Jc!_)FlUgvXdY=&QdeF-i-D*k!O zfBPp;F-n(X#(KIzweH)!w^x7VfBi5&`mGSG5u&Zw-1g?rYP7^!{CEEY zx4OUbck{y{zmCfPUhEx}AA~Tz{q-j3{Ku>=bKU%y!rl1A=Yk)A@{jr0`tJPp^G5Vh z;veF>&A+uF@27+RoUGmu19RrrHp<`J?pgQ#+1d$d&*)aT`8m5n{F(Xg_Gl&K>(4J> z-bcLsTPsU%R$cZl6;v*MzN-(9Y^&@2E8woqippLQJiohPRtO6GJE;5j`K=1`cgw4* z3x%`lAl<<2amU!sUzH^dz8>_RDP@w)NFnA_3@6k8DYp#;-lQLV}-xtJ8f@*Z&st* z)&l<=tuU|MogC#d%irDIe*VEGC)+yu1Mw(?l+W}Xj16CvT~JUDKYHu?d?6V-OX!ia z+LV@aodT}yoo#cJsR~u3>~0)+>XcIY(GY14F$2c$U|Q632%`nS@372{>-QgicO=gEl?n8i>(lIjqoERAj@ zJrg`eUHxHX@6f@jaVuL#e{1-;in5RiPx{EXmdl>sJ^kongO0EI`LYo4{Skwn$r$1v z6y`Kr{7$e!uI-Xl+Y4a)&~~LCZlNx$bQi(R5ouVv&%mf&=!4<{F*5!-uqJ^H8#*gv zOW0D>uW)}1yX0}{Emt%;l(N}r47PXw-6tI{4X4?xw}h^)5_edzf`qZigsQr-FbP}%nt;XfA7d>xT$ zz`w*qX02U1=wEUc%c@#g@+w|cL|^%+yg8#`{5XxPNbz0-7`-Rqr)i!2NJX}2@6w?J zYxl^E<+n|Bz5?HIYe^aCb~VLFjF@UXN$_jyYp)q3&Z(P^%HYXQpK+1neuK9@RKl`E zVAV=VTApga>#qo(q`m46Vuq7g%tZCUO#Y^*d&-mshlQP{^7YMVs&Oey2A-Q;MuZiA z5Zn6=Eo&Bs7bJ?OcZVag^u28@4P9?}qOniWU9}usT*4R2k5XPSeR94)1?u&2dzMzd zQrK94RF{+BWS~&U<(%&&Ihq$(A-51kz(xsrNiS~F_FzNFA>HpTGc}prDh94b+{1An zttX6N8%lX1q9ZniPhf_aMT=O5+DGlKU6_#Sx8AF^*D-w>L)DQW4h2_?I1LRGdA{|_ z#{R=LYaYG=gH}R%9ALX4!-o4*RCB%lKBbn+5;E}H_R#a7E0QPo-n}(4dhn`pec$t} zMn7rnrb_r9;?4E!<9(~cKw8aA9uv5lC9K#k)0nP9`~+H%xzB>deQzB9BHgpk`Y%dS zDBnPWA!e?CSqO zY+K#-EjpenognmC(>2D~iFK1OCLv)uh|y!1TOVFJK35s7o>MJm-4H!${ugBe{2K~< zGxAi6xb5Y2M3XC=O)p0EccjEoSaLE_I9(!KNr(M~J`7w*Ay8EnSEpHO$PYAEo{NQT zz;G`$zPillyR1><$464l4m;$^WHC141lD+jH1`x+ktCg0fS@Pbg#5Y*{zOy z8jy<(nlx5e&)Jl-=ow7j7|=4{5sYY&gNoiE{MXjkd-_3c-5}yGOobnm#@1qMxmS_s5uFY-rQy4Q87X5F@y%%m{GruBnHBJ3w~|JG?lp{hZ^&bw>4kS7s>b1h$*@nIp?) z^boJGH4p)!jvlAATq@0%QdN~so&Rs6&r}HY)wd6(X({4xFTdLLQVQOG{iM*%uVoHH zgvOya5Ss~b7o@~L%I5^-@z!LQ?!9B~U9=1jck$lB!KI3RPK4ts&P)3CJAY|BgZ(eb z5zC?gAyc(~Ond;JfRG5vUt( zA0p@%U)^F2g#*9Xxlif%v0rWI#Yk#NEj9`od*`eYew|A0qj13{JBp1jL3msIwwSZ- zq&CgcGnke%Hi{81^}48^a%@#PX^)In`5((Y6G}y^g&e<-xsU(BH%rs@ zn2*C+dHZrj6!Rr}Y(-foCKLtiDj3*RrSM;?2V*Rm^@7*?D-kBJkB2KOHeuN^zN#)w z%GmP*8{3KGf%{tb^OydW68u>WU&}QCgqkY?;?G(zRtpElB7;-ttEjg4(&Nb~WdeV8 zh4tQ1eq0aMT%9TSe)N`&(_lPDLS~wb_K2o`p3pg+5J$=?L;c#-aFBjgL8#p^R%keP z6V;f4Dd7+#PLT0X#72?!hL$#DpoRvIy>vrM!+KeZo08wV6gVf!sMECTVbG7<=;^j% zGX-oBeT?~ghcQRR*{>v!SUS6h?V%5e=t#7gu%t(jSae_R)2O%@PB_5>hCo2=&s#>jbLJn91Bit%oS&d~o%8&iym5fTvl;26`cmqOdQ*iK7t zzexB0aAm;aO4n4+i;aS<8BR*_ZB9($>(U^ZG%tel(0&;0BPIBp{H{D7ulAXWe$77RYD}cI zUBn@@k-v$k&1fS12PnQ17S7-DF%mm5N(ak9c zjebP^o2~~TbF!r9Z1(F8PKo(WACm>|yH|S3`Q^ERyTUN_7EiaPZwp1jEpU5qwzw}K zgo@efs}DzEfepr=W4FNtOJLmX@)&28#8}}?n33{Q;Sgw>ARl|=@d3X&?B$WtWzyn) zwRbbjpD-Q*=9$(U`L@nOWhhKT2Ak0nz+32rk;RwK5#s~!U-i0bszltxuY`8>MLQ|O z1eI_>^Yd1l3M(cuqAu>HsZ`Ki+EOEaQPKA4N#Xn;)(zr&f2gQ7bo`1$xN=d*w4R2> zlf!Hu1h^=i@Eh(6Pdo4&g9;l>td8Y>{+Yl}4=CsSrfJS4O zK-lrW%fynoK)8b!kFhm!0|k@Ci~xm%t8$t;dw)9V!F?T0?qX<;IDlPnB2Ke1UJ_N4 z?;SK~w_~PSa&wW<#{?zP2uZyd3VmphZa_cAT;tp~pq4@=HoWyhi#aqDP`}^}q>XAG zvGni%^uA0U`wW$A_-gbW6S)W}p?H~`j*5tOa)C0T<32GE*`jgvmGswyLl=8~e6`rW zAqw<36t}T)n5NCrl%ju^#8g+v1*2lYnJjsA$l3XC${Z#(>&ruLacR%j{hBs5XDD%4 zdc#{yi_(Q8gO^@pO9}<#Z;YDB{r`)c&N$c)AH()X^=FXY8~`8v)}G`yarhln94n%O z!lzkWnk1)ja6V-V^V`&I=^e&d(R*xmi~OiqHWXAn_N)Puq_o2x)M#+C|D?%8mo!~b z0!ojLI7C3i;TJc_)oOOJ_snee;*K5a0ZmDAHYs5n(9&UizxK2*JsrAEQ3nPQpl}-b zo+x#g^}b8{hK-GRUxfYq8F-9_hs%@zUy|`o{6Za#tl?;6O=dA(Kb5Pdv6dEWlkfdQ zU>=hUWA!&)BeZ6R6|;_=y-~Qt)A9~836_3wV5UUG3kD9J3CvG_-=M?y^E?k2WX8%7 zCZe;Y>qm(#kTv~&xy=s7?GZzO4hJCMKo4M@{9Zl8{z6V*cW`T}xB6{1e)CEk(YO6} z$>=#bsW$P5X4)tIqj9aDi~lBcf|N7JlrOVJ_X&M23wbJ*XxLlEqCu`%bY$b~F%(|@ zRy&+Ae=Y>^fVF)f5bx1z*@@j&9mYr~UtZb#%x zf-pD!2T_bn4-Du!FbyqP#Ay6{T2k#bMCO+;21%MIWJrs{xisYd&nbz*{rIl@nt++} z(6@$hyk62RJU1_saL$zy+jzxnEJlif?hBXd7gmxZ+1P*GP9FW(s+v{e;g_e^egy;| zdp0Ob*NmzyvFtt48-{f4ecS%@o0J3BxobGi)8~{OtqtAH#2BMfS<_=9` z=RAJ(lfniH90{|R12JLZ1QDEzGnDvvr4d6@ObIyu3&Wi($e-Ij#_VO%CB%&qDLv}s z9*CLVesPlmhqep@V{?Vz<|+nU2${21-tXA|<|+Ci#!=wQtOaT@;As+{vY$eF8Sk+_ z#ET&^odq7d7W{G_CH?(>Ex-f@$o-I%WnAXfh{MoMy<7IXjfmt?K>KN?feNdgF;Hn_=)AtXY9Gsaa|X6 zF}PQT`%$>d)L@u}cR}%J_wrNQ8wN|Dk_~Z4)hDsy^~B!xH1cVvYB~I(ZJ(db8h@HG zicloQM~u&>3m#z`)5y%iI;Pmdpf6&u*zG*lDji@Z5L%V`J5VwZA%l~VqxP#EnOv-6 zKc+RmT|*l@7?Qkuija>pM!#w#a(X^veCEZm0P0M|jK%&Liw&f`k}#v}Vx3w+Z&e5_ zYdw<@&)}wRJ0-Sln!a;Gwy4`$dQ=<|SREaIgGm9a^Gk&UKB_Ty?eyQVC{1~%>X}`* z9BiNsyw3_^qLebLbkeD@c`ZMj~Rq?*ngQQZKFgE zOn;S^!BFf%4=LhSfosx@US=`WY|QU+*@wMCJ;m1{Wuv7FwXujRUDm7=Q@!Uzi+1EB zK)#L>@nxm_hQLsj9OKu`i`72^J=t$db7b5dbl;Lee}y{*A3G({3+*mHpd+H;TYga0 zo%`%M=G(Q1<7Lh0pl1OW_mozP7BK90_F!^KRKfw3OkMO1c8O~oIg- z>F>ONde1aK33h+)JERMTbvy45rW4fiPq{16%;&uv{x>tq8D0>V0ZM}UJob3NOTY67 zYdE5yHqf)t)l~`712gA-h-%i;L(>fCyfX?E+m`4vNmgogoR7qfAP0*(FH88hysq(7 zef*f1lyr=t7!$6 z=xcJ2!dGne>x`u1e-QtZay12mp1b%i_v7DItivF6#q%dcZ>!==48|}_u<6i72S}2A zDu#*^cW9_?$5y&y7xbhA$t4@pJ<`KFsyV?Vq?e8YQabIC@aGo7ASwgyT3rZI2py7Wls%aE%ryA;iV;gvtomAt{l8ClHR)4=eb~c`E;4pQ>x-_1cj_^j zD4e(uyI~7rFL5`et@x>n3-vA^yB8LoeC~xaebGTPU0=I<+O9@}cd(w0hNxwIX^Riz zXkwp=sZ_#-E#rMYV#b2nkuLN+11J8=pZxmbPWwIYA865%q%tdJAlQ@r_33@`LM?UX zR7tPsCzus4Klj3z01z01>prMkL90P=B*gxD4RcyJAwvt#{YpDZM@w}L z+A6B&#yB)7vJTAOf!w-4#D1n{pMs-z92W+#Eh|`EY9fPD&wjJk9=@y2bQS^~%xG%` zt^*r~X6e|Nl&M`4${_h4?e>rRZZ?rESBrR1n;(5)K|57581b8%FjB=ka9EjU|0{+s zZS4g|K-1NU<@OM=M)2t)=C_KDr0K%~{I)0~O#u~C+=<|0uvMJREwA0cZ5mYqX#sY( z6AQs!c4jAQs6?ddiT;#ofzZ1c1kXrH82-~F?&fM$tDUMwE#0t&uVio%17~46q9?Zx z3{#jjsTQ*7jP(>3(!#`8H-5guz$(uf2P4qv7zP*y53j?wPISfFH6)B%%(oi3VP4XT z#1@ya9V#S)MkGb^a&tC!e6++<=a~JMC}dzDoiisLH?zcP?KDl|K1!E6hhNrM;+s?~z@=ZA9{vsQDm_Ust1WFKx-mBK3WaPPA};MwuZ%kD(X8 zJqf+O@>TU-%b&&JHn6^z7jd&F1&dk2#HxI)Xp=UxRr(SZ;ipZtUg-zM!#n54L~WPW z8mh)yS^gZBH~Jkxs$Q1G1&@yr>Kiq%EPu^o0}VxELInD{z#=@>gmz};ZAa*1YRzyP z{_24l!(CB$4)$iRg(LMI8Z$|Ht;3AJucvILN0j_{`|>{kguu~wtZ~rCL~hwTt^Unr zY=NAFJw8rhW3p6O$hm>QcSk&9Bk=oI6N^yMvS00@Kn;`%Q}257S!#H*MiVXv{{tVeu)+C5PKt{ z1^@b^CrI@pcND_Hu1ZMSZ~~rcc+F zyCwYj}$6 zd%D#{Zdf>|wyH5E%|g+!7y&R4z-OQxV5`;oZUn zCLn8inYV=%BGBR5jZe^i0Ocf#t*eVxJ^!X+;WSw?mYS=H{mIONM`@^CyaW@SMxdcS zkQW#E*15#t1t!pqozT9fJ9zrJK~C;j}YGD(BmNbpwS5M%p!>( zo3yBC)Bz5aCyS)S%f~UH_$2&!XnJV~pqBz{`|rKqOr%3he{vrPCI%nz`L6bigZdoF zbK=YbT*@vs#z4}t`Xt@a$#Rl_#ta{dpjMmCdO#na|Gh~$p7cEhcS8OXXTijmP&$(v zZBMkjl?2WsDbX3uoE;_->-hFZe`d~V=#PNQiLa*H=Z14+iyC9Z<$kbY@_SBDh&}#N zG)@G%uoFdve+d70Kux6H(_&;W5Hsf=Oz;R>4_kA@y%z$u^`S>t5q0Xi zMvm)hU!MKZ%qSKQQ6bF~Z8`TfbbG1BnV-iTNUSt8@8P@vB(?z}lUtZOO1qmv2kZwQ z$S@Y|C_XQ&;C~9m1&$z^^6lI}7PV9yS9p#FBWVKiD+0nTdDrn+9B}a(#Su39!oLD( zB>nDT$$;Oph;dAc-fJ?`?;wGZ2ph#ijB#w+zj+H6-lxAW5nF^#8vuy8PYvVKw#i(? zm==tF*zbjn6FbtZO|p@k^_~q?$*AEF!B89Hc`^O!q*8f@Ci%4gU(m#aQe?6EGq$Hs zx6Jf?Q%gZYyB~s7#B`2=QYuREYQvUcWh$p=jtY$;dQV^m6se88D^7OUvVX;#!~1+= zf@aNMs;v#j?-uA_MjO;_cs_Fy`IcVCz9Ql21ZfI!qz*7VjmUhvPRAqg?C{Q zv5j5RW$03$@|ts*RtNTBIK5=e0VuXpG{pai{B?;L*7H5w54$mxBj;Uzjfe9{48pOE z=A))^y##!_<4y^th$tU*|GP*^@?X8FQ6(o0#}hnf(z}zkeHscWq$T%{ysyHSg~ie^ zeQYSN7j+sju9is*IKl$qv?t(=rQE{)$ILYR$n-l|cn(vSpVq=?BiYoM8RK4_qr3Lq&d^ z*-votodE%Z3b>_Eh>`m5;?7sa?>)rx)r-cu7JW?$J_N!lN}I*6{Zzuichz)1qyZff z7fF};6Sp+0H=oR?xN6dwc(7#eB5*-exHRYr(0#qx0n44b5Dl*OJcHhP7@4NHWp41M z@4Isor9+v^8V?(f{b&85pC2TB2p9e>X_Mqk_ha0i?gLy<<8hTFMKlqfWyo$H5JGuG z3YCH+*9?bZMB)N0YAiPpG?_~t@(gI0PXrttFfMoUMy-8ua6;zgB&o{6DYP0O!n1fi z5k5XqF;!a)MsW3tYbruTtapvSsrY8f)Lm*Ub6V!-+zCusX;^<@Jt3Vp5PhG?S}!_$ zi`7)vXMKmxtMnb=Ctu}S`@FOMKD}}?Aq#dwVCW{dK^aDu%(6;a3!T>YW$747R2MVt zD*$g2iv8q%@*okAHkMr$jzxnFLTzQ&Y{sIlg+*T`Q!O2TH9!#k4+lX>3_PsYntKTT zv>XyjCI|g|{FGi+$hq|{yUIcA`-A|^bQr#lv^-JM=-e2L&W*vb+Y>!F&tjoYA#t*6FAk;6;d_DjVkGIF zLP?y=L`0iU6c;**cmEB<#V1iFCE+SQh{lDTg<@Q8+6hMgJ_+Tng2P`2bCQbPFw*UE zqb}<3@PLKEc$IB+ydy1_tM^JRixWp>m#%(hX0{17AT1m+hSly`DE{B!z@&k`KYzbEQM-Q z8d!Q0z4Qj=)gZQ_>Dus{p_Oh7-+%m?bu+lIQz<%hF%*%*4RO4fEBs%4Nh3 z1i&?sEMS;5X%6;75QyR`Lg`2%$HHs}xSHlBFi9@kWsUpMRhJEP)n&b4V|8#gE8x=D zdK1DFg{yk}E``uQM%8mbA|3mKc*}|nhTa-O ze{R&nq;(_R9Q$nhC($_%JF89AkdfL^i~FrG_8_DR91pRgk={xp(6A9U#%4WgbN>KE zQ@#JC@~63<)8)8C)-o$}BI6uAa|6B#lTAz!>Kj-}E_imSvTp5=MUy$M)vveytC2#Q z6`M&M>T+#lDfxWiDkDVZ2cz z!dLc%6&O-V zkVZgY2tlN~TS7Xe`Oe^dKhL|~cdhRa)|uhN%sR8r{_TD3YwxSQF}(3>_-W{_Qo?;# zbz;~j**PpwuMuELc9FjwsL{%1Br#|(8ph#T>&Xl*Lny%HT~H$1cwFdYV5h-qC1Rzyhpb?T%lmFHeFYdlm&){@{%Rbb!B8u7^TWXciIj=+|^D+DOqYc3`o-4&- zpAX`3@a>n3X15!=gl=kDuoU@MzKyY^&oL zBtGS#8|P*!K~v9wWoLYAHSA$xcIUOgqh?A^#~K6{9USDMgM~Hpfwe*4+NgYUUWf!9 zd^-)=Clv=);PechgqkEP>9qA@A{eGvleoJs^zRT@1*-Fl1G28w`q8ll<)a#_VshGs zoWo59%-)D!OansN2Ay0FGPscO2og^rB3iC1FUMMY+Ol_`T-C2>JA~!C4jqrGms5*? z+mXO{J?o3m?^}r3y0?nte!L<_K!N@d44JoD;{<0H45`I|IMNE3gqKiKl2jVDN6Asd z4%IwdtuzF$6*?G8@00(Iwx@(7A-O55SlqjU8WrQU?4m1)t<(x$0)Td*z?SmADMpB zAiq6`VUEx8g2lZJl=@BOcouwkf5YfVPLC(6`FX-`-*^cF_f|A4P7C=7L~tW=N(VaS zWPPrlJWFD^s`^6(i6?_bQ`k>cO|TWIbHypLn#q17NaIuf7vkj2P#dU6X)9ys@lXZJ zPc8_*&WI_sVL%WUS{VaU3+#mUQ|@+#S9-dqu21wv%t? zpd(C3+Cdh-L0xG_gUmAVdnl{`o5as7l76$P9U?=uHh@t#A@VhK@m`h92-pK)%iTHM zyK?dViLu}!cGC?c!_PO!o|Y$7)u5|{{qs|CBRzO-Vm`|D32G$PA0+e!axuf;{%soM zE>}OUNP2+gm~p*kApE1m3K1A+1*gpYOx7!>15yAM|G!1!MexDt2_0c0jT?<#k;$+K z{DoT`0*%p_vA*qFgnr8)2_CdwxS7#n?h>7Yt4?@K$7Uu4z z<#8l8w<`)lCC+AwoD&zh`#Jvx$5ukra?B>$^x9(Y1k^H4oGfmoPL26_tZP}Gl*?i; zb{rT)GJI`nZpUgSw4U<9r-6J;>UL?CaJ)Z|i-6cm)BF7(vjua|;k1#`w-C`#v&bvu zbpMm1Fw%rHkr0l>YTRc5xt^Z1(G}1+1~9Uv_L&=q+JJZZc3jlpGU+7JZ>ltAU)KW~ zNSN63W;%zddFr&v3{)aYpC8uJ`F8TFMxDvgM*&! zbE3t01A=GbMLss?I3GVX&ts;oR z5fMwYucWvU*1x^3q2FVfA~&MZ*V34fPYf!bx5H2zY#mfyXBT<114?>1w_SX)(9Q%U;!v*K^PF*fLa%i@ljCaq@|VpOZ`Q zIt8Z|h*v`I_-|NZYTpyB;&*t5PG%zcxd-*pb$eDDm6jAu6sg+L6{+pj1OYsffgY%` z@zK{iXZ;rSC_oQT@z%Y`4e>i}`Yq?xYU8FJp(sY3<27)wM@1xEYbZGVexu`8w0W>i zGLCsOvxJ)3N2!9aQdMhCJTEggGU(7585cdZ_5EE7PFXFlM;|D5^5-vfJAmT-(`A@j zi*>f-n4isn2Ce&;@$<9TZ_jX!rb=Jzw893%E%WL#3g4Rm!xRkf`&qvQ2sML&Nl834 z2KPbR=-P!3X|1_$FqxGfCMDzdXX0M^Z4DujCS*P<$X_oo^Yy6--dr%c4Nag&ABF7r zd6}pZMTUx%N^xZ5YCF+=Y%DG6->5qr_B?luJTSEEC^c}>h-)c*%+%oO)HRIi7dARkccTLE>21}M zp;$E%X-^+(5%QBsS2i(K@{AkWR%m5Ls4IHOC;_X@UyLAL>gsee(|v>(Pb&yP_DpMf z;k83+s69p~c3wiN1vph~{!E`uTPk(qk?YZ2r9KEdDCvo8wA!}@l!9wjeKBH1(2>-+ zVx>SLdqGMM`KztB^6lM7ca4M~oV?-&KaJwEQNAg!Gw==e(@l6V8^`zPM~w_N-aA&8 zxDCbKSu2q{YltGYt?-0%JSu&M6Z(*UP?=VTT6zVX8beB1!lbF_{dP9y-w+~#xW%uD zJNBhTaHb}xm$mf^(QVmDTvBQVPfo9v5?%D4+?7^D(wm<9QF#P!V5w3@e^TY;fQby~ zog`wi&k>%V4^jENdA|It_8itz(6G15deKg$38gd`4wH5T7GVUC`}nn^ug_VIx;vuB zY@=_A22h?8iX}UuV_DaND|5z@R?)v>iACtLo8-{mqy6jp5U@uiS`L~jI{c<8I>7|e zsDuze-IKG^@R=0JoV41~Kj7U1x(;VkQ`vN(Mj~;U;1IutJ-f<0w>Qc~MWrlceSHTa zZ!ItjxI$z{E^OYCQr^=@Y_IOhn;;@m&Tfprdsx)3dS6dX-RkRam~{`Z2AuyiB?0Z) z00u|8XJWd!Y=7I*{uDl#zV5Mx>gZ3)VOC{{uA!vD8=-k0gr75mORV=q0Kf(iXeOD$ zadMwc6l#gIrEZl`b3+?Y`J{OZNN<|o-=t|(OX2hV=dqS2hT?Qe*c^YZ`ZB9HM(E+BiLf==lq~AXPPPP zy@!(FEH%6`H$D%Q%@lFQS~W=ka9tS*W|CFzWl5JrgW1bg1dTjE(i%(?qR&F7u4T1! z$B84BuDo6aEr21+G{552|1C|q$Jce!1B_h84e9GZG87*En-D|EFUg^aZ%NWgx!|FL z$V#0T;FId8ANm~abGQ`ZnB!m@g^8*8F~)UEn3S8-GR5`s27OQ}>!Vf|C@6~!#$flE z#@tZ8@3RF!GWUwH2Fa_3KMr>en!yj9<690US{P^mhy8#C z8gyE@=is`00<`Q!bzp7W7NsoAFUImR@S}~3?RlU&M}1B@KeV`VdJcfnYCiPnj0D=!Tlyt^wRvo zY#&Wgra6zJ)YxKGV+0$X_?G`jspp>z7jW~U!&7h!cb6bDF1VjnDZm3zq- zi`6UbCy4V6yHoI)BRaNE?J}%56y5w?$|FU7oZ}dxM0{*u%%e+dgwhtDhy1`mX%dG9 zrVx)2PF+s?D(CX)rCZIY!;?VNX%qlJ)(rs`MvvfN=#lEg2DoxSexL^No zb}B^kIjO}wie8Ih#8U3r`t6#$mBjZU0QX)Dth8_=acg`!5k+V#MI3|Ad@tgR%*h>o zZuX{V10`aWGxxt2N59vPsa6#NE%@W}zIm=R5_D>;Fm6>#K-m4;Azs3$RvxPlWHyMp z75yXlnOESK`9SOo-t&q&hj!qrQS@S{V$(=pWDVdko|N@bxeGBpl!>O}rKW zA2gZdqZ z8M3bOHRKpiLV2Jx3HxV$Iq2mkrZh^zRY$7*H_o^ zl8IhPn6)M{aQ4dnCqYI$t=g;Eu4)$Zj0C}19qqP7JCDWIV)c90KnhWNgRa{{IH_Ev;bR zqy2WnREF~C1R>)SLekUM60T;Wvk9m}Nk`pUYTiO&QDJ3a#qV-nR6tT_l`$y>N-ymc zZO`?s)C?XAB9fxPsS*UUp56 zgJ%9pB;_&qAwu-+9?OwXKHI)-QgQ4-z)B9GV55{&2^KD08@?LA*%COK^=NsEnI+m1 zhjmJ{q)?|PgS>NqVmA`Zqd9=Lvdo%;;jPxwubG5T!Dm{d0;y)!KP{uC1vKMw=_X)2 z54FU>qa5(qb6Y;7{tAYP6CdX0;&(eb&vOCIOEfZu7ukJ@F3%(NOFzp$;o+3A1#Wl; z(5%zYi8E14mr@=9|Aq6)!)obQE(gRt1M;?a1y`UJ1zqLrDlK>#1(x;-atkpdcE=>1 ze?vdD!``Onj7Q{E5#38rs%rcUe+iXpxcO74w^e71fg0V2L=&etvI-c2WK&C{zKTP+ zSeX3=i2E@sfp+k(d}mnnAe|0Qc+=a{&Hhwf&jLZpP_EKn~l?t_+|Ui85CH0i7UjG znHWXTMGTCi-*Zn^GESatUyBd`Pg^0B+JxJ@@gV-eLo*)!_iDcheWL4dPH1nojZEbrvddMY02@X*}J0pk97KgMd?rOF1k zNOMkDef6gWL$9Dz5Eo?vvs?Fqjy=aCdFCyc+1Ndf54Ien@N-u!*|slXXULE^4!QiC zftmSd-Gm6FmqmIS2^l|a?}_R1C)Zf7M*Zd|4EvoYZ1kb%RHjEtC!d=p!ln!d;Ai89 zunmvHR<{HxBAdP=;zHrEYMzgD^U^4xT*;4@rT>D;e|w3(07ibHNM}niJDb0uWKJ_> zbNQSS;rot(@e$@=2n%*3fwaeyKhUQ;B=KobbL{FIv1X$nvnosBId!F6@c99H_{f6A$xm#*|DgVn;_(+FjZuHX=n`OS^r z>mwUu0B8*tSEM2WkKj9UqB^G@yyvi6*@?#q1rtk?CuH)22PHu&PtccN#ob|##~4C4 z|A~MMt%#2TG)9u1GRUgU@|{IAh5KRtxlbo%?PJ|4<(M9E&tJ#{Z>O#v2GRV@FakYJ zV`-x<$3pkX4|^O0Gv6A~myuAh$_PX?&Z)9CK+{*AGh;&&1?fLsw%PqlQ1(Dazd!ue zb=-98dyF{b-U%IRS1Bb+pYW5rKuxbQL{&7thrk+6FiN3R4!caekd)yN;%!(2GuK3?Qo6#It181{zk0|u=mpUSd{5;sCiHZ zacvgZxxOsqC}4~LT|H`msYtTnti?)sYBMYGyglVss9!q>`UzP1M^o`Kt@?*}8xOZI zO?IgLco8Z1nMo^^2Uy_2S@OqSQ6hw|Ya+nq&XyrDiSGVqlMK}Bc*%G;y!l%hC zU_9W*bQW90r+9mFeT%>{zE`;tfwl0Z0Yq%ONiBs6E5<})JYUM|cTc5fAeh4ojK*y` zCjRcLqy;YWq+9Obb&R9e@a6%!N{n-IP`>BqxbhZ=fE#45o&Y9}jIbmjwqBR}_vj{? zNyL9*;$GT5O^n%Z?bWW!uGYD5jf)kcqlXZGdNk!dPw;W@j#r?pf02M*TIOu|pvd=K z#64^Dv!DW)7Aa2X6wp?u&F9k5J7$}i+ zS;`J)lZi?gfj3tniEaFH59q9HjG&sSYDUq!CLN(^eI%quoRHkp9}jzGU?5-;gL#Va zmBRux6=a9i#N#AakERy-im#_s(11&cX{XLvz4`c`D^pSO+;-CX42$~w?iS&gRuk#H z-lM@H?;YnYT>XSDMIo747;}(C6|~kRt%Ki7hQ;%}#M~^7bUid|cqyX$;-M7mL%%;D z`U`7JC*&(vj+-2N4MSyQ<{!no`o)cdwsOw{ABS`&@9lGp8EiN6OHnFbwIh4XV$Vv5 z?L*_(01R+MWm09;RRy1ZAn!aW5s!Le6-l~eNma=S@h?{V?Uotra7v$u5MCLtMlG>c zhrCe+o0WeG1%Qc1ZbAdTra_Fp+9=1(qGb3^W8m{=#9)>tq`$+>C!CNW{}m>Jjp^@X zi-5Og)|QY9zk+@gjXJV7NBuc63{rhJ`tHIy-7hFG$F3s1Vp*$B?T+P(tjEw&dMQh3 za4we45xg7pW2IgO?YAK0YrON0%wsvd{X-=j801KY^ZTOXm>0K70!*E0lf$E-maxZ|W6`bsCI2EKF{d~? zrW>dj+qy{RU?L!=+VuFDvf z{MpYEHS4oC$KzbIH})$X9v9)QK^W|cHXJzYk;!9U`8)aJWf{Bfm*#nM9I2zyt}#&e zeTuBV#zdhc!2P8h0q^+&(0XR~X&>@dGDHj(NDVO#I=SVbpzcubD+ynz-F} zyRLU*xsAIJuhy%h(Ty+^CK@B;;D5s}fq=d?iyw5?_Rwwz8d_obsNN;iYdCK#NP&^k zo-;|n&VEP?$%On1mNKYDO%Rkd)W2^w%Fp($-KuYmJ17h7LRD8%vAG@3Y?s6!49T&! z^(m!Muc|yqz9LR6ihvxRJjTp{Xxecm{E|-}V>CSTa6a zaC;xs1{{0p=H7({$;;QxNY9E7O?%rdhef+VwfYuQChM^-!-5Qkk;ad13tR@`^t#>; z0JQ}G3y|oP23?lluZZ@P58OOXYof8nRQS;+Bz!}Gg*|dP56S$R&e3)jRB_Uuz7Cr_ zNM(|y1^@%jIrY*aMgwuKFGO0+dS->zKwCdsSoX_Jyn|ydI!za9kwWQu85M)Y`1yLI zWx|#&h?-L?mLgtyZAS4P~{v)155IH-11i``a}-!umMv`PPK6?vXsR*EzzV}P-Lg&Bc z%ZpXX0%d3fOuXHKBF$k)!|D)$ns21kiO*-F$k$0_&;DfG9pA5}?usIN(O)h`&Bc$o zEaVm*cLVH*Z>+{HD+{}_$|vOBr&K%uG==Y7YMkbD%=Bk z<`dWxij_4#OEnvn%r&TH#l51YblLOXfCK z>#QfZ3T3wH7iwXg5%Mz8+(BS<7+5(UnRApyZp=R43^$E4CxsCdSL=(RC^$qb>1`jq zd6kS-eVlRuEbc%zmH{b|`-C?-1@RBmU&f(;aKh`))@BE0T4L2IrraInnOz|Id z$!{V_=U$w0%mYnAcR#2{WSoT%ONEcG*@I#+;rNQJknS$*HvewU?yi_9!c<0&u!=cS8W$5~3UxKw z5Ufz%Rao%5_h_t`JX~o;%DHC}x_oP(mH6XYv) z61I$Nr(LJt<$gfn!b{N)(W;)rkmF3rTCBQ)-Tr%fBx41ZZYWiIQ%p#Nk|<0c>e1`{ z&dK?Ml+K@08Za;(466XDYZI#=S;bqTuD8|3bO;Z9c$f|QmP*=rY+-Ab5&AL>ii}8* zNEI!Ok#JN~g=*M9gQe46Ps=yWIqeyiFNE$36jr>V0`O8&pmo)#{`oDA9&t+G;G%>j zmUk%?FysJ?`l6GGz!s%EVFXqa~M~Xlf&v^<_x|w9qrv%Kv zHbfsbr=w8ge|x>}I)I(V;O*YEGb}3RL+(Z(MLj-1;e)Y^Dv&vzAs;AmJphIoW* zV_IGG4@^anolSJWjWN|Crr2UjXRaJK{PTh{X8Wz5Ac+F6C_hh<3bZ3##N-=F5MBK5 z&yMI$?{I+Gi=gcQwjt6f8P~EeCA{jZw2|AKLHj^!)AjW9h{XmzRuJ_(1iHi>AjMYy zK;4wdrS&jU!k8`1hI8a;gTOd8PLZA%X9>r z`vzP0e$aN)Q*vu+lM)Jurb475^e@=xz@-2L6fyaUu<{X)o|0oAmDg{GUa%I|_;dce zAp`d~Bm)KMz{!c>G7)hL*={>vr^snhux%HP(`jjPwmgkc_qM)3@ir~PtRQk;8WhKX z{781V)Kv6Ktd{w1^h`Ok{uQqCZnT1hb0Iz|+T?A#ikl6v<^a8;i8QDP@27fR-^Mpm z$~A%t>`@XD2B%(|4h6rwAB&Y!>u;7SF6N80e|xbju%1 zhCdom#h~Nw_6=+WY0k!Vo<(9%kE`9a3LtC#ARq$B4W8MW1$XA+j`r zjQ#v8q{PSqbyC`1xVE)s$t=0u`UP&}OgJZNZe#Wz?rpI9$mz(tJJLVHTAwSC~U)V!8gd#O^`U0b8aDr`xYV zpp7KQ-learqFF+NRjh;&MV)9b&!oIc-HszSno+E5U1k=8YJtX4&lOTq`5h=xi7b{k z6Ln(FM)0uJlP8N9tsGSMP^x?Bkaehkm`|Mz&Lgpe+O`oJO7Al!QB>>iNQf~7m6Z-C zr+z|obREX14Q^RQuNHy01eLIKh7%JDXji^4maXJ*hu4ooQ=m>E3~>4FL(#}toQE_~ z{|+xn4hi1nm(C0$v=pCGB0D`eOrHrvY|~VJ>(i-A?P>oNX*C&)g>kVKnJ4H0VA5#V z9rj<-?4B5li+mx{rYGdr9sPF3x&$!|FG?wXsM$}iIIoRlP5(o*m+!+k?%TR-a)suY z?us1Q`jt8kw#H3quTBq|v8mEKEiUSGxef-RiljqxQ0r}xpf_D21rG(^)?-?LHYgX! z#&~4gin%^Km`~1wO77f4fg}qf^HeIjko)N&?ZZP-16EpXH~lsVpk8{&N=*_vSNDO2 zEbltb-p9uccY4FE=sn*0n_(YLij=*xJs~LDy4-zCw2ssx0J*lqRwa+2;^(tx{|>Ua zSdvn!Lsw{2q)i<(eqR>U&3Q_T_O_-04+uLSH4*s)B=?9JBY832+*G?0!h-==`}Iq%Pi;S4 zX10ytyN>tl1@f>o1pRxLRe7Bw2XDpok*bVL&gQx6<4xMejZZCpJ~n;+{%<<`k7gFm zQyTUbt_EHMGLSbuj)Kceu;#kho$IdCPb-j(|BM~;i6L@cm@?gLS@8G1T;IxCn|SWA zW_Zc@pdiQ4&Go6H!N=)o>QF<_!Log8v{A<72SU!N`d@b{r*YAX$wwV_azd_dnbRLp z)n*Sz?VMgeD|0gBA}a|M_gMWsx+YB=W>`jA!pswFlre4ei22n|j~U?KifXGR)*2YL2NplC&sCm7F6;)Y%dc2DYYpU%Sl^DYeL;4Hv*U}=I2OuDJeV>l8_xsZ*Pc48a& zzTAhMra{r%E?ByUOgOyB$S2=eU9hmlP5fE4L&Oaz4P!qE4(&@MT1|i0;sYF1G2qjK zjTeJicC11SZC6nvf!xd`2ui6iopPnPwrVlnZnHnYK@r)O$MJ)kX>!HcrPpI?oNrj! zHDeoSo=iaQ6qN{xky1*Dm+eTMxnEaE)uuZwyI*H_0!~=1chYq2rgx*r5ZwNN@SH+8 zAfu8>e>L-AZ~oggWeN@dsmRa7Z&Zqoxwy>7?L;E;2f&mw_hG~5exgr)#SHA|PD}LA zEj$?*V6{41tqp*$=Qa-PonC+VUsK7h9y)hs{gqlZ6~Fwk;OBMJr{`Lw8@kmJTYjbI z+i-o{BJu6o{kmMz*XPgf#e3&@zl+V48oT|jfUh?+qt^#Jdn?yJ1DbC|uD)Ey-y%$} z7Ea2~reYs7zqws%jJ*}TI$z(XtEJ;^9NF7o0?eUQR#G9FD%~`se z(Fa`(NB`fkWG3yge%2K`5?X~7-c?$ZpO)uW#nmafJ)#n_>7TOD5o(gEDidxrGatCP z!X0kR-TkX#L8$m{Rvj+pIxDoU^u-PevxQ=uHHmK(<{EE>%2{rfjkEQJD*axI83>kt zY1bV1^i-ukcGD|Qnm#t8Blri}nYSko9Mu!9|XEgXAnysnSZ69qK z;jv5u>uR$L*7t6TPn#5Ft#V2IBd+R~bli!d!b%SFrr+5s%!PGNjV)+ruL!8l9;QTn zPn@hLZB&HDo@Kggj#M4kDM)t0jnJzCx;R1?JFwAHl5WVS-}ng~b!j%g&ASoaG#Ji& zeDx>#wfOVhdT0zQs!CGvbcw~l<870VLj${V)_Ja9p`Qp})}*HbcUEFY?N3D&OYOQc zpg*||9r2P>)UK=G14iX3kB@fTJ(6R8H;3i$J5}1qrGFLHidn+$bZGz2=HuK2ov3J? zX%&kcChLK&t3RMYi>8E{@Lye7KwJ{2#|7UWjx4YU#P+fpj`TN>-Rtbc=@2Q_3r&Z(3pORmQ2-|BR}B9p72Ra`_A3@z&;xiiIy{twp)(Y#r3Ld{Saq8Kjg*5oDS|zwIT7} zVQA{7Sc)MIoAXc0C12X%#unsGe*O-1MREQ)scuO4dhAE?t{qwjt&mU9$v+)GDy&AD zMY2vgDVAU~0zO5#`Mx_@i)qJW30!f@&+7D23zH{P40@g) zn8m$Je*0+{^_CUIWfgyYUfbFMqA~!2c+2K2g|-=(wjA0#4E%9UtTquYi=bayDy;Pg zK~X+syX-^;j3uanqzW-JUlPSw3V!iz7=uvoGW6iMRTF*`0eNH?h21}eRg*{)ALm*~ z-g+^Ro$d1>P_paI$M%H)OZn;NUvUT# z`?Yi~aCCBc3rdb=WX^dP>7YMvL~~=%UY&Ke%W{Q+-H|&k``yNP%>uBJ1<&pMDmGk(in%Up=g&dpH;78frn0H)f14s~ z>+Sh1)404S_M>sR&BWH!iO=18M9W_8%I3C@c%#Wc*+kXW$^U(+J1_v8_@8rPtiv7o zMH@|GJ+(y|O;l};bNV;#PO{g{LLTEPdsub=*V)-vVYzAcZFsgsMU4WIlcBPWWH zOD@%Ax;^B4fTt|jOS|qqnG!8qEWXC9M-+oJrlt?`2%@>SxHM`ClUaBFm=fbn-r!qx zjW_~|$_?+kKIkV_zKOSCaV}m`Xh_C8c3HVKS)u=mrxYYaTl7>&ZQUC?SM1wN!EDfpDSM!srSwr(%nSv+En__^9 z_vY*^`Sk0Iv0{Y@51kV0x~#?t#w+7eDW{WHu(`L~7hmVP!UT2`KA|&y6zCe;wfv+i zI_|Mu`j21flo^?)ievodgMy`5XH%&NA-vPg4WvUlG6hs&WH9p{%@rBi+{x^c$K5SP z?BMV_nce?!I3Tm@bTaEd#?VwyM0y3N+uOcW8O!~vBmZBA2nbaE73{u~r2MPtU5OEk zyx9NP0B$(ZP+V!>2KOcf_~Sr0XB_^0%Yn#ZKC8kANM5YU+`4-M@*Nu3k?>4jB>FXS z>U&>QY5&dht38*?yMqQuuK?+5ZV9D*NqnBOa|hr&TAjNq#2Wf0FW&7o{faRFEyGYv zt3TkOFT!wsKugRkEZ1|7ZWSW zlJ*BgKR;T&$Py^DBZ*3`eIpU+G3=T6xUKcX65GW&8UvHl?}g3o+Uzb;@efoM^JKUA zlI$5q$qAJ(k=A$(=v@p<)c#;Y385Z&3T$AL++jVz&&{VSnim-_*e3SvbeEY z8W77b#K)z#b0>w(TRUs93N|l-l3zt^bL>Cp!QHKY^|IdMRhK-X`~_x@T8xs+WXt0L zflrYM(c+2jQeEvyL;3v-PStKnKaSO~9(>%$K9&sid?tSU!({Y+hIpb)`H+%>vv`PN zLPTszE+Nx$*c^S&BhpuQ_QJG$7YZwM+^IAtMCAVa^d~^+kmdzFPrMiN0<7u~W17L1 z)qm%Ed_^pQfLSc^Y|6piX`yjyBcAt9dW@c6b#=k-p*x9pt11$tQU0)Iy!((ox&IiY zgMc(LLYi_2xwE4w9OQ@G9qzeRHMOxm~Qz!AfiiB9LE+9*O)V{~5W#rYl z1PX>s3OxN>B4p@U$#+(KwNCRtwP>S3RmUj9h{7M18YL>JPncH1NyToSrR>Ypw~XtD znl_`zstCPt>wR>!xHfd19Q%o)xMY8PWw%)8gwH()sSmP%CJ&cUU$TWF>cxP}cXmSoo781u68 S_Imjin-51Q$2))v<^KW8", - "children" : [ { - "name" : "194\\.6\\.133\\.192", - "children" : [ { - "name" : "cluster50213\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50235\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50226\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50228\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50217\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50214\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50231\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50223\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50232\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50204\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50206\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50203\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50205\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50210\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50208\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50225\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.128", - "children" : [ { - "name" : "cluster1209\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1205\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1235\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1239\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1200\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1227\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1212\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1223\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1217\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1207\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1228\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1221\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1215\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1226\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1236\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1232\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.64", - "children" : [ { - "name" : "cluster50317\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50283\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50292\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50291\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50294\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50285\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50300\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50281\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50311\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50297\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50319\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50286\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50307\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50296\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50315\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50316\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50303\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.128", - "children" : [ { - "name" : "cluster1859\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1877\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1871\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1876\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1854\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1841\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1858\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1843\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1857\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1842\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1872\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1869\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1853\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1846\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1867\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.130\\.64", - "children" : [ { - "name" : "cluster1976\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1969\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1961\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1963\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1968\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1979\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1967\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1989\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1970\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1999\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.192", - "children" : [ { - "name" : "cluster1150\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1127\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1139\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1154\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1138\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1137\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1130\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1151\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1131\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1141\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1124\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1158\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1140\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1144\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1136\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1157\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1143\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.0", - "children" : [ { - "name" : "cluster1592\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1567\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1594\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1586\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1561\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1585\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1562\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1581\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1566\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1598\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1568\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1560\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1574\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1573\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1583\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1579\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.192", - "children" : [ { - "name" : "cluster50364\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50372\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50365\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50377\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50368\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50396\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50375\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50389\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50382\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.192", - "children" : [ { - "name" : "cluster1533\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1531\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1557\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1555\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1534\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1553\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1550\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1540\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1538\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1520\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1559\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1535\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1525\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1529\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1551\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.64", - "children" : [ { - "name" : "cluster50124\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50159\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50144\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50145\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50133\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50120\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50132\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50130\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50142\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50147\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50156\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50125\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50141\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50152\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.192", - "children" : [ { - "name" : "cluster1250\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1276\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1248\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1246\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1251\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1259\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1261\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1260\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1243\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1256\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1272\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1274\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1245\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1249\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.64", - "children" : [ { - "name" : "cluster1041\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1075\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1042\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1078\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1072\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1053\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1056\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1064\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1055\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1070\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1061\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1059\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1040\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.128", - "children" : [ { - "name" : "cluster50171\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50195\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50161\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50191\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50174\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50185\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50177\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50166\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50173\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50170\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50189\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50179\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.0", - "children" : [ { - "name" : "cluster1283\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1295\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1302\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1294\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1310\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1305\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1299\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1281\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1288\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1289\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1314\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1315\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1316\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.128", - "children" : [ { - "name" : "cluster1107\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1118\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1080\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1093\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1102\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1104\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1097\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1087\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1095\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1110\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.64", - "children" : [ { - "name" : "cluster3071\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3079\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3068\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3057\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3058\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3070\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3054\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3064\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3077\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3049\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3063\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3075\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3065\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3076\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3061\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3073\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3055\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.128", - "children" : [ { - "name" : "cluster50468\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50445\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50476\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50440\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50473\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50477\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50460\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50475\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50459\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50447\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50464\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50441\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50444\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.128", - "children" : [ { - "name" : "cluster3097\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3089\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3111\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3093\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3099\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3106\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3108\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3112\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3085\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3094\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3103\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3098\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3082\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3104\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3114\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.192", - "children" : [ { - "name" : "cluster50485\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50493\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50511\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50510\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50494\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50484\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50481\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50490\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50501\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50478\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50491\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50505\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50488\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50509\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50513\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.0", - "children" : [ { - "name" : "cluster50085\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50117\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50113\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50101\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50108\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50090\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.0", - "children" : [ { - "name" : "cluster1789\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1777\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1785\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1770\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1793\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1779\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1788\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1776\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1773\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1798\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1762\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1772\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1778\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1782\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1774\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1781\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1760\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1796\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1775\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1768\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1786\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1771\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.130\\.0", - "children" : [ { - "name" : "cluster1959\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1957\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1931\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1920\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1938\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1925\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1932\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1927\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1933\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1930\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1928\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1924\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1953\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1936\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.128", - "children" : [ { - "name" : "cluster1717\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1708\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1707\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1690\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1714\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1683\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1703\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1702\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1694\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1700\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1711\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1713\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1718\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.192", - "children" : [ { - "name" : "cluster1418\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1429\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1420\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1412\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1400\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1415\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1437\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1405\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1427\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.64", - "children" : [ { - "name" : "cluster1334\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1332\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1346\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1350\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1328\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1333\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1321\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1358\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1357\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1356\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.192", - "children" : [ { - "name" : "cluster1914\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1883\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1896\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1911\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1913\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1915\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1903\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1906\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1900\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1891\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1889\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1907\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1917\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.192", - "children" : [ { - "name" : "cluster1006\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1035\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1018\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1026\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1020\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1021\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1027\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1031\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1036\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1032\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1029\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1004\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1011\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1008\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1025\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1002\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1030\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1019\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1017\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1028\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.64", - "children" : [ { - "name" : "cluster1445\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1470\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1449\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1462\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1450\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1454\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1466\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1465\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1474\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1444\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1448\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1463\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1457\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1447\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1455\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1442\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1479\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1467\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1446\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1440\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.128", - "children" : [ { - "name" : "cluster50034\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50011\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50023\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50025\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50024\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50021\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.64", - "children" : [ { - "name" : "cluster1800\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1809\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1816\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1819\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1813\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1806\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1803\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1835\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1822\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1807\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1823\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1832\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.128", - "children" : [ { - "name" : "cluster50359\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50326\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50348\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50346\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50325\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50342\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50352\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.128", - "children" : [ { - "name" : "cluster1383\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1378\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1393\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1395\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1396\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1373\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1388\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1379\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1370\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1368\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1371\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1377\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1369\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.64", - "children" : [ { - "name" : "cluster1643\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1660\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1652\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1672\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1654\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1648\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1657\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1655\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1641\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1669\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1662\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1647\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1649\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1666\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1678\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1650\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1679\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.128", - "children" : [ { - "name" : "cluster1482\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1517\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1491\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1498\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1490\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1504\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1515\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1480\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1518\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1493\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1503\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1514\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.0", - "children" : [ { - "name" : "cluster50520\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50539\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50533\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50530\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50526\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50543\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.192", - "children" : [ { - "name" : "cluster50407\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50409\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50423\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50427\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50429\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50416\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50420\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50418\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50406\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50411\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50425\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.0", - "children" : [ { - "name" : "cluster50275\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50254\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50272\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50274\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50245\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50276\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50243\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50252\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50263\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50279\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50273\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50261\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50260\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.64", - "children" : [ { - "name" : "cluster1613\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1639\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1615\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1628\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1635\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1611\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1607\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1629\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1623\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1633\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1610\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1632\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1614\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1636\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1600\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1626\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1602\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1627\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.192", - "children" : [ { - "name" : "cluster50047\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50055\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50051\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50059\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50050\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50076\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50077\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50046\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50053\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50057\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50072\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50044\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50043\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50058\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.64", - "children" : [ { - "name" : "cluster1193\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1175\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1185\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1171\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1174\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1167\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1180\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.64", - "children" : [ { - "name" : "cluster1755\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1757\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1725\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1727\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1736\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1735\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1722\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1752\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1759\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1758\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1732\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1743\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1731\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1733\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1751\\.secondleveldomain\\.com", - "children" : null - } ] - } ] -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz deleted file mode 100644 index d8431facdaabb588a98c1a71989c2b6a6a241c36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24960 zcmYJ4W0Yjg6Xx5tZQHhOSKGF2W7@WDd)l@!ZBN^p?f1WX&VI^!>z<06S(#79FCwxC zW1yh)+<@ePKwXUN9Bs|$tsRW%U7d|gY|NbLZ5=FJ7%aU1cW=gE?c!h$bfu^5ur1N% z|5A_iXe-;wh?xO7G8M+!X0onu4U{Q7pJZ8W(yDVlBE>Ww&@bME$*@w6&@EN9WsIlA zqJ4_i{(e!dYVmfHoSeM$vEj*s%lOU1|CXP;2r7TXytwIa(;Y?xuN{BoRV4BwYmPBT7E^)hKnN$*Slh)q8(KvDTy=3|AmPhdUbR@XZsO{X_ zBV!#=9+PdV!S!z7F^`|eDA7Dp?o^k~BFv?k1&~-?>KXhURl~sb)#^XeFTY55DQh6@ z`+g~~U$4&`oeVi-trn{DIl1q=gqPT#WckUNm&Y^K zrs$1^r(7&!hGk#ZnGQz9BUi~?G z70qLub}wO;w;!C^{oUPI`+5aF{2uLGIrNNX&zl3}zga9wO=xwUotyjJru9~SbPAUds{vg92@?Lyqmu3{+xpv`eqNu$ELc2$X4je66U84 zl2hH)*_&Q6rr!hhNef?FM^k6cUs;QiiMiTxn8g4O#J7dNg`dVSk~<%+nAycAf(HFw z5BDSH-a0kH|Hfhr819R&J1I0{Fem(+J@~r-pIp6k@-h#{sj341dgQwM#+A?CVlFyn!a@@uT{0#y)_)nG-}O@yXz0T8B+%x+jDd&nhhk{ zr#>a5pW)$lG`lQ<@4N%Tip{>VC%!h}?_`$XfF1Ce{A2}=M6~BD#7?-LT^X%#$njR# zV^HERvLvA;-RO%#i@V}cv z9AAU^ayz!JeCb0_LTd`aw8lx6gCubjr%h*1^?-e~r`K{rNi=ivn?qa{zP*ZEc1Gfh zTj6E542PVp7=>enN}77fFVnis0flWvs;=Yjq|Olgid(UmQRN(ZkQ|4c<&RxYmZlpolEQ?tP{)XUjy*)*U&!Chb~B6F3t4TQW-_x* z(Jzn=V~y&8VK(v=2fZQf_yaybHz1zbIDjNEtYJq{WV3wtmvVA2Om-Xw==2P9u;7Sl z6Cfwjtb0ciLWE&t2XjHB2sFvS4mJr$q3v;!`4_MQv0x&FFe3?gs_&iRS9f(8{uq6V z14oVmy;e3!(~|D!gHWRu(uxi)7&Oi@>gX3Uj5!0pjpdOFtEsFejez(id?Wcp1P>6c zx{kdX54wn%1JE=<6}oyj?)den-`RCjO7?Qq5-&~8vH%fhe>f?7FB7tx)e05;=+xH^ zw#4pi^qo_#5o9cTtY%`k^62M;{RU;Tfm`4qRk*pfc%H*Us_wCGTS5~z$%0p9E%ht! zjRZ-jRgma{L5!eWxoUm#v4qj$BK#!drHd>j@DT3 zkI1O#hZv1~LTYfRGS7-?>kh6&G@#0rTt=y!H=VXI<))$KHM-*+e?J19r&A%dq78NM zLlDzG7CdhxA6r%*-^PsiJ{*bcX*oo-+=jhXT|^DB)}mEwy(65MYd9gX>IXfe*qCuo zuV5}XBH;tS@U5836oyqExP){$^{tt+iiqjr zj9UF6B9H>*Syv^1j>gQ%;F}r?)+G%2NHy$oa*nbO z+1xwNCgRX}COvGpfMS_NGXciatXwJeU_pRvgE#$b9Q4d1t+?+QBTxhK?QLpHoYOBQG@?aZcM$#0PSGU zoofZa6zY*G!zPsVQW~D9Quba|qtiBDjyia=Lu0-=_%a^oK^oZ5W`{q}QU|t>FVLN# z8UQa6vL%^8S-0(=ge2Z1M)g;jX^!h<35v|dhQ=hP!Q`xcQ=c@a<~>l7tcm(Rgu{m7 zNkhT_TctW{M=5rPs5lE@cXSnr&89dX*9Q(f^%S+Spo7Z-VZLY&8x`pF3#QKREYo?$ z03Eoy*;YsR|BDLE4ld{}-zX zz}%z%8G{A9lWvA(C$KPH`aP8{g4u&H)ecWQGy$zWr+l65aS% zSX#?Jd4+X1-@QZ7SY9m2Hv^MjZ3J|R(4#?~4##wP@iy%$Cyz*mzUo1o|NMWcTey)X z34|NX zkPzl*KQkHZldw_zJ5oz3;{xA0kH^yU{jW(TRP81$t9!WEh_KK>a_2U$rOBGIFRUZ*2R_n4%ikn#80Wlnfv~;!ZrA4P?Zi6N z&-oMj;pWO~db+&Z=1upJHz!;C%lfPY%go{PNo4Pk#jto2_4)eFc1(ABdLoS3V3|m~ zm;T=9z;#l*{~xsgZ~0XHP)p(es73PsQwzEkIx9D&`(9^HcUNUDX0V0p=_3X9PQQF# zEW=v8{IR&GBIeS_+3JpnJtz&{6eESjEBC%0!}bUT`?{>btT8|6r{~?2xc{R!pKKil zW75abwYO{gtCP3(C(91J9{1mexv7?i7`(E)eou9ZrR0VD!wKgkg63GKI&A6mToD_q zaUP3{VL#q#iMfl~Rf9KM*B3Y6mr@p<>$K!4)90x(_sl=k)biS;+ENA= zoqj7X(){;nFF?!wV*LgUq6ngZTO8{LdAo+8Hb)kmCA0Ck_~2l{+!PR*EqV2*C=<)NPym z3vf>Z8;~c|Pltd9M4=?z;DHE4e!2^Ei}b@=W6+Y)pn(xc@fSEie4#%B1llA1;V}px zL}EOU9@jJb8oJ^;J7+(%%fDkg{D+~h>mK`#zx*Axu1ETZ4hDQf6XP%OG7^~0cR{z}B8sHj!uY8P-g zt!ybBqnUd0{?ZtpIM_#0z}UPFNrrL)qGbNhBv;gb_6MY6U3E`^6pViiB&5WnLOP=S zXE8AOKl8vj!$0ps*dvp`2}=gYNe~?uh`LIyx%~JreYWCIILR*~ROOoENAsVtSC*Dw zZ%;uT4D-q|uKp0mEsIosvr^Zq=5nhyFRpGKibS25ttUM2o?;bzA7jxr|EJ5lQ|Bj< zQ`9bK`XUn(@erZK^YHQ5w7HH?!|8kfl6<9|zdJX&wBdQD>HYe+qe!@`-?MQQRj13e z?ZEyL!&km|?wmK>#9@$hM7sr5$FI)uvPVEOyD(2x2H%!OW&qjv=I6DEDDf!qVL}q? zxDZ)(Wr4M*hZPUs>%~^cawb|!cTQj9`#~Y>Kp=c^x_Ng}2=|zzF_Mh#kS79xp3ga_ z;TplJxDf6i*~eG)Y{*lgd02gmnVE#RyrUx`pt5A=2Ai4F!fXv60(s?CnbD{iW#R)4l5wtuyu+*}S+4wAdtFIcUl!Y< zfZb&0SH(g2e0=0>OKM}Ba7SFEBX}TFDV)Uq@3&%D@cIK!^lWH(@;5 z(xUc9G(1gAQ=p0))PfqMt+N$TBDL;%7DOik;{kr=z4%whY)Yn=(ZKG9>;6yR1LJ|W z=XoR$f1w@K#vM93S1OK7@!-1nqKwdz#&Q!a97k8N|3uuuP4b^kbMTOVR1C@x|DV1O zLyJp*UnD}A=H0Sb!IaaCP%?NeDXk+4$zglt%e*l0idMC>>m%ufd-Od-M| zBBu*KGOk}@!w%|f)cea0_UC=IExYGJqmUw+)xb*+c_JSEZ^NqozdM)zHUDWiVzHg* zN6rx_9sg@o{?|a7E5$y(5WtGhjhjHZpyV-8?n-1No68x)Zp64%ZP{O%{}#`JvRQ#| zR&s&Y(8IEj-wEa_2X6P0MGt}%Z|T%)^-I`uPeqBv0z}eSdo4vn*Qya8XUqld>J?L? z6+d(=s1CxnKVggv5DU+&wl(Dg=0k3bQyq%hl1D<-5$w!z^Y{2pINT=Tj`(kI!T$Mv zoy<&O2GIj(GE0J(iDJom?KUt*f$-aG5&xRJ@Z})nB=)+!%cYlAGR$eSaX_z&CQZx%+7jt$+>`9C9xVDau{1Pjwn zb=B3};F&2UEtMV>+eN;p)pcpyS#|w%*v3wkW@>9dkfad#z${#9>0e%z+Qa zwQ5WU&Fo>)az2CF#smSe>g-79BUm%S+kcj3t^b}%%D28itY)?YllqTHNXN5_g9l-e z1XhNrB=;rJAj7%O+{R^2#9i+Y!d^@t>t0lq9Btxy03mD6Y3XhB>KaQi@dN@qRrzAf$(IZ4#h# zv6wb7l5Ji?e{!H6?^p$((2vY?sOr~|{r8PnFP*rM`Z3?!_BRz4W# zNAba#7$X$T%n+yE)6CbW+eN*oWSaC#BTzO3#ix^MiTU#ROx|Cs2uTY*V%ASFEo=PVI zstxy@elZBN#)xH25{uq_N!Z51!YF`zX?$r0+VOA~W<-{@3bWkQ^-EK4H)9t>P!!); zR#kTGeg%uMylZpLs7uK|WRsjnCx(L;YRzt!zfq#!3!|je=t6gja1~!(%EuJ28>LbO z(eaWw;0DY)J|d$Fv4J07-$|EVzQN#J{|y0TBY%o5C~fMWXITCNh!5?41N)!l-lID5 z6k@SH^gR&(7IU!`X`Y6-%EZur3)sh#g1DNDN=gPktL3d}f`jPZi>n7&_GE|w)U?K< z_*zk42pS2g0L4)$YMIv{)Vh!(<68OR;!?*H#^XgKt)!kzp*A$IpEpt)GH(2H@(vreUjwGgJ;Lz9Su%xCpKJWHOa4E-oeb4?>vig?ADpB#9X~ArM$?37p zRRhO~Om7^&&C%V-x`pJ6sWPmCnu&aiJ zAi0QisvCx~x><_t2c6i5;`d@V`l9ARSsYq+mL~SKB@Iy5KU1DWXDd~zTcso+GC{2~ zHR5fq?at|+PR>-~QIc&%qu!tTUa}P8G23OJ!t*1z6xn4#q`O}hfIpGF?=ma}_EVQtEj4-hd+Uq$L z@xjfGi(Jt5xF=#)7?#I74eMkb>VHqAR+MwZ!?q_oOTj~g4L$@LzErGcw3*xeFLOp{ ztoP-WB8YJmb{H#6!Q5%rLK6@FfYI-SQbN}mN+{!GCXwTV%fo6JatfPSB9*jMdmJbj zh8zm%^qDLkJbc-^K(<`ypl4Pv?6rq=I*aK(D3h7)msEg>>%6nIq{JB_s4+kD_LP<) zt>P`>TUbMDk5wKQ1#F@)wPv_p%D9Ga*_V~7t)O;eSizQ$g88;%wZlBKtf@I;i`<8l z)0bx7&ZjNYonXj1wS?hp#kF@Hk(}ApMPp-r;DUnMaCap160X%wllC;t7~AUdqzycZ zA6{UCw=`q4MQ+n|Kx~6rFyIDr-qMRJD3L`}KoAEMpNXw$JrgUcVP zJl;6e8EOI~EyQ&Ki+ELIfSt0Hhf)*bZ6U&7N_zRd4KS$4;<;tckvQ50eyPd32$)Dk zFYw_t&B3ep-at}wjT;kD|6!G`RHU%W)bM>!(*+^rI;#rvirY+G%o1 z<<@@$BDtW;I;10qc8crJQ%mm`P?_2W)A@>$7Bc83h363?=R{ULT~?_n)y{>W!$@a- z1CsAkn(>6H(JL5jV--Rx4{3?mP>ERFu{iw8xN+f`c8f-4PuFRAhQx5x*&_}0pfMlQ z?-h=|M2%lXi1AP87>|PK#SYN{Mjp%4%;hj2gKdLEPD{@}DiTg8h=OmJ;H5o6TUrBg`fc&;ekQ@Ms6dN zSY(G=wLCHkA3*%RnZ5Ao=53~#+tsENd{?Uu3Nlo)$XC@@K_Mzqgw!T>I@MaR|*)nNa%mcF?KH! zzGSp(HLSI_(JK@swF8k40B_~C&&S5UrL|WqV<4cVQ{LPbj^77*eBiJgq^C)3v;;EA zW%i?NCs>bb8pMgZQ_-QnP4LP94nzVvz7!bT+W-g)f~{fKQUE51omGqjZ`rvDsRzmM zN^kV@oZ22yqu^SF8-pc`L+*C8>PvN{)s;@`O_dxmvzz2%_+1r)O%M${?A`pqb!kanmZS!fGLpLc#hyI+{as7qUc8)BsSx2eO0zt?3W+~{Hq#9lQ0uh#m zk+Ytly$E8@zgD>HQT337BNV4YmxDMkCMTbBNy#_x!$!!Fn%r1Jbo+V50oJfP%AO$D ztp@fYx&jAa*u%Nv=_-j)U-yr@vC))xq~49UV$mz4Ftr3B=?1%uibX1(_F#UY*WCT+ zZ`v=KdD+8F_ehT3R`vCKUy{dfo*o}X{$5YutWuPXINR`>jXoFY>2xk{gZv8&{iDy{ zR^SgnF=d})WkNZvpp+2TRJ_c;G~6L9rtfo)fnfT#?aF8$STN$pG#!=Qc~FO&mS}h# z*UCv{%q*B@d2%|5*x(Fqd?K?&Z*ruZ27-SXJ!78Z#TPU{R3-EP%dZZc)fXKhN~?w&o4$*vZM;2B z)eL>MT(!?PriU*otFN}y4!@4o2AZo}^cq`@mN+?vy58siEfP$p=G*m}+hScV+8c%? zHt&8`F1FfR_Q-pK;u)0L==-7V5;KcYrCoKg-21kOv;0jF4>Ec=0<-~$bl8`A94i`W zS(@(NiM0Cgxu=aynU4a?)?s;9Eg7A800o-zHuu0K=X6YR1va$Z5z-ZTO8K_2%wm$g zpk2)k6_;m-#U81ZS4XzridO$3+`d9LA-%VON=XlH1RJ2HGnDh~ttTw)m>yjM(gPt9 zOrQv>i_{?qVuu*0J10R(W;Zw}>X2`TSfQOD+6#6hqbf35U8xNU6VFQ2|9=0z>hoAy zEjD{+^z2Yy{d-*1G(wK7K2AEG%yO>ne)M6j0HQ2)_MNrFo7!oN{KpXF(TMLZkn`hr)&o%( zSEP>f%haVkivu2MxF<63sgU+Bia$fhs09>%L_IpV=3Sj|{1E%Y|1D#5lRDfHC;#nE zY%hqLvQeZ&%w>}3PYrmu$3SYt9Sp{_QPl87WX!LLbdKCrnN<5tg;p=%0Oa?M3==z- zGG(Vjx^j#(!y9(jG5Op;g17h^&Mv)td_2PW2m#8%B89#k3Ge-w$5EMqAJo~qR{oMe z2L_!pr{}uGEpsTnoa7zr*bTbD_YK3?2_Y4JZh%@kgBK)=3Y2Zy_*$VvNr*4hHCewT z#B(zQ?~g4A8cx=thkW4h06r2yaaTnT^ef{{4wo8*&?qJcMRBtunRrSL(Xu5~eXVG5 z3w%AopCX2Z|%JGo&{! zrg~-SjF%I;cdrXlhDy@2tLpD-6h^k`Ph{TdhFBFav-ZsbmK(2_|kCTw~)xx)!R zWW<%Jg^tOW|IH#9%O?aj)6Sae--1`uE5|jQp&vLngZpk6c4v^1y({+1 zodoGPeE+yR3}qk5Pb_~i!|Aw85W$ZNL9bz35-}vDqpCiSIO&KHHt+`i_Yldbkz4a> z0t*YDwL)YeZB*(pSLYJ^xvzfKu8x;r#P$laL1z8JB9hL8yNB>Hy@Pvc|d#zoXa@0Pn zgu@jv-wlkA&xG5iAMa$7acXBsVX%Dx=%f<+_oR zNgZZKNKf7=8olk&r@ts_!xWLEl3D3N%?Z=SC<=+W=pz(ZOPS!@$bTM5W>%d3^Eulc zPaHeelGD5!tZWvl8KZfH@H*I&3-jKxn~1GZkUd+NEC?QLb;VLaV@WPPZSNXnDQ86r z)6nTDF>1LMsu+8z*Y+cJ;`H!As3-RI(lL0Qq*jgk>X5;4Bz-vbLFf*gQU?hng;Arb%cH{E_S z1!aCH$7qC8up8d^N;o`!RIxqjXZLlTERv0OYGeqseK;eZkg|A3Bq( z6Y;fS&$oQD7NF|IfE7^5Y2TjM8qJ5=g#(MH`FN?flBGG&Q=+rl0AKv|xxbWlKxUoR z5=!9XLoZ~nunL0A7UbO4p5D&^-5V1rd@-Lt{O$oHjXzaY;G-y&P%{3by_LK~`HIk2zCHcQzzv)S`6 z_^yp16?%aS99)RvQ~Bhwj9HaI{9#l3Yxp5?#NdpKTrT5CK2E&LvjUjy{D%%7+!0YK@`8x=7TiBY_B5Hm^u*x`a5CU`aApN)y;XE^A!Tp7-n&Q_6sc5 z&FYlN1krUy?cCaNgaoemmi2-<2gBelfz35GxW<(`{wD!V3NglV8?zk1Q$K zk*;qS$;0ATnAkM2FPDl)`rbPRAk%;G2MyRR&8j)tT@5NT0*Y@z?lsL9BH6u=TxrY?zQ%>sP$C9Qgo z2C~Dmryc|m)hWJPP8?w;vC?+68pjeBbCnh4SZ)OQO@7!;e*(LWPT}{>-vm-h+H(ik zAXE2FS?-0Wm#iQ5%v=Rw-c;bmkP2iXcXE2${f3t)@(Z!W7eq<_VZb#q26d}DjIi@= zx2Xtn@m9`ad?86W?U?TH&}|p+9R`NwVq+ZH;0w zdmN&N)vQ|O)&22ds!m0&G+sK#H(T&fHSqb&fip`-;I4oJi5c=s@7$2(DF-paMY_}U z$?IHUFD>w6rE^=udgwRi&+@i)J@N{Lbz^--S~NB)3zNg)8FN?Tuc%)%G{0P1*+>gt z_@7#!vboJHW-^fsWo9O7XabuM8+uOUIKL!k-W)mKi_E>AQbGJ+pJg7EKN&rgfaN`& zf8le*U>2rr(Mh4U`mSgZ#`kR_g)cPsN&Mhc7p-FBrY|bryfZjPam!?TH|W=J5^9fA zld~63xU9uN2o$U*O)E3)7x_)(CbOowWX!AM2R2vmWWb-P3&E;>9X8 z+WAeDuxPO1Z`zpf7w||9K>8%{b@o6RJlq~3S`Y@!tRCIXGB3DvgeryKG)1eu&eRU#TK%0`Cm?I45iXy!?w3h{IM{qQyYU;M!9^D zzV@vcoHB@u%5yLS;vqPFao(EP7J?ijwPsMvD{P*mhvKpHr} zUWSlu9ORQjfvps}ea%gRaiC>mSRlZzO`0{p%x`u4Fty!z1$A;vTdxp`zsIo7_U&KT zO$s)7M-fQs)&3Y1Uu=!=FX@LWPhYJS%Qd z85Qc>8ln7qxrAdogV8Gq@a%dWZeMP*>*RW|-imJYG2AX{t4*MIKL&A%l>zRGV?9+y zisHtCl>xTi$bcETZ3Wut=rg6oMHfFlS?o0E%k2=1BW)}9XpS26xV|cvesL z3AS>!+`kWgw~BU&yiJ%C8XnJnvsD1F4UGwXxY8DzA{Ief4v|PSYrvcK2}umx;ISF9 z@Ul1?Fv#0(?w3F9dbh`g)2|>|e($Aty`j->aq|KT;T(Ug#*4IPusW;3H_{ z5V)hn9)tcy#pn?jJB%iGl3$==0k#=t9$eyWS0$x>9A{NFUtzmAy^srBN#ciX#@jwi zfIt#Ol{nU#L}0pOYo1Gj-~^Dz{p)9OaLX@rR;i7@QKUAL+eJ$w8nJ&NsNhQ%m-I?r zV=W%i)JH7V^>7;MmVUcqey9TW;$ue=7rMgETwY$)4e8VK;@j2?b50CnrYVxvIThd7 zK%>hX&70Gb-`Wve0y3t|>E5K(STGz!4A*R6EQZKtHI_FS7dCupop)Go6C3!pX9U=d z5_jnjb^<7h8$K8iGSgmGdmG1ECU@8~PCgIZmV4T6(h)vnDIi+qq-Wtinn(P&-6dy( zN6&NU9nvr1z3LgRn>`-3_vFw3bWl)gfGb}+EC9E5r>~7TMhCcikeAkkw%=~nK*P?e z3!z`iXEZ+;<g z(deX?Z$@W-#1DuQCRlpdk6J0Hwq@yrpVJq=uiMLZUnXNdEU@_Y@cN-ow4GFlgp?fE zV}&BpSX|cWn`rRgk$|6W=#n$&Jk`)3iJTTZwC$GTb&>w)z4j!WLcOJBTbpFEIuE#L zCY>4*w4L0wXcURtJZUw=c+%7q3bIdtwwfyu+0f?2BIRWjq3gge`}F8Qpwe zHC#rl=F;me-auu-Wq?1BP_#PI=MOhKjlS#PwR*jxPEdKG5{>|x2-04;!gRQ$NAKFE*&!#uN7|~v+!SoUR!~t9XzR3s5A0;?aB@EJ{KBv%|No8_PCJ@8df4MthF^YDpllJ=0g= zrRCGlmbt{K4ZaeLSi~y8O#brmw#`;#*TB^Sb$(kS!ky9SqTnUiuS%}4K9^61Vv)O0 zK?A*L)!L=h6)kf*Um9zKV_lx3uYo}#Ijl0%scc09i{`aPe>0}f2B4lTT;zTn>>r1Q zzreeWrP!{ZmM3>lPPe7SG><@=H}J9d6YYW0K5miQCPAvN7&9p!8)kSeZyVr8nzf#V zhw|@bXolOL?GRc}RHc%_`}+F+;-Y-fhxfjGe)LC1QWx3>tt@W{y8 zv!Q5Tyz!zFbSb9*{yCDBIvJu@Snm-+fPc3%+o?Q|G{}Z5 znO}u!1GTeb(&t2rZPM(jCETM#$%xeWiBpv78MNoYMP(^wF1Tee#$H}1Zl!c`9t+6N z3R$v|uGb%cNRFL@Nzs8I$4)&AzZN9ITl=3BWC{m@ia8fmbwy9tRRxBzJgrS&6J^$v zWlfIy^UtRHp^RwtB^_@tzUyVmN~7mPm*L@x;0~jFp2iJA4rG6k*yZM6q_Anv%V`T z=qBQ3FLz<|iQ+fzINs#qbVhXVp5tznSl5Ht>#i7rwsX?>8fdk=sE6BUUF^8C9XcX2 zi91NgI82M*sy$1#(=T>Ay99fo_ujWfeUmFtN^*ojIvusXLQ@v^@ce3%iAG)($M6C} zPbh?g3xDaZr28|<(X&rBC%VgZd`TRyv3Zf}BsJVGqF`I5yugF|-|r*bY88hKozn22 z_wOd0Fq1!|nGzGQ0kWx%jQ4T(86|{pgEA+t8+NLMOv`atGL}CSU>&owq!P7>WVu8= zcziOcJo?bEQ95iz=WJoi^{#}0=PXKP1qEEbaOfEGkWZ(!^UR}|)QR9O0>L0>gBVsh> z93~R#BOQcP(CBM0-CbD__xdVQ!WjJbLLvROoWpVZXMDwn-o|VkPWY_6HNz^t{_Cwc zG7LK@wbLz8*1O^U=lHTS8QmKG0|1!cPn~rnq|=0ydFgTdH_tb>_IolKYVI(4y|Z=* z6#fJoMd77rFi6M(KCpA7ZeOvKA51`xis;@duakoF9BG;GSuILH3GZfx#`)E4YfOu2 zWUY)*V%5~2WYK*Iw@0OZFN&xf>l?6(f)~uOl7Q^_rU6LFY%@w8Jw8?=Y;-UrZAZ2ky?!b7(^=>bXGh9uO)uzd zCh~%^47cw&M)l)%RU}=Ibh8K!N|(h;;)8cSr&MlPQgJK+JC-1TD#h_!v)|S%fLLDx z)4o4|M`je<`lnr>eG}-y-{Ad|tYsVn6R01UWik64Ldif0PqCPP3SSx77=+_SiLMN{lh`2^ zTeWFns;@|w4mb_M6V@P)XAm~x9N??+Bhk*(%F~a8!Dj9OtUni{a?Q z1497I?#0el3yp7Vg0&)36Fh>Y(AhgWDv*=*p0g#rJt1_Zzt0q}ra+!#)5u$pqrLSO zdp>3LwCmrM6uoWnIHMe#vRyewUu+6}mHZX>6>D$DZO$a@^DsGkwGxnm0N8no-WmO<| z!PC@(vznUV2Jbla-r<+^(!a=*w8t^0QQ1D>*bT5}h$7zXn_7Le#i`^9LmnWLP=!AQ zb~OGRrB>=+YN?SZ_oK%$Vy}hoZ$Cj>QMj?+mnRvv$IJT#y;JI*$#%%6_~d-D7p&YY z0FCokIL2V_wXn)rd=EGuwU;c^Dm|t(1i8cAZr7(c%PxPU_!)%=qf9_>BQuo4e^H}-Z_qDW@YCJ|_abyW7p}ShbbN6zQuG35@AB#Jt0zpFXqO7Z&i293 zpdwLrmJWQgRgDAz+h?`p3f%q6Od6MtaM(9(Ce99+X<7(H`qAgV97o#v43gJ@_bDru zwJENGht9zylS+OPq?@ztTn%Aj5RjcpaC}TzOcxOdKrJK_|IjCz_ zZrH|l%!r^wnsR}Dh5645JH%y%MOVA?ReZ}xVqV+4H~xbwIeY1_T3WZ4J{nvTCb+R9 zdvn2BPMDd*R5yktv1qoOar;)`K6g|!yD;%_nZaBGpR%;amt@hs<4-+Etn^u-kx6se z>w@MV5P4^$(6=82Esz7%1)O<>yh|qt54rl?bjgB4ui5XXM}YS7hC?VlCwwP8VB_gm zWDk{4%hTlX?&Jx&^OI{SR|Jk~4Au-2|9rcnZlSVapPzVRIdY>}ZoN_J0nYuWr-st( zBEZ>!_vwi2`Y#(YgtqBS4$_Q2-=cks9KuX1V3zCPVB>q@@IwX zt&Xd)ccPs!q@OsY7|+zUbq#m#3J*Qr!M*lB_oaa497K=edfWJP{u7GS^R`5Pa;^lO z9YG3edX6M|v8|YtY>U*x-&@>l-P#B<#(N*ADSJxzre*zpjn~?Yc#Z)0i$602gcF;Bz)m^*JLBf$Fe% zAKjB#K|8-e23p5GNMAYjCt@QnUtqAawUO(sF($L^1P;ouIP0||UAHyqGFp*sWE?ZE zqManPlgCSC8DS28-0WxO61*6~JIxH3SCQC6Z|TXZL6lAac<<0ti?f&HS@E*(SZUdi zDDzqPdB@J%WEUp~+_+jBdvy%b7&q*qid9zU_0OLv+%`YkKEAzORPR(qs3KW^ zPX0cXCGL#rF%6!mZY(^N`6Q_&HDpKG-l|MY$SIR2j6KS0&S_63SCG-qzF~>B_rruX zn-q>ha92dree#Ivz9advjT2tm=V!H2@o3tY=npS=*sE!%B?yEIUo$S6aaT_fYMkHy z@pHPK(Hmap?}V+u$R|Dlt5ENC*3J~<>xDgqR#`tYMW^4hq(#xrBmx&!O3Ua~4*FbZ zPCLMhmzISa<-r^~c)M;KqNq>mMqZPDLUsc^nrtUOu~}!&A8}!#MW?<_8WapV?VPN| z>#oKyQ#YhaMXBx;SV~Q?YQv3Vmn+$LC&KlE5L0g8E0fVk69a=>TPqo#D=gCN;~xRt z@3AN5iiQICl{s172So1HOE)sx9f~hLG+(2dCw>g0?t29|d1`ro<9(}|Tp@5N{wvkt zq&(79Fk_B(a}h#VP+0D46{EXIV||~Im$s(2_uJBHT%GK8o(-SsR!X5%wxZV|%4WDj z{!~zQ5SBB)xuN4J-UyV4@5uAE5y;6Vyfrsi(yr+@(gBkOB*l0?mgzKdzcc)`PIg=7 zjd6u9+&bkSI$0i>(?e0=+8HXL{=d1jdIa?yjT-i9Y~6Dn8ocda-xOTRnZG05s6fgj zmDQFGqFs%DN;+*b!4|9~2TP#1*VAg_QidJ@Wyai9nKCptX{f-TV@dO%yJt2TfECPW z+|qqrJ(P>BWCj2oxd*H`W!y6zI78ECS-IA zW+BL+e~Fq?bKEdgYRU3sTww!MhS7*Jv5juR{pd!Q-v-coL7STfz zcoxwy;M6mobSkZ?cxbfW@v!I0D%bKxG{6{sP0ccBc|FZMdYp}N&SNcJ_TX-um2Dg8a4D5vMO1{6Ptbz0~#tdl1i-R*JA}+F(qEp1 zjc7^F!RAHu_xM;iA)rfhfjzHx*D~@u%3JT&K4h@C#*jL=`1kwp4g;r!XTz>Ogg=Lf za^=)o6zlgq_J>i7r#DR~HWuD;52>#UI0F&FKwy)rQ^A4gl1~yf!y8V6Cr@orarL z{Lq>CC@OZCEQovi64RFPus~ju#tRV8y{dOnWR!H3jt@BDg5G^9KcuZ3v%9fgEm_C> zEHSf@px38hpwt+&UfIh%g3EZ%wqSNj+0>|G@oOKDw?am8Y)F>Fn@; z(b>K&A%~ve)*1A8OYL7JpnXji-T@yx{Jtbvf7QzkqCKlS(~MDfy|ioYfhTFU+xi@lky zYd2-K@hH>UQOq!m1J%$YtZ3ik9qr?&Lg42!ga(jKxwR1#EQ_q$H9xQE! zauw}l|D4({eIyd~CGdldaO;<3dTI#3o=g%)Yfv=Hh3wiJaqm!N*zMF9lTM06?(!a5 zz*q#)g-V_OpBk<+8m_JjM~?`iMi7JOy?3Jr5eCtFi0DKK(OdK~N%T5;XY^j8$LJ+` z7Y5OdHW=UJUF-XI*S&k4yYD%>JbRz#T-Joh%>it^ug_j~U~O~#NUW<+Vv>o~MhTnp zXh|hWV@2FtyT+#h>fp?-Xrz@xc?0z1k9yCjyGHIULx6 zNo~ts0E96lxVndOB?Tw|I;4#A=(DUGu(UTPBm853>0Xmr(Q{$B1 zMLvg8JHRqXKd?DJye*d)V$&@arGFFPrmn;j^XUe+rDE>D)FNGK_aJe3ZlL7dRr2Vg zIr82yGkMR4Fo{W?wytPPpvzs>k+c!ve^_44wx(W_q5avJB#Uy zt1WfxnIww+N6x_-eEC|v!PA(eF~(KOR8>8Yk>c=N=vkbCVPhKKY!GsL=<#=zqPZeR z6q3f}>C^yCMCxio{x&PDcoy4E7D@lMfDfb5U_Q3B|7%pi*UK5tQz*z(c2LHY6wPQ z#6&COOYa%3{w=$#it102f$%V=h@%~w@L{(<*#iuC{h!1!`!6zXSZsUmRQM)ONAkpNUZOkMqN6-lU=kcG4a&f5LHK@Pce{M1sEn_I_v;x^ zFb`;)V=Q;pLN!oXL9nJ>H5snK)g)MIP8kEf{`&7w8t z_rUw%&h+iOl%j&GcRB60F8(pMYtfCMNqpu?O&&+Hp-6C~fw!|>z?LT$^T!u-NImM>T`>EM&8S(V}$7YYXp~B5?DjLFlac{U+47b%rRh z#V0bu``rEm&I?1C)*0fZ!cP7uJ9EOt<4cjZ?c)`=*0r2&2W{s`wiUYip+7NrT$qZA z-%$G@WxF?}iYODl50C4Z%yJR97g5f(JDhuUd`@{D#W|L{6TTg3Ne&RbNRof;@ld{U zr~bCmscD4$sxzY6%#Cg3_WJa-cfIkwdYVywQGzE#gf%mR$~VZ#rf*Ic6`?USlu1G9 z(>k((uHd2VZgxi3D5pv{<9b-qud60K%$hm!O-mnM=$fUT$~F;|-72L!LxQoIE?-9< zleYI$42}oyo8+#5d?}d-7-e0IagpSc0fz!zy5GO5NGz3|=zg0$do{DcC?$glCfc;Q z<@gozhpk%MU4%QO>33!`RD65>3acdsBe7i)JK;HHc~B+=J^1^*Vl6i((MVz$yQ}2m zc?T7`5lMQejt=6CPP`xx7wM^|w23ROrBXgGTcKOOBXVK}WJe$}H3UZ>iM{_q5EmLg*s;zekTliesm3cih9S4e zC7`M-yWRUO#b;gZd|^3nFc9_rmxEjr{fp}AZBhfY6u>~l}FmbYU-+0E;;>xKWtjl z-d+!;A^5Re?}wB2rJQ(GrIzD5Yn9E2%-^XB)z;ObDYk7bmX;HWM6-QiQk%jo$jd)Oby34^i7>@3Yu3}T zs7|U@PBn(#kqg@&N6W+0-I7W9+^X25GqbGQN5;++8AmC94wR@P4lhJQLXt#6q-ZX) zl^I$j7cD}%wHD?sc&x(e{4k8}tzH zJ4^=0?yS~w-Pk>*5V|^+WgFv^cNL+AgTHx|McFE&4?m&zJuKjcSU zXJ5b^dvO=HLNtp*Mqfs%l{sIRmIzOo0tfQOFFj0DD z@o+Vh@|FO{=4Sw0Vf{K~?dF0G`B|c>xk%JQUS3SD<9@@g(c&$|$^$AT@B$ooGt6)n z8G&oT!xOMNU#ulyO{S=Iw)5=?TMgEA4+~c*yQE5hG>3G0@#M`72ZD{g=d~}P2Tj%_ z$!i3o;xX7T+k|r>lf&<0`A_$r*U{99f;4JrDZ7N3tNmpbpS^$H<2PZe24p<+6O_n? zeH!53`0Hd2;sv)JT5VArE@eos7~I%Kojm(8e0=o^j2VhL^w`l98tdF`?J&ZTF(FPjR_m7s;rMIM%D$F^oTWx602t+8`e%yX zX6{4+`8(37CyFC=6(E~IcDq>N8>G8ii>2P?F1M}oV@wD`IOY!xz3DP=H(7KKJ(Pmt zgET|#(bVo$p|5a#cQV^X`!niQY(r)0!G+KP_IpD#Uj&Xd6Z+Z|nA{BCHZA%j1e zf3$%xf}JQe5_rmBB*%gL>M5`=x769`BF%YaxXqR_rh9=)IQGJo6N#h82I6uK1rL4w zrV#>e=d26?=eKiz#)ga*X*eJc;=QF#YN&Khx`P4$yN`fW-v!T0jo04|lm&sje@dMM z!=K*gcC)6RuUkD20*N1w;s%;64;}u@ezf1H{{c^(5MC{8rF(#%s2$;3GkBR~nre|h zH9Z8YT1gIQIN*r)7CTV2PFccwU3gIIX|DN_3?j>$6l>tQT~lVQAMn%A3y{^oAYO+6g9Jut>WN$7Q42mWE(8#DKd^JRzntJ=C85fy(y`N4bCFP06m8V1CsMjF5uCkAWnWC1j% zW)k>_sp~8T{_|WN3HvmV_egIp@e9PQ$95zzW2`Ryrr@vLrL3MD1np7&(O4fp0-x_8 z>s=xPIu7TiuEC~&%z8!g4lV2rQe^dK_1(~4$hUJ=u97Jv-6+(CsdV9M%D9A(d7`$Tajst zep-MT{Kgr$3yPW#O(|*n`P1;gP;Tpa;wL3g1s!@je1wvAcxwAag^;lS=E~Ca#NcMr zGr!8>ge24TJR?tvK`Gj63ym&M zMB>NtYtZC&d)4wfO;d4;L0#v<8UH7*;bL%~i|C=V>&%gef(DK+4DIZ`ji%fC{5Z6H z!L41k>ij9~QPN5k)R*LVcSB_U>tQ4lvC?X)cei;2TNOBr`Ug+e4Cpr-aq<%Ln(8wM z*GxSwzF&)D1DHs=2s2?9A~qwH><@q=BwKNAmuXhI-~q zE6o{&*KwcG3FNW+;$14{`xGesb>GRUm;I30*-oWx?{}Q@QUZT$uPwh|$wG87Vb+o7 zpF!2MBq2w)l>na)7NiQU6#Ev>Ea}Qy+jqB~9C$B;`F=Xsm=3+9z=m&^6;3kRhgLnV zjMfdNt{yD@vvt&7oU;0q$zg!1QTWqhsmMGmAHg@TuxB_oc#xT4ys9%SNOIYGZ9lsG z!N(n~Xgi!blm}?)UYR3$Y~3s<=sTcF-mCsqw!HNxvWUNppylTzH^McnXb2se3q-Me zNZUqL8d-J>bI>-5l?Rp?8Q|RA`W(&R9vN|05)L%WrJ}?eQG{^Xes=C8wVAYG@3oO4e1r~Z2sxL$)pLH_ z&j`85nQyn`HD&f$MN>=}LBR3a0dPFf#vNcH_sPt~30*S7g-%=#GSY#o{33i$O{-!9 zi2SM()?zbCk`s}RWdi9YW@twWCjK6W^i=YBf{<&o0j?XU3$w6=UTy}Z^;S6uW!^80 z^7vV$of$q=m?@OVa{3^eaKRl#yj*Dz=6~?R@lSQYNZ_B<=|YP@u3p!4t<1?c7yKX8CZkF^#s9xV_PH z0lT%H2vBEc5g`Lnm|4S56D!^y#SaCq8xc zlRQ4vo{6UX)vpDX-_$h*2r9(@cNkbQU6t4&Y#Y@gU$E}EV*McxonPNl z{8qcJ%Mo93H7naNsVmA*La-+UOCZ1Maq|Oc+yKf~aG*bv%|~rYuBQxkPtN{R@fV>f z;4jfp+F%tij@Z17-v6OS)b9DR;+Ibd?`*!N{BdnUIB$}E)rZILljH4ePl{g6?U-xT zv$^Dh;ca+k=t1L?HK)#NCg*c<(QdMp3zmGsJ7*hSP~}xqLrcQ}Wz@_1sKPnh#%7WS zzpLF^iwAP5j5VbJ1Co7}J~Y4`vmDGETr80DnI}G^z7p(XtNA4WG$G^n^E#=W(`wdx zSqfjiuc5sCHAj{yWckj}U8^J=(K{!w?$V3zfZDWhn?W`PR~KKIPFDdTqf zdp%yx#5?(=cED5VI*T*$<~GZ# z9Qu;1DX(UC2WsUn6wXTP*#c*z2IDfaSS(>$dGx1W#Qgz!)(e-W3_?iP-pmQWPeYL* z++gqa=LZAG=>%VQ`dj@kcVl#YSN22xbmi&G8L-XXF9_GU31dm!O?EL|3!tw)xQL>nW8ah;9KLez`$XtJLK5d zgUSe!-XZ-UGEfI2tPdG5kJmAyArR@Pf`3s^!ZN!EcdPv29b5E1dzkJht5lN}42Wf} zse6>!`3~bB=*0p&-(J?<5)#svWqlnw}4-7fqjrp<|!wQP=clM-=o-e_8!sqEJGx_$N5qnmr#!sK-nG`F% zs~hFA;pqQoD_<+>u7_S|3IZJ3w{Q{cmb#);>dd;C!N57&n+ZcY?hcF?8D+qZ>h1Km z7_Bv!zn`rFAj1W3WoJ^p**9;AU3VR|Gs_{?+91|OklK2Om za~-oF(b0AH9W7toh95r9(#zdCgR;Bz81X2TjC~$78d)uLQU?^o7I+#ra5RaD;{&sp z0$e&C17{^l?Dmrs7dO$f2yyCalM}7n%i@R9i4F*g>EhY~|APS}>O!A{#Al>9r` znG4N#rVtk)EbSQYPz7PCLC9qWb=dSp)@y^E5e-o;;uGBQ-tBV^7SKzIsf_8}(hi1? z6z-liWu^WBKSzponENn}er7jl-+}zY{N$e7O@;6ntvqHDvo-j!WQ04=5UWGGnlm{(aob z>#w-!=k?oPsR?x&1@zE}4(p@kxY9sNag7ao#6#&m?85fdTTF<@kPeG6{|s50W1>sV z%7@ziHu1aV;SJ@dmYZ1kcEa4Kv(!N>z~h{ATc=*Jt081V_m@SEt>v?zDC_6uOsC@I zF@2n3cfMPD6$iC^!(qR^=#oq!^&XU_Lx6v3U*Dr~(99;Qg4v8x?W9BWw>EU<_17*f zNHOJcFyNU z=69Foo%i?0s7>>(n=2R8<}I}3{9NY2W$j+sXTS3{Bvj_sO|k%xijoh!x{2r-MO8z) zJ_r8#o8`5Y+7;L#Gm~G|jXEq|^Se6a&t<3ug*k={Ku{-3Rsy}());4S^h{;Qpbp`I zs2$Pqlr7GQySu^Tz`LERJ@V|OBIv`}F!V|~5X+!OueyegTRh%$sc$?S2Hw~E-}81pnlMNGhQdDF zw7HT#46rwWkD!qob%6?vQF$kdjG_bgZGD^ggUvF9kc5*!rw`} z#Qu=vPCr3(vz;ysr0fAZq&_@4yo-Qz-j2cnsJ)D}2L#Ti;Ft<21v!@+Lp3ht?l{`X zCm|N~gR0naLOI>3CwmClRHA23ML4=lA9Z8nEWYFprY{oI?=*QN$8<5y6HG5aPCl7H zzuV0g=!uWbj`6*V20N&=w%&C*f+31+j);n?}*&=$J`k;`WU3Z8G6H_q0(*<3lu3+Oe=E zahd^GqG~+b;bi>PR9;yc*y6T|vCv3giqJu%>meM2z zMs$(aCOf6iu6=f;pOpj}$^Hx{_VRB?#M%cOA??|*W@MHxeruzO1DpSr-vXB}?C`G; z{{(`32>%ISoKvH3{>I!C_A|g~hgh$yx}XgozSi&=&kA{jJ-1W(w`Isv8m=NPfx0;@ z(Nr2X?iyQ`-;z0muEEZX97Oo)JL77tj$ywpz-lkNdy8*7O<=!hfGbzq+OJnn_Bwe} z1kS=iSN`7c;#(k3dzulDyQw=wJA>f3C>edb<;2te#$zpo;J7#$HEG!{NKw@Sm87-q+l<%Jbtoj^_ zo%0qlTjWKm`Fil>;S$152X%nHh8uX#{P*O@lY(vG;UG=xl=-_$@2{lNgYoQ6BslQy zt`{13eRkGu_>e{@fvzuzUZVzeinb$@QN^e%o3{xE-NPQ7k%9(YOd78z}Re=_O< zQFYoQ13lcFn_s3ipKh - - - - - mapreduce.job.queuenameTheQueue - - - mapreduce.job.nameMyMRJob - - - maproduce.uninteresting.propertyabcdef - - mapreduce.map.java.opts-server -Xmx640m -Djava.net.preferIPv4Stack=true - mapreduce.reduce.java.opts-server -Xmx650m -Djava.net.preferIPv4Stack=true - diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml deleted file mode 100644 index 75d0f86cfa9..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - mapred.job.queue.nameTheQueue - - - mapred.job.nameMyMRJob - - - maproduce.uninteresting.propertyabcdef - - mapred.child.java.opts-server -Xmx640m -Djava.net.preferIPv4Stack=true - diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz deleted file mode 100644 index 2662ffb54f41588f5b336ff0548010bfd40e4f1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29907 zcmZtrQS}E7WM@uq<7h(f=3;DWYwkjC=V<8)bgBRDx*?JD^{V$SIa|F! z(>Yyu9NynY&P_g{QYcniSl(S+NRL6^&w?7gn3Q6=v-Ekx2}F*JmERBOpH0)0QmGfO zzk8Y8GVHauZ9SRse17uen5lIrYxHV(8jbUyxD!8_I5P8^YfoT=Y#j4ie#f)d-7OO>txS`m%t3Ix}(~*OB}f(+_oc=#4^}cfMBJ6I$j%@GV?=Kv@qPZ zi-TCX%f9HBckw!=5Oxg0F)o^u2=K3zOo%4Bd^)pX1Dvfz1I ztZ-ruj#*ghf18{7e|5({MurM*XmKihaQ61;jojrm1@d)mu8b0X<~nxA3c@+mS_mfY z5T47fc;OE<3SSWNZY6gYiNb4kE_s&*uD!jUUTt?yd7LS4j5#85L`aKb$5#$el?K)rjbHy_IGRRT>J96Gk){&|NI(}B=uR&9Kiv^ zR=ueDxij|=z)4z;<<`#RjXcDD?LWSa_78jcC4W~xG$bp|pwJ8Nn0~i)dM<8Ve739! z@SPLbT^*1;w<;jE2C_J&Cka)V+?2OA?+C8)2uSw>zE`)rJv+Ktzsb0-q#dg~D+Bqi zukk3$F5yc9TYPjWd&wws&j@7z5AOdYecldZKeru6-wrq4d37Vp-mu=Hwmv|A5hr5( ze$yO%_vQC*I23~n&92FWxV6ymLF<%VJ%98-Nbvm>Ghq_630KNXF)SfskxN+JTK_UH z+sLchvSZ=;W(7Rlq=@U{2d5l8F1`3UJv#H{)~I3t+@;Z|g0;(#c`uy&Ba<#V<5qb< z3ejG+69gP6!Vgr#FwMd0dNl zFUVlt@-_}OehVy2VB+*PJNP;K60~|yuH$~N13q5=J!4Xi`o6#O-ZZs161wl?cKJEG zuy%FCFzV?0t0%{vO<*1#34YHc_UIZ>N%svpde%;CLAx!59Ci;go>AZQ)+!1-vyk17 zg%(LaZZfXbec@FE!oT5!#eVZB61k9Ir4eTwD%CtFJ?FRtVnk_Vpq$gr!!#D4AIs6g z6`~WPTusfxkbZfEZ>?cZY^;qbAX=mbQNdMh;tyUwU^#Xmg>Z=*7Qv;t#xxwGNvK;5 zQX4{p;z}6DMQBL?1G8Y8nS)W!)6!Ekos)wo8_0kUh|`$W&Fl0d6&g$Jf5E|J(k7cY z8;6lPQ1&PC{T0KDJEJl414b$UjxwVLqB&6nAwl~)RZBDU(2Ip8rhp7mIJSU@6rFYu zq!f~Z0|tK&NR1xn1sNuT=skQr37S=yK}E8f$xjIh{;NfMfMR=qjh7jEENI==^w=2r4vuXB2G($g%d?)2a-W@KGq&yp# z(isUB9u8`$Onw6_0u#4HGls$@C<|9-MlA(p6>{fBHp1-WpcI}klRMPqOfkBj|E)2S z2^~vsG^<&v#092JtssJ?h$S+Bs;^S5iNr=1kE;SQAwZyVg~9Titn8pb{h~x$9QCgl zM&hUv1Y*RCTNcZhO_@qi6DgMIl>ASynF#~1L{-{)5Ig`3*7zo)gi-lZ?Mp$M^ z`1ns;1YQxGv_mR#64((A9Pu@6t%z|1AJP2=L6q$9Z!mPw$^-}8(xI_ z*lD{zPHq^JJWjS8zdAe1ka_&H->?y_GZq*9)oxl75w@@KuOAW>d`HW=#gPMj{XPS@ zVssBS)~2qi+1qQ6Aafwbn$In_cX%hGPg$EHiZ)x@$=F_>{t*a2wRx^2Z9jh0d1~H$ zP0%gxn3Ke5y#Td{XFZkXKD(*Rpb?3R8=3;I}rfy0EN) z@WT*6S~&d`PKWQBY-MoRb&_6K(E{)y;eQ#r_z?gIN-;>6f5W{jZf7jA;#2B=RMD` z5dcV6?VyoxmF%;~;)H`)=prgADD%dpT;Re!(Flhxl1?iA`ZO>0YkpDwlBHaK|&Dxn$lD{(pttrK9d0a z8r$)W6bE@xG=~|g%l5A+)RmTXbTUj$S>Xog!k`Lsv!Y^mO{rV9XhUiEDSY5;Wlmxc z*2dsv+5THL_NI}SEgLKM3JMvPB8fFXUBl8-(875PS=6Kh&sdr905 zu5~0j0CSqdeWMv2ke&sshfSMFRUwWim>m!MSVH}2eNYU)I8LQ?^-j2r3w>IXe06Fdy ztwoq?dC9b1^ch~SmrJG{Sd57~<`iK;s?5PA(iNW4(AeA8xRV#DMpfe%TgY`|7Zt@8 z26=0)?#MRkK?`9z@UPqvI?badR*l~a|EVZIqwPf-m_`2YQp0CndC1O~di|17UE%8l z4wRPquLVwpU5d}_rk?tM9X-{CxlZzxY6phWxt1G|4I5vb^m-%^gb^tMaal-Hk@qG6 zBas>#UELoR+bHA=VWX*uI6K!lT5htf&l88m;#>Ib?W_2MP8%4J!#yoj^ys#P^p)SO z19L&Ox)Ql=;+PuMp}Rt`B+9V+?Cf^zff|onVlvuuwPm5c${%N$7@y>HSL>E-At%&MZ&n-dd)2eXT^=%H0O`*TdD%UcChwwF2bu;o>1kSy_CKj3eH zYyM7D}YFJNo#4+Gc`ox^Fz9p`t*GFg3>(@6Q6u~X??&bgM4>N zL^P3=#SEaW#lMQ!Q)iq#MtQimmD=*h8RFC-rczu@-*=*{ZMuDwHVoPZo+~)lpY*LZ z@H$a6eZL+0$+N7w(veQ=Z^H98CC;bjZB#RHZ>KxMA0d=8 zF6Z0|9g7ZIRq|qiJqVMCZo7@I%p5`-b?$q)d{W~yGkAXwKoT!et#hS92FHc^yqK?9 z2B&PMV`g1U=VE`a^7HU7V|YJzw<8-WjI?s%ASAK-RxikeUr#&NQh9D4U6U;&wh{{H z4L=h|2zrlcNsRL?nPRNH_-Cp_Mwe0Z@~snX|gkzFu}wVnu8lNv=FES-jmcrP;`(V%^YY!B957qeLA>iF6D`DPzO@ zbQ5>tXyy60kK=ayM$af7!J{J6+;3h_KPeBjWV4GjxmnPnsi?W^{>X);>r`(vJ#@{C zPo}L{&{Jt_q+b~6NmgISFnKm|`sAYOWG2u4BmdjZe_nzEFR=;p+^P&Uae5SUaPG+B zy{Wac)c@X%14|$7&q$G1BR@`SpjYE>JYz3!UTm_BUtQiU&aBuwL&Rrk{IeGaSqDuy-(MjzI$;?%1P&4PS{mPY)IxT?y<)kau>o zZ;{7;x{C%=w$ohb6<^&5B_i@Ay$3~>&R%nfX&S8$s)W*^` z+16Bw-N_zk?>959ky%ml!SysVL(9PtY^9=yj}ieXSv(X)Vmoid)*18Ny9m^{Nm$%G zkH0zwDiVdhfk17whv|n5kyhAlun8CrG+HKEDGDKLZ01PgXzt*DW4>RKGcx}=T*S$z z(DTEsk|3dMlnloSJuA!DfZ6DRuOv`ys_E_oj6uH$nBK%Ejo@Gwa!jQMQgcXt@t7t; zaEui~$R#c=izWrgt$SH5O)(h+_jIEOaDj-INU(`=R8o&5HVliWARfV|k2>+N@>`LM zsRb~yvH0_j|8jv5xyCe3b!T`O#6*YV8-V0?PNPdL!+YE2FDMwtQQ*|ktcEDfvrdYP zU(fnbWE~B=(o;>x4T>SiueNGE6I2)KOIF+a-PE(rrbO8DD=9)6@3MOruM-4T|TG{hNOpN z+bOcIw6BOPyd~SzqUR2(1$bZMz9&U&$7UVV@etT$8q@YL*zNRCyBYS%yQsLuWDpmb zyy_rcqNIm0wKAuPu7*RLPdU#Hv22^pV-LrP!Dmdaqi32*r)8UiC=MLK5w=vw<9;f* z4o@cW!JTq~a3XR(d8e{`Ggd6c8KOC)P1qfsD6p#CO(runFDnqAcb@d@nR&*PKYNhN zaI5c+D3`q;!@(5iJ~pRG4728ly;{4nIbx+BCiih`bafWDVyebVHaDYQh&ly=lt=5aobf(uJ z5TT^w>+N8Y1mE)bGq(TsT=wK4ecJHuy|;1eY--7=<>_l;{r&##WyZK#P=;(JO3QS} zdRN;F0=O_ZLA- zRRNZCPI~r`1C{>R8S?0L+hADE?O%-R;<~~lm($)u#;pr6*wTCwIQImvz~){vq8xG+ ze0UEElm_b@$(kTW7W!WWO%0%jEqHNF+75YM5yHE>h?T+-rKUE+cb$fGYwKYhTx~TD z*z6|+)(s*Vb?&7sZH=6HhrMVQUZgf~C}D0WmRi`9{L(>Gv<8znlavCg5gN#%JE2)1#4r(%V3o*$3_h61 z5VU*HP@#k|TdV<>qFXi!#O=gA92OB{CD-90{&1~R5v?ARkYmTlKE``rbz*WFeo+nd z1_YK}_H;SESumG9DG<;|E0T*Qr3;C;DM~aAWEPg~xPB4Df78f5a~S3M>amebA{xJQ z(Pc?6mB_LFvcS!EIC#**#xsRkGVH+8JOb(5Lq?FNr4ii`r6hy0deI8~7G-{Nj>(W3 zzCfzxLb*XIoNr@xzlRlF+8J;e6p7M=gxTd`nSIt;2h~C6q9&JG=rxWfw%l4`#yCZ# zN$b-w4q66PpgP2C#HrlFm)`(eTw92X}>PtX2g z@@OLh3nb$~GYSkea*ua}60;V%khIt#Yext-00POR^cP1=jP?vTMT8sG0*U4B2(vt# zUrJ4)#>u%+Rl>e!95_LNUOCF9Q0dT74o3Z#2zBV5yoe%jvQRp@7nVqc=+jvk6)2-j z0aRgKh>Z6?*d_+J;gV+uc5t|E3W(pt(Lhf_9z;}b(!sd>tbJbP*<%x8Lu zPTutIa^Qa)bkGJD)#=7v(#$Oxxy)+moM>1f%3lh1!)PHEs)@8t%)piCpaaNKIcVECnQm++MZ4Rkz zQZSyBbetHTjI7A-LYcV0H<>!u`;oM%M7_7Rnw-*4A=86#M0#F?_i5!SyXur~9~hS7 z)kCY}YLi$yo{dO=_;D^rRJqBWsFxxEr_4?H5&yQEwB0e0vu<2XmgK3RoD|PO8=N4S z;MyC~RZ*ZZC*tHU8yHQV)K&Y-lq$PQOXr>4R*JS|XLA=^ucXN(LYFlJT0 zU5znoGZ863KO4g;52xt6TBVzFUc@t0rW@cI*)L7I^@}xcvMPXMLRz!>Z47HZ!jS90 z^%8?k-cyO9u9n`Qh_NLzcJ(_JCLNC0umZFS9)z7LWKUuh#-yC7s@QK&8ds!CUFiJ_ zClqsaptC-n+GlhlC*$8@@u7nQd0|zrYqhI;uti7eOwHy`$?_{{i;?r~+BXl%%`8PjAa`Z^- z1!zhI{WiV=2J7D%*f>s16;<8&FU))O2spYRDn`-5`|opYhgKdkJYy}6YkDdWEe@@6 z4x_?7kr{Y&o~#K|KYEhp*>I@+u}i!z**tK~QI2d@KsB-0Gr8Dn<*%f1+zz5#Oi}Up z@Cq))|1sh?fS2OqQeh+!O;dNa_8zK5GkghQy4|6uX z_X-_F_!)bR7E3>R+EK<5bTIZIBIOZ#sDAULnB%zGrJ(_kkbk#pP2kZo3Go$ZDZET( zeBPmdNmR-+jQ$>RDDX-^>G>JUyefmm-0Z-DC@>#l*cy#;xg6M+3nd?hkr>8?wuRHD zh!`PZh`XasLbdq_BcpM=pAY57#)YI%APf&7q)TKyhvNHA6Q9^lBIojF5(zchXSlcq z%i>w+^B*Oq?2?kfm??%Mzy-`MbU3Td??0I#D%%rH0((tyDr6$rm~~Md+op#D-C$9* zu}}!MoVHxUUZ`mPmRI!)Yyk<6>(;nw07}W-)2p+Q|4qrS^uiR}pb|nAr15=GeV@DLZW>#f{rXbRTp{UaqKEFkway4Dr zhNtg39a~U#dOtDD7hfxep`pB;*@WX1Y7vwR^M{FE_gEPN0+<3>A`*%}JPCCW{c7r| zxHgj^Lb24!n?e~bcr)vC~S8fV9pU-E7%z0A@}F_I76d7ToEu}9b?F> z0v#49hTDv@MAz?=wFfnOz!%aVI@^Aw@9c|-t8&YeAHz(gh(+bM+4q&#&3Ku5*jDq2 z_ZzT8d_ZEGkG9@3e2K;b<*>K+95A=d>+0y9 ztUl=FyV(NnrP+Gp=DKS3n~$#y`m?+DsoE!Pp&c(09pd)MJU!Sa1W;FP9{#6!{=TQ? z{MO$?IeTq9%44ZD=XQ?&K9W)xU3h^synWmQk+D;)O4v|+??7GMl5xH6w%%_od|JWB z;3JKgAnn@*rzQEHO5~M~*ME0~4Gm3DzkFNn%C8yo^p?MD(l_SQ2SRXBEKF^D)85kh zb*929L}UQLoBQEi!SLI^2p%I`1NfInKG(!MH1{v@{41RGRYLJe1zP8$)|ac+Br9R zXfqe@iD1%{KEy8nPO)vCpf#=e^LrVB5>vmja8VQ4O`5;tUb;4I{X}5>UW7>!^ba+S zZ|y{hv8}rjrOXqMv0p`LQ9s<3=)LucI=efct#tVl6oH;w1-wi*@-)XJ?kjKAkSPJc zpXdEgI!U>5`XXLaYO#lmCy&EL$GHe^SpH7+@e5bk-YX!IW{1B^W+C0%%cV=rNa;j+ z86x6Hr2${@EkQngJy-U8*NH@`(=E(_)s&81MvJoE{Yyvh6c66Vom$z3DWY0KmaBwd zot88;th|!P-K9CQL1UBPwg{2GF{(Frc5(g`o(MBZ&gU`F^hmA6A-Zxo^~o! z%aVHNtZ;e17ng?lVWwY-2<2O!OYCmN;9v%XK^v*2N$Sx@}A4<{zKhrlx~#24Pq+<>o>w1igku^j@e08{O%dtENIqOf-Sar)w6}C3_PGW zPfL!Um2VKM#X9H-1GCc4V(bRT)w+GB0cHz}}R1%Pq86sU&cK+4|d2lC0aB*)q0oeRfrIVfYn7t~Z* zF^sCqvGYKkhEDKUpkb=ufHVWZrgn|v&r@QC9&${@IREiSe$iV@?g)JE6>&NmeB4EH zA_9~wv+}Z;2>E@V4qFWc-aR_@U-vi}^d9W3w3();Cdvr?Kb{o<0q?Jp3%!2#Z?B%; zPy3Ipo=&^RwqFy1U2oT~gaO~@qnxex*ff*hp@u9oF(JJHPnQ#py&wB$jfUN$kiy5k z<>1E?mDm7({qCcxTEVZEF-^m6&-*jWyuhcW+8MZHaf;7Y!@#?x#!zL9y?Iz;r?)oB zN71~1r`PY-C11r0#qO`IL{3G=@4ak8vMoL5f_lGgbJipspMiqodyI+DYWr2R* z3o`-thc7o@duNtQPI-a1r#oX}Gq3%EpU;8a-{+6t@5h5Df_&fC&x}2L zF&k;<*A*T3mB`6hnwR@GYW&Q9^L=knA1@H2J>W+AIPdFKdFJ}#gp$AW-N}QOQqjjD zb@1xDl{rXYzOtDs5Rkd+(Ym>%vX>SL{OO>)h%|d4K9!>3L}8;PV~w_st2mWH!Br2+ zUEG8WtF?WOLRj2%401W(?Ll2s&}PvjC(?}37>W3n-0E+44DGCmDVst<*yEux_=cvd*YMI{2^_};S_n@5I!GwF#A z4vm@Y<}dS3ce9wUnS_;`mG|@b7A?C0H_p_~b&gJS{*&9 z^`xicq*bDQ>Cc&(66<`Au2U}zxbdb7D02n}u9|`_bW*f27s(4Ab^B6g z$HUSC*b2@#`!E}{)8GK`lI2g+bBk-y!L3e67KMYMHIeXW%bp%4)6~=l5yfvir~W7c zn?3B={2{_j7Ob0FE}pWfx(JBBcq)^9lv#6)rEN*sqVONJ)1WucSVS&bgN|xS+2}_( zm5C{tK6?Jasmf;ou;EE%sY^qhKvoBYwTS9j+=M;}mI_GG=+3ppjmV(3DfeEVYDk;z z-r_v}qy8vu@UE+fuL84Km>BYIa5#?Q3O?CFf;OX@<%bVANB2yWr33JFQ143tIj9t$ ztbcM=B3O30_?bXAHvxxHeJQd3=_T@ znb>pM2G*^`Z01cc?>@jj^tC1&{iOtl-sLHrds&gn zE(SmN9^=-G8p$RSeF^=+tKpnS*cS_L@yg9)a7%} zgr*-O{Z@RP{E!Rh=Gul`*Q5mPwi>+5O*Thi-581{n zI|8FlBCDD^a7lcQC{mZh;U$?WKyPve zly7(^&Dtv2||fZ>IAvT5yNyi)=jrM*369X zCOCUeY|*PbaO>r|h28&%vjBR^q;3KlJ`1eJUP%gtQAw&7Ty9ZwGbcZnHm-T95yUl) zy3?T$9RqDc8H=+&i;_9XvQZ*46 z(N!hAQLKQGlU9Th@ai%ot%mq&K-(_WpR<3MrG%qDuJWpMY%~I-CYVaBh&S~yTcTPM ztdy0)hapG^^ue%M?se9v4OfFmZ*BII>3z%$EL`boRuLkL*PF9B>D6riAIkt$M6!_e zd?cRJuo`E>OiL?08Rz{T9HwCF2#ogf_S&~nDxU7wYiZ72c znwlMu(i8}D=uqkA}F){Wb6m!&*J!uV5T;3rk0tv7=KxD8A$oDES zzWMqo1J1$!6Af4X^CDG&erbJ}Iz%-dPd+hluG<_0oskV+#98?})qH)9kSIoVW6mr~ z+9{7zGTCwsVMtT(Ut%%kDf>S=Z`5Ko6k5{lUz{DL<2)X715VKq32wwch$m|z9DMkk zqv{#Ka(|383uobDs~FOjZJh(7W^A9o1vZ5?37A2ZiU9upBC8L8mr%l1wA;GzJI(je zn?I)}cy)4@+(QfL=>+{9`;HXQ<4?Qn+$G|=`0M`_6F@ZbuX%U*mfKKUj60eYjaO@H zRrFjmeen+-rKIvS|FZO(THMphtf*C-mTDlTM;hyfD7hO4c>pO76>5mee^0c7nV1%hljQ%(~)G(2N8!;&cQ`b^S zA85?}2#3xF5ABDh6yQTM6ZOz2`C_*VoIz{c%gk8asbAvn`QQE&A=u>U$K29O{%Wqx z9`vrnLFg&uuHMcg_4Xqte-i%3Xg1oW95eO;^*|O)E4sC+|7_SAm%ilw@!S`E9Y60N z#CiFzh#$ECWjs#P91GJfg$^&XB)^1?A{Z{0$s!=E9xyVO3+2({ZyQWe^eTr3+w^~0 zK}O~`!!GF7&yA@z5s|4oVck6phdyf!qRsz2QuJC~MvP0%e}x=UuYI zfs+nF#rOHdj^9%?PyU^2-#>1Z4nauxJk|ZRdpJ{RpR|Vh?sNj#xGE6?+U9=V?H|d0aWmGqD#y@b68&l^jAtcef zO@2T811#K_yQ6kY-mcRE{T;j~bIrzbRYOZyB_$_L4p%?9tG{zlAOs}k@)iPHq|;_H zsU&#?DYFI2zU8!j?tOj+?ZxR@ST{}2Fy}a5oJc*NU`@A~8%Umh;ED2;=Pg`88bv`q zy1rdmL_;!l+hf+i^I+{?h|6O^4mT~67wZ&f8RJpMp!SXifzvS|#5CGXnM(QgARcHn zOvjwc8X4yEi;I^HFB)D$w`T>)E28z}Xe%sO=H7`jR+654h=<`Oo*?oxWP! zH8Nt6`7!^f24mG{fT;0{+Xq(~Cy6dIfRvO+W`>JN^QX-S2W^^ToNTlfiW{*xN^Jl1 z{fXH2BW7ovH}*zYgQo4vHITt$f}9jSLa;P-2}-;Obn`RLnDf>kavv}N?6Krn>mgcN zgHhhmqV$?~b7zW`8MllxD$MXup$65oa4jYXa|PmaoYJ@R&Ff67BRw}H=bp{xKT_KN z^f5!9Sh2toQa-b}h0+p7WTdlt{f^fZM-;@k<)gqK%|!pV@z3%s-mjwEBXS>SS;UM! zw#>xW4?3sBBl1m~>P8x?lMWZ+j0X4(h9xiEa*gsV1>R3a?L@F4oSG&uW#>wbnD>8o ze(%f4LOA@;WRgMWKT>f2kwQ*d!P{j>F7`T`n&MF*whsj8D(B_~?Qdl(_i!=~vQwY+ zXOz>Q;v$zB>(X5?8}C^;Hei~OsWI5$)dYV;f!%nErY@#O6OWeSkDRDJ=>!p|M4=Kf zu8f>LRNUj*>2V(mbfRt$dYYK+)ctz=fzvh1$fOHuu~ z@&j)I3q+KY51pPLH_SncM8}5%Y|DdGb|#|u{ZKj$)gkQqCX$YF6I*oUl{?J~ss5dx zT$V3tb&Hhjzn||_RJXHGs{>j}1b@#D zjG`fy`__Li*f@+Vv+fPot5>-D)Y%=>=-2?HNrOPquOPs7QAfzSJqoIAZf zuM^p8!vS8O=be~>Utah7GdsQQuM5wP0fG1bmL|Ua?%tNB5*-1(uV?|9W7A*C7-d#YT7_+V9GWqQkc2~BW3#R%kGcZ-MG2CdQrFh`g-o#n?FkIgso!g&W z>k4aMN;i`*RAa2AOFDGq9rwbSD{VL1iGKq0K2v{tCT{Y(ERjkccwP4iFUMv{a%ZrI zt#BYtWyZnC@lFtA{x^A{!(?b!sAorZN7EIb`y@oe!J3kAC<_xi3J<(m@#?QCfnIYJ za@gF)z1}C_SU{a_P0Zzue5^`&1a_eW;vwwe>pDFa9`k@&=%haGnJlA}CFMwQNYWXJ zNKN9|z|5bvIBU3MEb>gN%HYK^g=SdHVZk~OTTHer48tjp!VQjbLc#j5V#n}D!rKEW zaMQio16_Z4k$-XF^qVS5ax*+G8lxYjul7tR==lw4{H%LcL3sTwtmxm5;j8!=SXc(& zY4Z3)qbZ!k)=v+l0J&jiNM1&c%01`LHbYlu}dnLll=mXAKD5e_wVbEs?JM&Uqi zoE1jkU?n1@2#10w1W$B+HP+hP{#uW8#n||~llgFN!#dN7sl{Y>#e&dIt^%TY>SvmX zm1Q{6>IJ7e(vm|h5zQt!ML1|O%S&~s_91v{SwbBm8$;s|Spxz-o%86ZE6^mj{9N-C z36{9I`jcQ{7lmr1U>blM*(qocEr@_g!hcHOaF9(xb!v|lnt6`E&22yPV?9ZuU@Ep8 zdJ3oZKhsUp@VP8KfUf>AN+Ta>EyB40*gL!EiUFRA_$bDO|2e{X{%p$F9+K;hsjuP% zKGO4l#78_o#7VgNa>bF?B1F1I3P`b2a{=eP(Hqf>5VaPAPFT~o7y;CiT9W)FO~r&p z8XY%7j4tLp(i3nzNn@KE9nDaSRcYOX~|`wWa`xziA5qIKWw#`~0}yZUN-KWGp{@LZ@X%Z|8;AQ@J-lSDXHi4z5A$?CsF ze`3^;)uRzwDQwXWU;lPy(6WTz6z2BPFhPq_ZfFi{3Y*(2?wc0xHBr46MKbrAQcFJ70G(tek;QtW_c?anK z)g)<3J*u8#LiJQ?GIAMqgH;!FHI8t(qeLMU0GtG%S;JfmC`|-O>IZQ;@f}q zPH*I$iwF}FlUm>6ysVMOJiMDVc&;rn>+w)SnN2?wx8^ZUknnb?D^=kRK~;f){?$q!s@V=3MRfeA;fOq4 z_H;|9+bR)gq-h31-;ExQ>_H>5;IGl^M9uxgL0MJ)5-A5vQ?ROi1a<{aB;`Fip|b|g zWk(QEoNP7$TwhaFoKDC)t)`gC}|5sKBgzFjz8=VNuZf*=pqmEkYHX1r>aDwjQ$&t&-)1$5Y>{3R$kRRA}AFrX)=)~{2Ma|e zO&^g!^2hKXZABr$yWK3)g!-m`hhFJR;3!xCNE^2ZYX=A;=Y09i+UE~pF`{uk3UjTM znRuEpA?b`VGfL1FUxpO|5X)=Xf^Rsq3sUTp6xq(Oy5E&eTeeDCJ>g`_Q-oJ>pEm2| zImR(3PbF2F-w#q>4xFgxSxLsPu1&b@lo>3Th3{3^Tpv)9@&Bt4h@8`2-H_Agq}88f zei~AB@;VQG@bDONb=nDC5%*Jk0S`+_ai&A(JJj92C96(`$o#VP|F=(3uNe_1hgz|f zBk0u_ad;f`tW1;t*p+`)Wl$BdPUykKk;P7Jt2${(0RIs1y7nzl+LQNWv+NtMS?y4tTUNgG;kBs>vf zo8Awu6aSDl5bhQQyeOPNLf z$(tgvuCH?PXxa)q2CeT|b^GngSM||7Su85O=GfT#533 zyHoHqp7WA(XF_AqD7mMAVYTM5oNb`ncC9gVnP%3N=#~|o?WtJO`y|%N?8V~OJkkk7 z#2yQ$hjby)d2VG}S#D6Sf1&5(I%vO7TuB->Yr?X_1`NK`O0*a$l7Y0}Giz8nTOBjYq6k@sW z4}qA_SapuL5w#lraIWr8u)QF7ICZPe^89~q!8Y|^`KOv^ExH*S3^IYU-Ptenz3RdK z8(8L12Dwka49q?99s_D2 zHbkR#%%6^|AM!#45k{Oc!s;heFof5w>5@b=8$+{bl9Tx!qM+sAB|%y8Vvn0In>(um1Zd$>+p zIZ7+K!{f2aIv2lUk=MW$;1c6Vw@&%jWOXLev+5X)*+Z!N=|KyC!3xH|uCIp+WE4^>Cu#cGt`O zv!lb;!)Nx+PM6=~KS9H;0QdWYpF9)rc6YqgyD2pF`~St8o!$R`@rHB+G2pKCW`0V~ zjIu~_l9J;n4kfjXM+BmihLF6sCM5ti<5W^MxeESJE!DAKfR@!v3Y))u9ZCcNINla$ z;on9lKq#@L^8rU_mZ5P@k9V-M%n}XqF%Ap=G4B5&$Dc{xj;U3LHrO=vKO9h2k|N$` z*zsC@|9BwB-~rj9IGwHf?M8X#7*`=ffGOzcM3d z-_M-vBb|f)sjt(dLvSglwk0h~=9wi`>)jj&^53LK8lP|wV=bVr-B1G%RKHBkq9;3q zl6Ip$G{I36R0=V|wI^azNWB>S2=n?fPPTg(yS*6-4f$LD?{L2&3A(|pXcdI0tKwyW zSo6hfR9sc$f2QIJXKWN3N@x4`YI2_HlLp;_Q>h7ZJllB=m_NfQKgp&@8jP6x($-dC z^3ltli+ROH*Vn{iZ*pk6YIl^9iGl}fp`biE;X`P!)7PNP7oqjaXXvhu9x;l%MurwT z+nl=G%tgix6oZMJCnxm;H-6$>uH62jy0{&SYtbix<#oyRQAirO(YgS^6kfm^6!cdG znFgw-G#Cq8L?!G|emp6(tN+**BvB+~-4p#N8E6^X$*6wa|IXyJ@Q3nmpD z7GM5EMoyawq@2;1YdDEjyUNCs_LUw<>c*Qjpy-RPgrLc8+J0b7%$g|VnD z6f=yqCD9}_)Yg(rqN3529s(tgb6DZ!wRh?@x897~<;O~FR`egDYGGxlkX27u84r|j z!XM$yDDOo?px%vzIgIg5cq7O_XEd2NAE+F)04XJ*U)gJf*_P(v4934)aG~*=JHTC=P{NFh`B;^@E`q~kh=It%8(}_NQ|0^NmK4ZlU!+au4>ZY{QwpCc3>C<9QE+gQD zKU{o7Rb7+<-yY!(V_ddzRKLsQ@2tD~Q0FydxSQzcezjz@FnvR5DK^WXgBHL+9CDCD zJbr+h_{VOhdM79viR&d;a!U>Xw)ku3`6FX93FJ3sH6G2yHyV~XJg{Ia%7#OJ=8Oit zYQ?R;E=NsSKZAx0`{*P6?uO6oEm%{@FVXRqo?l^Ehpj+c5plm$?h(fx-8CVTq{a0G zzo90V9yh5e5fqJTjz`!6%J@#W__w$2X|`Ok>{d~a3UW_C{?b^olD~uK$L|_s@BifC z!*=Nu^9p-!M^(Rmfr;opUnnDPx-(6H-(ke)G{Zk%TDj3zMaYPe<`IMn&DfP4z%0gO zD%8y=Zlr zTBxL5v+voMN_JU0LU*|{49|fwL4EWNa!IoWYm~4)WfC`Eqw=mEkWs{vz|>g4%vR%W zeIQk>4z1$s2!{{ogAv8H_;OKG-uz$rcyTO&atv{7qR8sVx(3-@>#Ec!_gV9-I0CEk zrx$~ZW|G5Rvol~`HnQZ$M$fb5XZy-Jp#J*%cx$^cl*>gq4?(7FDy7y-a|KqsNgZj+ zMG4lNOAJ^ISA9w|=NAZe7vhX-rAzRC{6C_qGUrvNQat$6HK-XbG|eqsj+17o=is&a zRA$7ZPJF?qKHCLoPNdD5BXnksT^65`-tg2`F{G$eR|mktSYV3MM39glVP$|#_#>%6 zq0OoFNLxIKxcP#c^Jzo96GBaQTs&v0{&Vg;pD^qB5H`dKAK~;XaQ*+Us;>;IGU&RN zPU(h2C>@9H?vMs0m2Tzrd`5)|~nT#&^h6zaN#Oe~P+FL{Pj>!83&eJ`69o3XN2NaKPK-)Yoju7 z1;(L!k$?V;QWZ0*mWSef{R04j!VTFF?)5}&o^W7PvLU{j{4TTXe3LT5asYzjgJUs& z;8)ZZ?+`k)U7ckzIeI51q#8xh(B5>|udyQ|3B)aRmV)O*@W=E9y})0`tcm3HADAAa zPjF-VakXJm3gP%La>i5vH7NAPkiE+}$8AD#;KNk{@3jY9zit=%G$RNY>TUjznHhA^ z6l!gto~m}yL_%m^+TFZI-6J((Xo4Jl^}F?zP83yRfEz*tv}UwXY)fkX2#Z6MiATgK~5A&YVj3mYlsrPshBDe8RvTAj=TL-JO!s%vo>- zrjpUCq|LpKqF>`4HbmKgz`YCX2DVh(@PcvuVS6)m_(^9oXsh-2=Ih7IpoMq!7E8qn@ z=85oY6hcv6&~4MXs6GRVv{&`dWZIhWP5I>@tI7IFFC2 z>g4fZ>03prB|(?;#--(-g`$<7K{P-~5Y?Y-2&IHP%h1|H4z;SJ{>@Q1e z7D-HO+F7zRZO|5Ul#RK~@#2@m8Tco`^z$KPpE>W-)xOi#SyIwiS`eNizn@|mmEp%& z<<{n!OnO$kj->+lUV>&;FzY8EC(1Wu7=MJDQCljx6BhIVSV!73Ye~*kZz^rLlx~gX6n+(-CN7GFXZ1xJCrIbyKg=hrZbDH7bL6B9)GBXljwBhoZBkWaT9&7Q`rR>c7Y1WrCTZO z@E)irfMleHuhzlu7 zX*D{pU5}8f%I8fz%>x!&!!M}~V(J>w0l#|{i<7+_Evnh^oSwj=dJ2GP`EW35M2ia% z0=(Y)jy(tN*MjPYk-9 zG6qP(mQSoBEIKLXL_A>lLaI{-!sP|`(hN81vV*MDc|+ql^ZZSqXYdPNc0{2Kf0Z6^ z5;1@B!wmVF7*Y@Y7d#kB!GXflgYj92E^xGL+~6JYg3d6>_MdI~O%UC^#EgFGpaYb) z38F^)@*j$Lybh+Z>BfjNp*WNGs_H5M?Uu4n-^I{3{7glJ0^c8dr@Lm5HG=_!0Yvrs zqTWITe;HRGwlbq8m7@>6ZGtRY${Dc#TSp3L-U#mK6KF#^{d$Oz#S6|8u}0K#B|38B zGeipcqj0NXHa?gf|9k}3NGW-fk@Xgf$>w{q zNId7Ci5jn15qJI2w9it@V_FAhyMhodsvNxLRgBN(#j>aJc-TmXUMt?lwaiLm^?yio zYQP+UN5SMhvtwt&3}za%A2$2XaX`3qoUWqf;m%y55qFe0j|Dpl6%t*%JkS$EiJMM) z_@ic9m}@7Q z(so*eDFef)Nd7Zr{uZn{Mh3(3E-Ak%z z12`bkBs_XUuAH>|&y*o#Ky`}B+DINE-eAbng747|lQNh#hh` z3LG%;W(WwXBK*vU;IQ>U?9HV*DU>(RifKdDv0_GhR>;POcGlvD&_s?@t9-yz+b2y5 ze>9IKYf%(=v~T8M>kU`oP$f3;<2-UoV;J|1D?r4BCi=Z$MzLjqAp9HX^)t9To<dGoa5%w96#VhQjm7MsWt z#dV!Lqrmr*+7-U#$zK(AziDa*=#W(y;hk05X9j0|8fKJh6g}io1_#l$*P)2q1 z2iwI27ogPuXrSx!z{odAsr=MLNgj_s%hSSY`|~0PGII@-na)OnCsa(92<*5Qbf&MB z%lXWkG${glrR)bLJkjWkbK16&^Ch*rgy825$1UCM<=kbM{I;5b z_Lgy$uzpV@BqZi3x}RG!jH?8=>A1QPixyQp#xS$mG3_+=krpm`z+tDdmGm2L0xPOe ziJ@OG2`yNHur_8gVq!4`HTA(dEG5#3k%F!LzptwqY~wxQck^%m_CqlOZF+fc-QZmY z+!XX!X9A|GOi5rGG2b`=6+V)f@^T;NieFuZIrwGBZGU2l@V>-6fn*k6MQ7539@rwM z5-#uKUj?dSa(?}w(RE{h*$T9I+8ZRCM5jJYJ$34RO4GxLOYkCSR&dr*Qb>4zWPF+XbG%-L&XQ-!a)(Inwi+2c?|Ih&s7fhGrAW>&7 zOIm^}`rtlqWdAnbLrPl*gV(+F$8DWEuI)RPUtmG!-#%Tk2xU%NZ|q*Rysi5= zmO7c(4_oFXuwu)ciGQSTa%=qtZYBE!NzA!owrT-=d8HNAWNa^J9Ih--1k@UuLHSvQ zAQf7xoAg*9yU3cU$W@rEVX__x9B9GBGlDzv214L_7>O1^a~?;YYQKySmI%L0Z0r*k z@r3!d0)4~7*8AHngNoUOvI%SC9Jjg;l(APg8Sgj=TxN=i+4@Ew9&Z|3n?G5I^HT-< zf2F;TMX<-(%4LMyJ^Lsw9nVTg7ZG2g#GDwvKPVZ7uUt)G?x9MF2J7s@Pjnb&4_=^B zyM2VnUD3oM9fN(VgPS6BOAY511o)vgpfWH?a&xREFV*>QUsx=T&u8Jnj_yM0g@3ij ze$`3xm3O^vGQo8@)36y3u>Nk0FCmvr4+?X$9I6YL=h9h55T8`H3F4$&m?*?_rXTnD z6>ScKaxZC_ho!oZrb1@`HTN(cy!i*DIC`7`3ZC?dN8s5jId^4eJPe?Nreoy~Yje)j zh!rf(PbZQu#_N2hW1mRFo(QJWwoss&A}U5vQ6ILt(pPsE5w;kID*eRWq6zckPM*RM z&~4MCN?|on_4NOEV^Cd2MeX>?t@-}+IUWZkmy#0P$5)VkxfI?$%`1;lb_qPk@#Up_iIe#%iB!&ZI=<~A$r3>XB+ zqMeagh*=76oN|;y0CEH};g)T3L4zNzv!;|2c=UNr+vueyTJl=_g*OPaNt+ zzslS|{4@_7>nLv9(jeH9*eqLkvc=-%PqBn}!JGF?V*F;lGMynR(k($+gJmYNUL}KN z7*3#^=bSU5e&0FIR*@zDeS33dVqb+c$GQ*_Hc~BCVsMe(@fpQPA=m z2n|YYaaYLoK{dgkh~)g=j`(wV?yK`AIu{O`zv3~|uen-FD-0I%lR^kVQE$O6Qm00J z>FKiBZ(rf7d3Z$aWAaW={8>p61+PlYU(Z<&FYE>b&tDIMk2M>aeo;yKmbU(MQb#d= zL~DV#G7vaG(Bs{qU0T_3ki3QYJ(sPQL$Y8fH=M|zI_V(bcn(f=PetM^=T#_E@tS{d z>czDyx$I<;Bk%XL7{39Yj-S6x#;b8lPFLwejEnYk{7-kyvyZ4Q@|Lkx-v{dES{`JC z)d=Z#nr$%5rC}-kv?JK#ye&iXDQFw5n%NcqGaGaCaH?pB_2JV-p`&ItBPxtfGnnrI zx}YkPn`7T2CWz^Eqm%{GHgL+=leNn6?hFgxlGsh zD7kxaoBw6%TP9p7n*=AH$-a_Mo^q^ZI;l8;{?OdQA*tf`O^V)xj35i`qHyaKIpms! zm^w#VhpT7_LP))kzRuM(TGLyun-ccp5yr7`AyI{km1W#%M%8$*C`$34q!AVI?7kwi z=ktf6=`%&=FD+P6ddH$VY#rAxO*|(hvg1bW?ita3j2;eLaxo@I@QkbYK|iu51E!nd z!u3pNew8$<#3Y9y%Ot6uHqnv0GEHLy?m|i6X7|Q#!>EEV-OqJ=P>ZM6_+%yD71F?? zjTHuBp1!nignl$DY3(4mGHTH)CrA+^`*Q|wFX?zGQr%Xb@45k6G)tg}l0I?Wt{F_| zY=SZ@tL8MD&aGDAZ56SHr3xckmPN-ve&~A1_nS^J;LE;kODJSu<2y5b|1M`*DaP$3O??<^K$6B;jmLT{Avm)LS>zYI? z+6uG&UrD5INn4O$f7~2(DJi-7zo$+8QYvNwyB0@w6iXEVr~W=`sPoE+HE2A+a(~I**f0L@4Gtz|ncy1I*oc#YSv?zLa%(iK zObnB=?XS5)`5X*=$*Qm13blSujyAu7rP3hsYx*hItG?`N*;1tP%q%WVm;jg>l0ANk zoC8q<-h964rq>my%n|l9ttRxO&RD#~zLV4Nvg_DRvv16@fl%)ZKIqe_`eWJni5vU@ zZ)VB&&}t1U#HgEs3K-mn&q|)T30D*@Lfx1hC`*9>o54dHtwV@AbWowyT#(>1O|4#e5Y+jWeezUHHQ!+1B7#}kSkt!fG~91?#xvM*4eQ2& zh)@J%k*EyjG=uIZ^UA>_;d>5!_c|k}-l;7@iK;mYF z+-Lclruesa?+6e+XG6a%eUU+FJ{1#7atw#@D|VA2$xbv(9B>{Y4ewi7iQ#7K6vvTt zP@f-62#|d-vYZ}BqI01NY77c{%T|5bJMnL=Wk))X zuS)FK>JGCe2sJ(;kgMSwK6<;iH~EqMnAqKK7#=4d{sdNg=NCJ}punM?P%zi3uDfM9 zwKIc(%*9ObvZ5K8m_u^Jaf`Q2^=qj2^vB$h$cg4>Y#O^TyU6LH^zRg6AkZLOk9&&l zVycg?2`(uinOrt$ghzfKTnQ0Nj~EkXJ+CZ-kSh5muuL;T?wxn3=%8ZfWYaQj)bJID z_E?{IhXkK=Wwy$3Ph#{dVqeHuo1#m$-E9%SMBAAKrSs=6BgAV~h z`!neM)*W*zaVE{vFKgHHL%;8V*jC%GDu&RH{Yp$D-X;8b0@KJ7p6xM+;pjxZaa=5- zxU&a1R1Ruo0`OPUDXUJGq>cqvvSRhq8rXa0dUhmH-LJ7&kX)2ocXUO$UGT1TF7LC~ z?*9ejpoq%e^bcQUlWEu72tAJ2tR;(_1LN^(8?PNt9!o^qOAO=#UV+Uwdi9nB(cOZp zo2~SPXank5PcirVj?CMLF*A$6suVM@+0ueIv|Uy$EQyO=rLS1*7ErS0*%M{p)sr+r z1hG_)8lAsVxL`ze)72CBedm&ty?uwy0+gCRDXw<18y|R6s;asCQkNSInr@Y!Zgqj) zT@r#KbG#4AJ~%hkp#?JrxQXTc+81kcB^nQ!b}!WvN|S%EqXLia24i|nza8cqaR-8> zewrub$Mqj{0$=`)oe`8#4fo4c?d#R^ff6lY4vsRA4HA50Ol;}sv4%ThbT{*=dO%!a zYU`gGkAoJkpTf-+nD+(llzG1_rFN@X1%jo2`lhCi&PnXEF3Iy7NO<*yZy2f)b-}?5 zmM@3d8pQ*{+_dkAXmB)y45fvLmjssxz%VYvFsd)Z1ezBT<5<*l5&3fF#Z+sQXS)%M zS+EzDTAC6yO$%a8>07aV9-1a*zRwxdZIFl`jbhPiyJOXrGbri{L}E}q2@t!IrkgY- zX1hOedDro3h8P#4#>E}KHT*J7YbY+0wi^!ihq8v;NK^MMBtOi4?6^($f2un%zDKl07{(&(wImoMUG>B9a+ztn;qz*$Dyo4$?V|b~A z*RrilpDp4<6gK_q0!X_s(py1tEi+yxO%Q5eb8AP&+>)*i2u7#`%w#{RQ@8J%gD__ue_7 z8Og*;hldqDdE(hj0BS#O$rr59zLr^M?I>=>nf*~*BjL)yCoEZren@C9E0)BP@Wc8m z*}tIsKr%~sF_~t)5kK@8#j1OofF&UjsFDpH>(*s zyUt5jvm0XFN1m;+U~NIz3hEm-G)Kg!nujCZlh67k@4qppqRi#(;+Q61_hZ0wA=yFn zy5xaxFdxw6vY}*I4sIwb*^>n){j^ZfMl=H+x5+zO&hebJx2S`9r%kJqBJXcrOS4uw ztPO<>}bZ886acLqYHzt+LQ(FX?D71jeUvCn=J z(*#`C>r6~?#8t%;F|kI-_hmo97-<8e5(1mu=-&ErOW901YU?ldgVE~d)?ok-OHonf zq$3s{Du<;O496$ImJu--Yv%%q=Dcwm9PFU-a`CwNyU)kz)^K*C$d0shh|=}VmP7ab zM6AIHVLr3RSoCh0p3ZAhfaa9er{>C!X7dE3s#PlC3L!PBK+UQ4%)2ul<%E)L!=sVwJ61V7duLP8v?%5OheR%@ysa`mzl& zvLkyTyaJvL5&0`NkhQxXdX?w|wb!0jimSKz!0q)r_gr7K$7)d)|CY&;)2Kv#Rf@=l z9b?r?d0dlHk;v*}5m30+*Y0a+|BKr7r~9A4WpKa6Csq-E=HIzn2lktgUyrSRnM(Dx zTyJ*qQG~)#241-Bttr9bC;|R(xsS7SqCd)bS{y{V3|T?Tr;l(=jt_F0pcU44_Vz~5NM&OFbRHPDHlRu(qs9eo5r z$lSS&=TtxTgdo0hXZ*U#7k5#7ssvj?`M6;w$ig`zYP>|xX7bHZxEp>FU{pfc@61T; zd~cs( zpE*i-Xinh-8EQ-!XP4szdwaUI#p!)2GILq^0cEq;V72L~dZM;Dl``!({QgjYS2#S( zENV=5}ka8CpgvL&N<`lT6lYsYO*1~wbu=Y#FFdH#q(p2d{I6K5)3@B<^zuN$q8=L#0wjq@@&F4 z%?{Q}n-gpcH|L{5iol5$9O_F@^gU3*m_*Pz-mrtjx;tD%CA)O{ee;I;9Hf1#dTO{% zs_Mdx**68BM9rMH7Z-@(9_W#Tu+fBj31HlZbSm1v`;Xqx{l!UCq6=7ifw%QuS(AXb<&7N@hPA*=a>V57jB_U_yT6mghF75V z*815NaZ_D*h26#O^*hyt$C`HsOd;q%DlT!paUAF9KxT{kIV#wsVO~k-#v)9xZAV&H zz+D(h_75mr08mT>)D|BKOusz1>}cSnPGO$o6YFEN0*jfg$C_T^EciZ^dWT^^IIht@ zdXnPcw-aR=@>3qU3-x)Cv*RI-G%luqz;BTR!iqp8g!GWmRIkz%65LnOqE42^I|zTEoh>fWQ)d@cJf zf{MI{`1{xB0|)}VNUnL_bsMf2b|2MGH;8~E1&m2AXH{7Bk>(U6HRO%>cYg1#!;h~ra zj>OSu=XXXv7(Dd$ct2@SG50?56;im9q`D@Dne-GP&ExlMR%*oGSPW}D0_!%k2ii78 zytw;2j74=9W?V*UjZM;#$W0#sU8#~;`s)iNi@TCQCyrw=D9Wm`WJS`GHS&H2^JFo# zLGBJvl1#h&jiJ!yO+B~D(9Nt?MEQCKrD6g4UCo4XBq^{2hYCxp+S=FOtt|@Rsg$Js zDSwf;`L6#47ci+eQHb)sZF0~Wk6=3;Rys-N{MFu9UBE^?J0~%OC$;PI4(`prwuy1# z*ewg?wxOh_ss2fGg^h-YVS!%G7Kl-Ci@(Lsm4eJJ-duF`vJUv~zV}`Y49=WnQR%8-5NN$f?$GRB6|U?TUT76P z^F-WA6v`3~h4AOmBLL%}V)9xY=CBt?r0Va(5X%jdW!m9w?RN^mdHTCOhNG^>+Z*(( zee@A=q8$Ibj(gQzxN*%Zq$a!1J_}~_E4jSM{V>bVCNthWBaFQU0_DLo)=k&iwI= zdvF$1Ro(rIL9~Unh+*CP8g^lh_flg)919h9E92VjKS6wDcVvAQo6ZwlRaBY!{Y!6i zOWDDLd$KWF7e}r^3#{+(5_R}0g((cEwqr9@u;V&fuMN6j&3y+?fi=Wq(>qx9=E_@6 z@Z$Ji=$aX_#QdaP3^)m9x@dQ`4s4S@UwI8Y4lpbZ+=*_uI{sd6#k>}zGS@0*^=?0u)@jioW{Ae&sW5a~&4 z-4cR-0^|}wMzpMZ700vE_IEq0TGN>}=T#-S!JCj1dZj~4l~7%ul_FOC!!1CaZ~m$? zf}=2aYIP*2?XN&X^t?-Zo21)UsPR|eIwC{4G43OnDY-ON&5^LIW!#2%41~Uiep`^^IMeRWUn$CVd!pK4!5n=Nr%+zAns(@Jq-sX(idjG%)#lo!QgIP z)2-*DTX4>u^ipOUxiHko0kZUuKB~GjO4`k?TjeLk;$Xb!<;MnoCCvGNiD-X>1v1>Y zQC=$9teM~XkX(KRM+QrcJ*~bHe(jVqjg491gd`ayB-ItCmqwo$FX0pVd10>GQxhk# zuX*1PRtqzkAP{$t4(dAV;A$gDl=fJdcAwH$Imm9;8WJh`HsOJV>%DKpx)OfgS?$l-vo$+w* zW*&^KUoD=;w3p;hf&|ODXEd4p|0^h^BbvHzGyp~yP9AP(OGO5){6g*mA+h9pHzjvV z+?Xb5EWj?izq#~YUSh((AsaXi3-83Qt768{O`~^;Pg++6H!dxKj}E8)V7l1-DoEJ>u?E z8ryb-{_==Wk@=QuzPlar}ItVAc(!HyykQ=`odWn`UrOpVDmm?gzWyrBWw z%Oi~Ck+~L zw!`sd`PJfYoe{u8iJVeCkGUD9{7;-QW7MK@Q2TD`M)&#}m}H=)Rk{x;Aw!IV$GzgR zdv>}*<*$72TX{Yc4)Sx3Vx_}pkDI9JV{F1^hy@TL)8wtJqXx&J~|69^GrybEDx@T-sokz{JVFo+TIi-TLdW1JPZPRM7e2^ni1gU$mcp)8lr81l#K*;$ zSDIXb-2 zmNzoWzN%A=>p4Cim{sbuAXk1A=vRo}f4N`El!ddGpC3&)Z^9(5K?%zD8gW+Hyk4Vy zOiaRlSX>1*0-X3=I))~dJoX{#S@Oi3ke zb29pUT&b2M%W`l*okaNY}pWv|Sc$W&)r8d|pEVLqV zwnh?a8UzdcCKu%)7O=2I!nnUBE76N^@j00>(K*Zhb3J_f7zp7cBGg>@AiOXKf3MH07L-65p=LT7&Fmo@L$C_*N#d? zE#@|>N>TR}WIJ4*7dz4x0oL@#o*Y%-W-N?GK`ptbUH>{+<8>x5T-yCS#!NUanzDN# zkiw|FrB+s+{uNND*%(fpo|1&isAY%edG4dpGzC;IG^!rq{de0jAxEe?VvbyPLb-E>Q-{W*NQ zLU+rzUAU_p;h3sdgEvZ2Mxp8G?e>@#c_$5ALsZ?cE7iFEt89PE1>sfF`^hsJ3(?K$ zRbj!QvD|f%+$(^D(z(cH@IyHa676w2IFr+ZpY<`*LfbUg{3epQGprw6Fiy-OBxaX5 za`V&itN0a+`wjKJU=4~k2$;5u3i^RBft(`SeagF;&58DZ=21h(Jqk=BnK&umCbefA zGUqdirD865xV#*wdeLr@gfjJ!O&;NfGF~;g?@Zy88_v-R`JNrt5ofbq=AYl>r5S}==){rc<~%x0)WQVF$jK>86$hd>2K8O*qq{}wp@PHsCW^Q z!YEPq?Ihv>vfB)P2SXn~2vQ-z(sDPIU$�p{_z5dlAex4A5Jy1R6LeDv6ujGF0~| zV29q}ara=wU0O_(v5g?z@zrf+g7HT^YOWlCwDSQ-JC~5VB;h&+6Hspc>$m`pCVo-c z=Z9kD2_+Qg4`d4~=D&!^*ds}R@}p9+iqLOOBRZW$9KVP_#QncX*w;A5i2wx=cQOc% z07Waf{KoO~ZFGf!F_qBJFv=MVZ{?NaqMQW+f<>)-tV>!Ilx_Y~ytqb{b5(Ca$|`IJl1i zNwSAot;}57!vA~o8>4_=%|ZGQJgP{=BK6p&o6cnluv+$6GetBQLllvnFOga`N=;6=KFoSC1DNY4)`o={zH1}BKI{+r8Ssg*z&7w3F~RC@ia zjHJNfpD6tPbJcI!^}01a(rqg?M*)SNZsor-+FA>o0>zPi_46AXifL z*`s#55QI>2Yzo@nRVObxV>2>(kzNh3G>{E3MmKHc*&F`JvXc808fm{gL5-HKe%VrM zufa)4I97NhzN5DNT4(j#^<%-5es6Kn*_ltWI3M)(Ah@vPfd;kv{?^Oi%?AYv_^-Ma zZ+f0vvA2!?Ol9}_(u?~&Mc&WX-nm^*o3rb_H}eEs_-V61B<(VWf`QE)3nxv-MMte!ifY%Xx{c-{Ezx~ZB`f}9_Sw7_%%T3!i zKSRZe*mIuO>A7kZeSS___q&74vOeWf_`jTB{&<;pG&`^uYJGVqH5l|432u6B97G%R z7kV9ccU9Ws*(t)CF>_2&g_!`Ge*4bvBCQs<7pfxq^xNLwTio(K&|Tq?TH@k3ZQbww z`k^(~@1_$%{+wg*^ye5;^o8cx{pHZD`_XjiCOnC`o73-kz31sqW1G{xN#Sxd`Nrv< z{X^32^Fx0fR_`oRD*mDW%WqHp`%saW%ViaVr=OxPzhC@3ZhB(Jqbp;NFy%gEp+HQ# zAD7*qVK`wBHu9`JX(b3p>MNedlxF#ikGyUZ`e5T1r?1I)&oYd+)-NY&p{2$+X9p%S zLNW;U!B%{h+R;pDybIwmMKE6*+agsaaNtvyjeMA7~qJYxF#FVJ9HhYP(U lEJhr9-uZd-GtF^`8lAyd`pzCvznr|l -mapred.child.java.opts-server -Xmx640m -Djava.net.preferIPv4Stack=true - diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-topology-output b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-topology-output deleted file mode 100644 index 22b3ea5f22a..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-topology-output +++ /dev/null @@ -1,343 +0,0 @@ -{ - "name" : "", - "children" : [ { - "name" : "194\\.6\\.133\\.192", - "children" : [ { - "name" : "cluster50218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50232\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.128", - "children" : [ { - "name" : "cluster1218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1236\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1200\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1227\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1223\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.128", - "children" : [ { - "name" : "cluster1872\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1876\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1854\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1869\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1846\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1867\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.64", - "children" : [ { - "name" : "cluster50286\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.192", - "children" : [ { - "name" : "cluster1143\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1157\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.0", - "children" : [ { - "name" : "cluster1586\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.192", - "children" : [ { - "name" : "cluster1553\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1551\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.64", - "children" : [ { - "name" : "cluster50130\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50141\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50152\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.192", - "children" : [ { - "name" : "cluster1259\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1245\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1260\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1261\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.64", - "children" : [ { - "name" : "cluster1064\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1059\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.0", - "children" : [ { - "name" : "cluster1288\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1294\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.128", - "children" : [ { - "name" : "cluster1080\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1097\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.64", - "children" : [ { - "name" : "cluster3071\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3049\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.128", - "children" : [ { - "name" : "cluster3097\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3098\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3082\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3112\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3104\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.192", - "children" : [ { - "name" : "cluster50510\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50478\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50505\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50481\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.130\\.0", - "children" : [ { - "name" : "cluster1930\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1928\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1932\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.0", - "children" : [ { - "name" : "cluster1770\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1779\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1771\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.128", - "children" : [ { - "name" : "cluster1694\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1718\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.192", - "children" : [ { - "name" : "cluster1412\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1405\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.64", - "children" : [ { - "name" : "cluster1357\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.192", - "children" : [ { - "name" : "cluster1911\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1889\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.192", - "children" : [ { - "name" : "cluster1004\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1020\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1028\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.128", - "children" : [ { - "name" : "cluster50021\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.64", - "children" : [ { - "name" : "cluster1457\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1450\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1466\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.64", - "children" : [ { - "name" : "cluster1816\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1803\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.128", - "children" : [ { - "name" : "cluster50359\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.128", - "children" : [ { - "name" : "cluster1393\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1396\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1369\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.64", - "children" : [ { - "name" : "cluster1660\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1649\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.0", - "children" : [ { - "name" : "cluster50543\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.64", - "children" : [ { - "name" : "cluster1639\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1607\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.192", - "children" : [ { - "name" : "cluster50427\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50411\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.192", - "children" : [ { - "name" : "cluster50053\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50057\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50050\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.64", - "children" : [ { - "name" : "cluster1733\\.secondleveldomain\\.com", - "children" : null - } ] - } ] -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-trace-output b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-trace-output deleted file mode 100644 index 5dafe18a795..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/truncated-trace-output +++ /dev/null @@ -1,1410 +0,0 @@ -{ - "priority" : "NORMAL", - "user" : "hadoopqa", - "jobName" : null, - "jobID" : "job_200904211745_0002", - "jobProperties" : { - "mapred.child.java.opts" : "-server -Xmx640m -Djava.net.preferIPv4Stack=true" - }, - "mapTasks" : [ { - "startTime" : 1240336753705, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.128", "cluster1396\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.128/cluster1396\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336647215, - "finishTime" : 1240336651127, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000010_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1369\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1854\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1872\\.secondleveldomain\\.com" ] - } ], - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000010", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "MAP" - }, { - "startTime" : 1240336753750, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1553\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1553\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336752204, - "finishTime" : 1240336755959, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000017_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.64", "cluster1803\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1930\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1932\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818110, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000017", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336753796, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1771\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336766565, - "finishTime" : 1240336770163, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000019_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.192", "cluster50481\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50505\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.128", "cluster50359\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818106, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000019", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336753840, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1779\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1779\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336765887, - "finishTime" : 1240336785565, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000005_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1245\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1261\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.128", "cluster50021\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818114, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000005", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336753888, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.128", "cluster1218\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.128/cluster1218\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336755247, - "finishTime" : 1240336763432, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000000_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1236\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1889\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1911\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818113, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336753938, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1586\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1586\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758229, - "finishTime" : 1240336761612, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000009_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1227\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1259\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1260\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818116, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000009", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336753987, - "attempts" : [ { - "location" : null, - "hostName" : "cluster1670\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336754665, - "finishTime" : 1240336777673, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000002_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1586\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1586\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336821839, - "finishTime" : 1240336824652, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000002_1", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1357\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1450\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1457\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336842768, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000002", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754030, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1869\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1869\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336757380, - "finishTime" : 1240336763176, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000004_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1294\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.0", "cluster1288\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1876\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818118, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000004", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754077, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1770\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1770\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336763994, - "finishTime" : 1240336767750, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000007_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1466\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50218\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50232\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818119, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000007", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754124, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1816\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1816\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758341, - "finishTime" : 1240336762025, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000013_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.134\\.64", "cluster50286\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3071\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3049\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818120, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000013", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754176, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3097\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3097\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336768390, - "finishTime" : 1240336771627, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000012_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.132\\.192", "cluster50057\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3112\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3082\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336774482, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000012", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754223, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1660\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1660\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336754801, - "finishTime" : 1240336758231, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000001_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1080\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.128", "cluster1097\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.0", "cluster50543\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336774476, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754270, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3104\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3104\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336762622, - "finishTime" : 1240336765834, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000015_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.192", "cluster1412\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50427\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50411\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336774468, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000015", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754319, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1867\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1867\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336759528, - "finishTime" : 1240336764456, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000016_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1393\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50130\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50141\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336817903, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000016", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754366, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1649\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1649\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336759909, - "finishTime" : 1240336763727, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000018_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1223\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1200\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50152\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336818420, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000018", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754409, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1551\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1551\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336755433, - "finishTime" : 1240336758966, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000003_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.64", "cluster1733\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1607\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1639\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336816560, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000003", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754452, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1405\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1405\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336747936, - "finishTime" : 1240336751464, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000006_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1064\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50510\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50478\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336817330, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000006", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754500, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.0", "cluster1928\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.0/cluster1928\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336756161, - "finishTime" : 1240336760016, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000014_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1059\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50053\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50050\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336824060, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000014", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754548, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1846\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1846\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758220, - "finishTime" : 1240336762903, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000008_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1157\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1718\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1694\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336824556, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000008", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "startTime" : 1240336754596, - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3098\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3098\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336744261, - "finishTime" : 1240336747517, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000011_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1143\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1004\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1020\\.secondleveldomain\\.com" ] - } ], - "finishTime" : 1240336776031, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000011", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - } ], - "reduceTasks" : [ { - "startTime" : 1240336774548, - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336786769, - "finishTime" : 1240336870553, - "shuffleFinished" : 1240336859759, - "sortFinished" : 1240336860092, - "attemptID" : "attempt_200904211745_0002_r_000000_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 56630, - "fileBytesRead" : 71200, - "fileBytesWritten" : 71200, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 0, - "reduceInputRecords" : 106300, - "reduceShuffleBytes" : 705622, - "reduceOutputRecords" : 0, - "spilledRecords" : 106300, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "finishTime" : 1240336873648, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_r_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - } ], - "otherTasks" : [ { - "startTime" : 1240336739508, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.192", "cluster1028\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.192/cluster1028\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336739565, - "finishTime" : 1240336742217, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000021_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "finishTime" : 1240336752812, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000021", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "SETUP" - }, { - "startTime" : 1240336873651, - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1771\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336885885, - "finishTime" : 1240336887642, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000020_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "finishTime" : 1240336889658, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000020", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "CLEANUP" - } ], - "finishTime" : 1240336889659, - "computonsPerMapInputByte" : -1, - "computonsPerMapOutputByte" : -1, - "computonsPerReduceInputByte" : -1, - "computonsPerReduceOutputByte" : -1, - "submitTime" : 1240335962848, - "launchTime" : 1240335964437, - "heapMegabytes" : 640, - "totalMaps" : 20, - "totalReduces" : 1, - "outcome" : "SUCCESS", - "jobtype" : "JAVA", - "directDependantJobs" : [ ], - "successfulMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 8185, - "minimum" : 3237, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 3237 - }, { - "relativeRanking" : 0.1, - "datum" : 3237 - }, { - "relativeRanking" : 0.15, - "datum" : 3237 - }, { - "relativeRanking" : 0.2, - "datum" : 3237 - }, { - "relativeRanking" : 0.25, - "datum" : 3237 - }, { - "relativeRanking" : 0.3, - "datum" : 3237 - }, { - "relativeRanking" : 0.35, - "datum" : 3237 - }, { - "relativeRanking" : 0.4, - "datum" : 3237 - }, { - "relativeRanking" : 0.45, - "datum" : 3237 - }, { - "relativeRanking" : 0.5, - "datum" : 3912 - }, { - "relativeRanking" : 0.55, - "datum" : 3912 - }, { - "relativeRanking" : 0.6, - "datum" : 3912 - }, { - "relativeRanking" : 0.65, - "datum" : 3912 - }, { - "relativeRanking" : 0.7, - "datum" : 3912 - }, { - "relativeRanking" : 0.75, - "datum" : 5796 - }, { - "relativeRanking" : 0.8, - "datum" : 5796 - }, { - "relativeRanking" : 0.85, - "datum" : 5796 - }, { - "relativeRanking" : 0.9, - "datum" : 5796 - }, { - "relativeRanking" : 0.95, - "datum" : 5796 - } ], - "numberValues" : 4 - }, { - "maximum" : 19678, - "minimum" : 2813, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 2813 - }, { - "relativeRanking" : 0.1, - "datum" : 2813 - }, { - "relativeRanking" : 0.15, - "datum" : 3212 - }, { - "relativeRanking" : 0.2, - "datum" : 3256 - }, { - "relativeRanking" : 0.25, - "datum" : 3383 - }, { - "relativeRanking" : 0.3, - "datum" : 3383 - }, { - "relativeRanking" : 0.35, - "datum" : 3430 - }, { - "relativeRanking" : 0.4, - "datum" : 3528 - }, { - "relativeRanking" : 0.45, - "datum" : 3533 - }, { - "relativeRanking" : 0.5, - "datum" : 3598 - }, { - "relativeRanking" : 0.55, - "datum" : 3598 - }, { - "relativeRanking" : 0.6, - "datum" : 3684 - }, { - "relativeRanking" : 0.65, - "datum" : 3755 - }, { - "relativeRanking" : 0.7, - "datum" : 3756 - }, { - "relativeRanking" : 0.75, - "datum" : 3818 - }, { - "relativeRanking" : 0.8, - "datum" : 3818 - }, { - "relativeRanking" : 0.85, - "datum" : 3855 - }, { - "relativeRanking" : 0.9, - "datum" : 4683 - }, { - "relativeRanking" : 0.95, - "datum" : 4928 - } ], - "numberValues" : 16 - }, { - "maximum" : 2652, - "minimum" : 1757, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 1757 - }, { - "relativeRanking" : 0.1, - "datum" : 1757 - }, { - "relativeRanking" : 0.15, - "datum" : 1757 - }, { - "relativeRanking" : 0.2, - "datum" : 1757 - }, { - "relativeRanking" : 0.25, - "datum" : 1757 - }, { - "relativeRanking" : 0.3, - "datum" : 1757 - }, { - "relativeRanking" : 0.35, - "datum" : 1757 - }, { - "relativeRanking" : 0.4, - "datum" : 1757 - }, { - "relativeRanking" : 0.45, - "datum" : 1757 - }, { - "relativeRanking" : 0.5, - "datum" : 1757 - }, { - "relativeRanking" : 0.55, - "datum" : 1757 - }, { - "relativeRanking" : 0.6, - "datum" : 1757 - }, { - "relativeRanking" : 0.65, - "datum" : 1757 - }, { - "relativeRanking" : 0.7, - "datum" : 1757 - }, { - "relativeRanking" : 0.75, - "datum" : 1757 - }, { - "relativeRanking" : 0.8, - "datum" : 1757 - }, { - "relativeRanking" : 0.85, - "datum" : 1757 - }, { - "relativeRanking" : 0.9, - "datum" : 1757 - }, { - "relativeRanking" : 0.95, - "datum" : 1757 - } ], - "numberValues" : 2 - } ], - "failedMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 23008, - "minimum" : 23008, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 23008 - }, { - "relativeRanking" : 0.1, - "datum" : 23008 - }, { - "relativeRanking" : 0.15, - "datum" : 23008 - }, { - "relativeRanking" : 0.2, - "datum" : 23008 - }, { - "relativeRanking" : 0.25, - "datum" : 23008 - }, { - "relativeRanking" : 0.3, - "datum" : 23008 - }, { - "relativeRanking" : 0.35, - "datum" : 23008 - }, { - "relativeRanking" : 0.4, - "datum" : 23008 - }, { - "relativeRanking" : 0.45, - "datum" : 23008 - }, { - "relativeRanking" : 0.5, - "datum" : 23008 - }, { - "relativeRanking" : 0.55, - "datum" : 23008 - }, { - "relativeRanking" : 0.6, - "datum" : 23008 - }, { - "relativeRanking" : 0.65, - "datum" : 23008 - }, { - "relativeRanking" : 0.7, - "datum" : 23008 - }, { - "relativeRanking" : 0.75, - "datum" : 23008 - }, { - "relativeRanking" : 0.8, - "datum" : 23008 - }, { - "relativeRanking" : 0.85, - "datum" : 23008 - }, { - "relativeRanking" : 0.9, - "datum" : 23008 - }, { - "relativeRanking" : 0.95, - "datum" : 23008 - } ], - "numberValues" : 1 - } ], - "successfulReduceAttemptCDF" : { - "maximum" : 83784, - "minimum" : 83784, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 83784 - }, { - "relativeRanking" : 0.1, - "datum" : 83784 - }, { - "relativeRanking" : 0.15, - "datum" : 83784 - }, { - "relativeRanking" : 0.2, - "datum" : 83784 - }, { - "relativeRanking" : 0.25, - "datum" : 83784 - }, { - "relativeRanking" : 0.3, - "datum" : 83784 - }, { - "relativeRanking" : 0.35, - "datum" : 83784 - }, { - "relativeRanking" : 0.4, - "datum" : 83784 - }, { - "relativeRanking" : 0.45, - "datum" : 83784 - }, { - "relativeRanking" : 0.5, - "datum" : 83784 - }, { - "relativeRanking" : 0.55, - "datum" : 83784 - }, { - "relativeRanking" : 0.6, - "datum" : 83784 - }, { - "relativeRanking" : 0.65, - "datum" : 83784 - }, { - "relativeRanking" : 0.7, - "datum" : 83784 - }, { - "relativeRanking" : 0.75, - "datum" : 83784 - }, { - "relativeRanking" : 0.8, - "datum" : 83784 - }, { - "relativeRanking" : 0.85, - "datum" : 83784 - }, { - "relativeRanking" : 0.9, - "datum" : 83784 - }, { - "relativeRanking" : 0.95, - "datum" : 83784 - } ], - "numberValues" : 1 - }, - "failedReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "mapperTriesToSucceed" : [ 0.9565217391304348, 0.043478260869565216 ], - "failedMapperFraction" : 0.0, - "relativeTime" : 0, - "queue" : null, - "clusterMapMB" : -1, - "clusterReduceMB" : -1, - "jobMapMB" : -1, - "jobReduceMB" : -1 -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/v20-resource-usage-log.gz b/hadoop-mapreduce-project/src/test/tools/data/rumen/small-trace-test/v20-resource-usage-log.gz deleted file mode 100644 index 6d0dbeb0bf94478a7b3318fafe94446a0667fa3b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 845 zcmV-T1G4-diwFpdo#;&f19mbnEplaZZ*_8GWi54cVP|D6Y;R`()mGbXn?MkK?^hty z2U}|K?s73i@&h5HrZpGsE|IKIA!=yT#x^b{y{PKHchXwQZEt$?d;5)nX@y@ z`bC{d7cOO97>r~rOUk3?~^b{dPWU(9AzB`gXmQ^mz z%9~{^OEGilNWRMQvRu8-WQoP6zQ(CR9iT^CjQy-+YBW?P}0%*$Ce zzbe=D0Fz;&pv{9%bb?Jdi!tX>8ikaPWE~*`{dEpDWvxeY5IVacHOFpt292rLGQAs7=-+yA5@H5 ziD>Hw&D1ojOF#S(20CxXojo=&+7n}?;i zF^R@g<~etETf{?s6H(~H{N@n)*tMe{dxxbupnQG!FXZ^cH{sLNY6nETgXm0LJ9zM* z+M?JbS-ReX8`S8ZExC27XG~b6&BhL ziPTHx-Q!AGi4t;4ZvqI+uBvi{XyUeM6UgP|zU*rM`;H9++j#7mF>ae>)T$}OW6wN3 z=R9*>e0&`&C(b`x{Xx6iIj_N*Q-ASx_Q84Eeciq|uYH+)7$X2YKoEp3p<@65s)fJp zG_G6cwLfO_TjyrBnl5Y3aM0?X*FH~fXS2uOCN*a;e0|;a$9_9J5F!FFrXE9_@aP21 zZs$@Qe*1Cp>g??1ezjNz^Pi?uII1s#o7wdCKKMPjznwiy?xv&q&FtaqVe&W+ZWn)v z10T-Tmq~t?Ioa^rQT@+{d*zjld;NB|-}c{~*E-$)b>m8Hmk{jM9KY)~ zuEy7m-r&53!1i;$bunzV1`+Dvbq9XKABvB?eybN=)1NH9I(}pDOL*0#_^^N3@k96n zAaRE9z_I_XC$2td`9pCAalU>Mf97J30)hXach&aArNYZxPaYdff#2gY;~aXIZ}@)e zy62C>5tH>fa|VrpbdHRQGxi(JUt0a~TGE#D$<5bbKHf+Q1c=Kd<<?n-nwzAs8;9 zJMh=AoIwwb>R7CgVX;OJohbkm!S1)jHT&((&*!y8uv|UXoMv~}@r5Ngul?s_HvcrL zPaY>XpMz0-t!$(ETH8kTP%!`A47EO(pPmTpe(DFetDC^NoKGGCVH3WdzCRiC+E-Vt z3nAsrZvSF%dNO#tyT1=^oxYgU{C07A0#1MZ?;+$|13!v5MBi_vYW8Qll>V^O5%)(A zA_x)?37eCxWE8uA?hL_Bjv!*;Km19MF!N-B^im8XLiEA`UUC$t#AK9<F z&%*!_O1T;koPbP)fNs*VB9glz=`f~(UwHo$KnSCEARv@21Ls^0L`+)|W~w?K!+~>U zdlDPv!9&5ItQa0vgdyBy7|4TM!t=#QYE?9WK*~Ub+a?K9#B3y{N;rgQ#YPdDYbB+l zq}S3!fheLNeq@`65QUuqM_w-PNd|H0UM-g_rRt}Nfht)N;7TMRb$6JGVZb;=6=HvG zH0)rUiB)kB|I6acki~W|h?#LE9B6zL2CfoMqy~`#(Ir&13Iw{D`-X_X;mrhyWPUzMT?sN_$uq{9r| z{7s}}6q9e;HN`=B<3K%$g-pXBC-av{gOUjU0u?`GKWLmQ4cTGxXpMT?37sH0q_RW- zLkyvUL;;3!AY_RG!U^$o5(VD`0gW&>qa2e|nn1SCmcx@y6C~s2mv{HUAWFP#;^T|c ztnc2Ryt%j>j9YQeNhh6A~G4(@4 z@0;E0*X>Sg+#UM8p}z~_H)jtY?xuk=TP+_~OJ}}~o4I9GAc$L&sHh`54tH%%Ok z8Gzx@#W{lbc{g*3R=qy$W)>g*S_a}+L|hL=M{j4Jn#p$Zso5Y#Q2HW?*{2t~8K9Y_ z$?_UBq}L^caVu;`Eo^1&EHFvO0%=NGY+0f+=BA{XJTfa~n)N=q3Z^N6fKEb{BMc=8 zX_Rz4^5WLth-e_;h%>AFKe+Oj3)AT_O&G+q(sseO)Tfg-0SCn_wX z8jPa+b|p@YIRZM$3K>e*KFQJma(Jv$W0l{`smVx;=}`?wR@kht6$slrt5#=SDV)t0 zZZe*F5PNRoIzXxOL_xtiA+8Ea&WQx!Ul z#z5%$TVXq5VJjV^!X`l~EDh69f^kupj#6ncgfvXYNLVLWNtn(Y0j)5d7uI8Bh3TS& z#&du<_}D<5Re!@kT}Hu-y8Lp;XsB*h+KyS;N}F0{VrrSxt#LifipYk@pMH@B>L?7u z+Le%vIRZLl0|j5o$R=G5D#yo?ja7d`vdJh-)FB%eV@oziE^VbFNG7HhO38+L6!9Xm z;Y>R#L~!K9R}w=rM?k04jS!mlY-G<%$>FgvG^_k(F|@K&hbrI`M9y>{#A=W%o314mg%)tm>QbOh(rN9iHJ3R9EO` zh3z>ATWKo`Osp)B@C+j^f`vSTJ?R7V@>~rBRXx08j)1zZG78~dotyk2;+z%`mTIi_ zn@~+gpCC=Du}|+Dx3HBqw62Mvb)~^GgrYV-1!V!!$84pU8q85uRAY{SPNs%>)Ke}C z$muC!sm5x*3DsmYp3ps=r}TKxwxIR+p)<()KK*t+c5%>HY%b{ckAeg=FKp z(sokP2WwmsrhqCT8*>CS$cED>hozFL$?1k>na1kA5z~}ve-kkcvC?+j(pK8enmm#Z zq)iDZVZ}uO%+>FI<1&v|#5CpzB$V~CA#~zGRukYdH&GLyfJMz|_50oadF{*O_sOV! zHyhR4-PWHs!Q=97HhtwphURfTy9pKxC;0Pj>HKlG{On9;(|-r^nX_0e?`!ccEP2wC3l zTk6+yf~xZC%@NS?>mh?_zdqJ;b+^^Z0mp8&EcqF3wMv)8K->T6$fd2coh2rAmPnb# z#k{Eh6Xf!Da^=fAfUrE2%vPTQt}F7IXbog)9Sui0cOdzOdKLx zfO*`)R@%&R6En-@jWRq?n5bd$x1~xGHH?9ZGL2Q`_!$pJqu}DW0zrSx9=7#ki^t`3DvCac96`s;bIqG$Tx@WHT&Ih$X49J zo8jf9P?fa;w=Q3Xe~P%HU7EZ9rlDqZjM_k@u&H0^(f!~akW=9TG61* zHMWAXu`Zj7EIzM3e!LI17Dn88hEQx&`bwK&nHMTHb`S{YfW85XuOl@g0oFWyp~JjA zRf95J%!nyvTH&_BU5Rk7H-jiC+|8?2qciMf`*T8!DO7nk@Y7lJrl8|U{knC*=BVB0r$c}SDqUIJG!GVw z9X#Rcw%q3kT>P%zxEfzIhMi_?Hm4_7lhst5_ja>ybi$h++Q7(Y3cVj+b(>;fI8|XU zh3fwlDYYXmzde?O$EMD|KK|+Cv7-@;=&uUaF7}BL5K2n5D4s*Y{%<*c$Ej+5w~K`0 r|6uz91QFY", - "children" : [ { - "name" : "194\\.6\\.133\\.192", - "children" : [ { - "name" : "cluster50213\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50226\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50228\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50217\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50214\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50231\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50232\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50204\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50206\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50203\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50205\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50210\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50208\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50225\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.128", - "children" : [ { - "name" : "cluster1209\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1205\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1235\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1239\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1200\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1227\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1212\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1223\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1217\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1207\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1228\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1218\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1221\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1215\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1226\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1236\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.64", - "children" : [ { - "name" : "cluster50317\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50292\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50291\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50294\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50285\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50300\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50281\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50311\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50297\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50319\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50286\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50307\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50296\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50315\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50316\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.128", - "children" : [ { - "name" : "cluster1859\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1877\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1871\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1876\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1854\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1841\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1858\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1843\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1857\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1842\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1872\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1869\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1853\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1846\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1867\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.130\\.64", - "children" : [ { - "name" : "cluster1976\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1969\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1961\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1963\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1968\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1979\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1967\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1989\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1970\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1999\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.192", - "children" : [ { - "name" : "cluster1150\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1127\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1139\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1154\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1138\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1137\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1130\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1151\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1131\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1141\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1124\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1158\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1140\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1144\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1136\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1157\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1143\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.0", - "children" : [ { - "name" : "cluster1592\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1567\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1594\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1586\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1561\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1585\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1562\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1581\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1566\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1598\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1568\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1560\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1574\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1573\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1583\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1579\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.192", - "children" : [ { - "name" : "cluster50364\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50372\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50365\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50377\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50368\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50396\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50375\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50389\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50382\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.192", - "children" : [ { - "name" : "cluster1533\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1531\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1557\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1555\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1534\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1553\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1550\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1540\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1538\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1520\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1559\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1535\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1525\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1529\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1551\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.64", - "children" : [ { - "name" : "cluster50124\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50159\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50144\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50145\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50133\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50120\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50130\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50142\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50147\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50156\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50125\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50141\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50152\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.192", - "children" : [ { - "name" : "cluster1250\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1276\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1248\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1246\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1251\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1259\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1261\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1260\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1243\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1256\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1272\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1274\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1245\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1249\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.64", - "children" : [ { - "name" : "cluster1041\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1075\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1042\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1078\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1072\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1053\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1056\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1064\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1055\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1070\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1061\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1059\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1040\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.128", - "children" : [ { - "name" : "cluster50171\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50195\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50161\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50191\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50174\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50185\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50177\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50166\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50170\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50179\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.0", - "children" : [ { - "name" : "cluster1283\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1299\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1281\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1288\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1302\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1294\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1289\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1315\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1305\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1316\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.116\\.128", - "children" : [ { - "name" : "cluster1107\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1118\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1080\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1093\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1102\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1104\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1097\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1087\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1095\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1110\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.64", - "children" : [ { - "name" : "cluster3071\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3079\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3068\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3057\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3058\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3070\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3054\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3077\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3049\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3063\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3075\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3065\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3076\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3061\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3073\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3055\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.128", - "children" : [ { - "name" : "cluster50468\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50476\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50440\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50473\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50477\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50460\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50475\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50459\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50447\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50464\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50441\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50444\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.135\\.128", - "children" : [ { - "name" : "cluster3097\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3089\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3111\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3093\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3099\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3106\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3108\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3112\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3085\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3094\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3103\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3098\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3082\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3104\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster3114\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.192", - "children" : [ { - "name" : "cluster50485\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50493\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50510\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50494\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50484\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50481\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50490\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50501\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50478\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50491\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50505\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50488\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50509\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50513\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.133\\.0", - "children" : [ { - "name" : "cluster50085\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50117\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50113\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50101\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50108\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50090\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.0", - "children" : [ { - "name" : "cluster1789\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1777\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1785\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1770\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1793\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1779\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1788\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1776\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1773\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1798\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1762\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1772\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1778\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1782\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1774\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1781\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1760\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1796\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1775\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1768\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1786\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1771\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.130\\.0", - "children" : [ { - "name" : "cluster1959\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1957\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1931\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1920\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1938\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1925\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1932\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1927\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1933\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1930\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1928\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1924\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1953\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1936\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.128", - "children" : [ { - "name" : "cluster1717\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1708\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1707\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1690\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1714\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1683\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1703\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1702\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1694\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1700\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1711\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1713\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1718\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.192", - "children" : [ { - "name" : "cluster1418\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1429\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1420\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1412\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1400\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1415\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1437\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1405\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1427\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.64", - "children" : [ { - "name" : "cluster1334\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1332\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1346\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1350\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1328\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1333\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1321\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1358\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1357\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1356\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.192", - "children" : [ { - "name" : "cluster1914\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1883\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1896\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1911\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1913\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1915\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1903\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1906\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1900\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1891\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1889\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1907\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.192", - "children" : [ { - "name" : "cluster1006\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1035\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1018\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1026\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1020\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1021\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1027\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1031\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1036\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1032\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1029\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1004\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1011\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1008\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1025\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1002\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1030\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1019\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1017\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1028\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.64", - "children" : [ { - "name" : "cluster1445\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1470\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1449\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1462\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1450\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1454\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1466\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1465\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1474\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1444\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1448\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1463\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1457\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1447\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1455\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1442\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1479\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1467\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1446\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.128", - "children" : [ { - "name" : "cluster50034\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50011\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50023\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50025\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50021\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.64", - "children" : [ { - "name" : "cluster1800\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1809\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1816\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1819\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1813\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1806\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1803\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1835\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1822\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1807\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1823\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1832\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.128", - "children" : [ { - "name" : "cluster50359\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50326\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50348\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50325\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50342\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50352\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.126\\.128", - "children" : [ { - "name" : "cluster1383\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1378\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1393\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1395\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1396\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1373\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1388\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1379\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1370\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1368\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1371\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1377\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1369\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.64", - "children" : [ { - "name" : "cluster1643\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1660\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1652\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1672\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1654\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1648\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1657\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1655\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1641\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1669\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1662\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1649\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1666\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1678\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1650\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1679\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.127\\.128", - "children" : [ { - "name" : "cluster1482\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1517\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1491\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1498\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1490\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1504\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1515\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1480\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1518\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1493\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1503\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.129\\.0", - "children" : [ { - "name" : "cluster50520\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50539\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50530\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50526\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50543\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.62\\.192", - "children" : [ { - "name" : "cluster50407\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50409\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50423\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50427\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50429\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50416\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50420\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50418\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50411\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50425\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.134\\.0", - "children" : [ { - "name" : "cluster50275\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50254\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50272\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50274\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50245\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50276\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50243\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50252\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50263\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50279\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50273\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50261\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50260\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.128\\.64", - "children" : [ { - "name" : "cluster1639\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1615\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1628\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1635\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1611\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1607\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1629\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1623\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1633\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1610\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1632\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1614\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1636\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1600\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1626\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1602\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "194\\.6\\.132\\.192", - "children" : [ { - "name" : "cluster50047\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50055\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50051\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50059\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50050\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50076\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50077\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50046\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50053\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50057\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50072\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50044\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50043\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster50058\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.117\\.64", - "children" : [ { - "name" : "cluster1193\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1175\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1185\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1171\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1174\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1167\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1180\\.secondleveldomain\\.com", - "children" : null - } ] - }, { - "name" : "192\\.30\\.63\\.64", - "children" : [ { - "name" : "cluster1755\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1757\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1725\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1727\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1736\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1722\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1752\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1759\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1758\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1732\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1743\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1731\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1733\\.secondleveldomain\\.com", - "children" : null - }, { - "name" : "cluster1751\\.secondleveldomain\\.com", - "children" : null - } ] - } ] -} diff --git a/hadoop-mapreduce-project/src/test/tools/data/rumen/zombie/input-trace.json b/hadoop-mapreduce-project/src/test/tools/data/rumen/zombie/input-trace.json deleted file mode 100644 index 4d820b480b7..00000000000 --- a/hadoop-mapreduce-project/src/test/tools/data/rumen/zombie/input-trace.json +++ /dev/null @@ -1,11364 +0,0 @@ -{ - "priority" : "NORMAL", - "jobID" : "job_200904211745_0001", - "mapTasks" : [ ], - "reduceTasks" : [ ], - "otherTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.192", "cluster1020\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.192/cluster1020\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336740671, - "finishTime" : 1240336743094, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0001_m_000001_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336739206, - "finishTime" : 1240336760537, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0001_m_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "CLEANUP" - } ], - "finishTime" : 1240336760540, - "user" : "geek1", - "jobName" : null, - "computonsPerMapInputByte" : -1, - "computonsPerMapOutputByte" : -1, - "computonsPerReduceInputByte" : -1, - "computonsPerReduceOutputByte" : -1, - "submitTime" : 1240335960685, - "launchTime" : 1240335961050, - "heapMegabytes" : 1024, - "totalMaps" : 1, - "totalReduces" : 1, - "outcome" : "KILLED", - "jobtype" : "JAVA", - "directDependantJobs" : [ ], - "successfulMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 2423, - "minimum" : 2423, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 2423 - }, { - "relativeRanking" : 0.1, - "datum" : 2423 - }, { - "relativeRanking" : 0.15, - "datum" : 2423 - }, { - "relativeRanking" : 0.2, - "datum" : 2423 - }, { - "relativeRanking" : 0.25, - "datum" : 2423 - }, { - "relativeRanking" : 0.3, - "datum" : 2423 - }, { - "relativeRanking" : 0.35, - "datum" : 2423 - }, { - "relativeRanking" : 0.4, - "datum" : 2423 - }, { - "relativeRanking" : 0.45, - "datum" : 2423 - }, { - "relativeRanking" : 0.5, - "datum" : 2423 - }, { - "relativeRanking" : 0.55, - "datum" : 2423 - }, { - "relativeRanking" : 0.6, - "datum" : 2423 - }, { - "relativeRanking" : 0.65, - "datum" : 2423 - }, { - "relativeRanking" : 0.7, - "datum" : 2423 - }, { - "relativeRanking" : 0.75, - "datum" : 2423 - }, { - "relativeRanking" : 0.8, - "datum" : 2423 - }, { - "relativeRanking" : 0.85, - "datum" : 2423 - }, { - "relativeRanking" : 0.9, - "datum" : 2423 - }, { - "relativeRanking" : 0.95, - "datum" : 2423 - } ], - "numberValues" : 1 - } ], - "failedMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - } ], - "successfulReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "failedReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "mapperTriesToSucceed" : [ 1.0 ], - "failedMapperFraction" : 0.0, - "relativeTime" : 0, - "queue" : null, - "clusterMapMB" : -1, - "clusterReduceMB" : -1, - "jobMapMB" : -1, - "jobReduceMB" : -1 -} - { - "priority" : "NORMAL", - "jobID" : "job_200904211745_0002", - "mapTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.128", "cluster1396\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.128/cluster1396\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336647215, - "finishTime" : 1240336651127, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000010_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1369\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1854\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1872\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753705, - "finishTime" : 1240336818108, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000010", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1553\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1553\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336752204, - "finishTime" : 1240336755959, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000017_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.64", "cluster1803\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1930\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1932\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753750, - "finishTime" : 1240336818110, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000017", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1771\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336766565, - "finishTime" : 1240336770163, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000019_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.192", "cluster50481\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50505\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.128", "cluster50359\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753796, - "finishTime" : 1240336818106, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000019", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1779\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1779\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336765887, - "finishTime" : 1240336785565, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000005_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1245\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1261\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.128", "cluster50021\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753840, - "finishTime" : 1240336818114, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000005", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.128", "cluster1218\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.128/cluster1218\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336755247, - "finishTime" : 1240336763432, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000000_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1236\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1889\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1911\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753888, - "finishTime" : 1240336818113, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1586\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1586\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758229, - "finishTime" : 1240336761612, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000009_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1227\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1259\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1260\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753938, - "finishTime" : 1240336818116, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000009", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "cluster1670\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336754665, - "finishTime" : 1240336777673, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000002_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1586\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1586\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336821839, - "finishTime" : 1240336824652, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000002_1", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1357\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1450\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1457\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336753987, - "finishTime" : 1240336842768, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000002", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1869\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1869\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336757380, - "finishTime" : 1240336763176, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000004_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1294\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.0", "cluster1288\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1876\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754030, - "finishTime" : 1240336818118, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000004", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1770\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1770\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336763994, - "finishTime" : 1240336767750, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000007_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1466\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50218\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50232\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754077, - "finishTime" : 1240336818119, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000007", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1816\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1816\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758341, - "finishTime" : 1240336762025, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000013_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.134\\.64", "cluster50286\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3071\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3049\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754124, - "finishTime" : 1240336818120, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000013", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3097\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3097\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336768390, - "finishTime" : 1240336771627, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000012_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.132\\.192", "cluster50057\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3112\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3082\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754176, - "finishTime" : 1240336774482, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000012", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1660\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1660\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336754801, - "finishTime" : 1240336758231, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000001_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1080\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.128", "cluster1097\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.0", "cluster50543\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754223, - "finishTime" : 1240336774476, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3104\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3104\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336762622, - "finishTime" : 1240336765834, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000015_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.192", "cluster1412\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50427\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50411\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754270, - "finishTime" : 1240336774468, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000015", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1867\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1867\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336759528, - "finishTime" : 1240336764456, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000016_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1393\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50130\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50141\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754319, - "finishTime" : 1240336817903, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000016", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1649\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1649\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336759909, - "finishTime" : 1240336763727, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000018_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1223\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1200\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50152\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754366, - "finishTime" : 1240336818420, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000018", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1551\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1551\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336755433, - "finishTime" : 1240336758966, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000003_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.64", "cluster1733\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1607\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1639\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754409, - "finishTime" : 1240336816560, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000003", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1405\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1405\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336747936, - "finishTime" : 1240336751464, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000006_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1064\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50510\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50478\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754452, - "finishTime" : 1240336817330, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000006", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.0", "cluster1928\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.0/cluster1928\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336756161, - "finishTime" : 1240336760016, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000014_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1059\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50053\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50050\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754500, - "finishTime" : 1240336824060, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000014", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1846\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1846\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336758220, - "finishTime" : 1240336762903, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000008_0", - "hdfsBytesRead" : 148286, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1157\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1718\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1694\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754548, - "finishTime" : 1240336824556, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000008", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3098\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3098\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336744261, - "finishTime" : 1240336747517, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000011_0", - "hdfsBytesRead" : 53639, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 37170, - "mapInputRecords" : 3601, - "mapOutputBytes" : 247925, - "mapOutputRecords" : 26425, - "combineInputRecords" : 26425, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5315, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1143\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1004\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1020\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336754596, - "finishTime" : 1240336776031, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000011", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - } ], - "reduceTasks" : [ { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336786769, - "finishTime" : 1240336870553, - "shuffleFinished" : 1240336859759, - "sortFinished" : 1240336860092, - "attemptID" : "attempt_200904211745_0002_r_000000_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 56630, - "fileBytesRead" : 71200, - "fileBytesWritten" : 71200, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 0, - "reduceInputRecords" : 106300, - "reduceShuffleBytes" : 705622, - "reduceOutputRecords" : 0, - "spilledRecords" : 106300, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336774548, - "finishTime" : 1240336873648, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_r_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - } ], - "otherTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.192", "cluster1028\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.192/cluster1028\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336739565, - "finishTime" : 1240336742217, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000021_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336739508, - "finishTime" : 1240336752812, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000021", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "SETUP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1771\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1771\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336885885, - "finishTime" : 1240336887642, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0002_m_000020_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336873651, - "finishTime" : 1240336889658, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0002_m_000020", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "CLEANUP" - } ], - "finishTime" : 1240336889659, - "user" : "hadoopqa", - "jobName" : null, - "computonsPerMapInputByte" : -1, - "computonsPerMapOutputByte" : -1, - "computonsPerReduceInputByte" : -1, - "computonsPerReduceOutputByte" : -1, - "submitTime" : 1240335962848, - "launchTime" : 1240335964437, - "heapMegabytes" : 640, - "totalMaps" : 20, - "totalReduces" : 1, - "outcome" : "SUCCESS", - "jobtype" : "JAVA", - "directDependantJobs" : [ ], - "successfulMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 8185, - "minimum" : 3237, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 3237 - }, { - "relativeRanking" : 0.1, - "datum" : 3237 - }, { - "relativeRanking" : 0.15, - "datum" : 3237 - }, { - "relativeRanking" : 0.2, - "datum" : 3237 - }, { - "relativeRanking" : 0.25, - "datum" : 3237 - }, { - "relativeRanking" : 0.3, - "datum" : 3237 - }, { - "relativeRanking" : 0.35, - "datum" : 3237 - }, { - "relativeRanking" : 0.4, - "datum" : 3237 - }, { - "relativeRanking" : 0.45, - "datum" : 3237 - }, { - "relativeRanking" : 0.5, - "datum" : 3912 - }, { - "relativeRanking" : 0.55, - "datum" : 3912 - }, { - "relativeRanking" : 0.6, - "datum" : 3912 - }, { - "relativeRanking" : 0.65, - "datum" : 3912 - }, { - "relativeRanking" : 0.7, - "datum" : 3912 - }, { - "relativeRanking" : 0.75, - "datum" : 5796 - }, { - "relativeRanking" : 0.8, - "datum" : 5796 - }, { - "relativeRanking" : 0.85, - "datum" : 5796 - }, { - "relativeRanking" : 0.9, - "datum" : 5796 - }, { - "relativeRanking" : 0.95, - "datum" : 5796 - } ], - "numberValues" : 4 - }, { - "maximum" : 19678, - "minimum" : 2813, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 2813 - }, { - "relativeRanking" : 0.1, - "datum" : 2813 - }, { - "relativeRanking" : 0.15, - "datum" : 3212 - }, { - "relativeRanking" : 0.2, - "datum" : 3256 - }, { - "relativeRanking" : 0.25, - "datum" : 3383 - }, { - "relativeRanking" : 0.3, - "datum" : 3383 - }, { - "relativeRanking" : 0.35, - "datum" : 3430 - }, { - "relativeRanking" : 0.4, - "datum" : 3528 - }, { - "relativeRanking" : 0.45, - "datum" : 3533 - }, { - "relativeRanking" : 0.5, - "datum" : 3598 - }, { - "relativeRanking" : 0.55, - "datum" : 3598 - }, { - "relativeRanking" : 0.6, - "datum" : 3684 - }, { - "relativeRanking" : 0.65, - "datum" : 3755 - }, { - "relativeRanking" : 0.7, - "datum" : 3756 - }, { - "relativeRanking" : 0.75, - "datum" : 3818 - }, { - "relativeRanking" : 0.8, - "datum" : 3818 - }, { - "relativeRanking" : 0.85, - "datum" : 3855 - }, { - "relativeRanking" : 0.9, - "datum" : 4683 - }, { - "relativeRanking" : 0.95, - "datum" : 4928 - } ], - "numberValues" : 16 - }, { - "maximum" : 2652, - "minimum" : 1757, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 1757 - }, { - "relativeRanking" : 0.1, - "datum" : 1757 - }, { - "relativeRanking" : 0.15, - "datum" : 1757 - }, { - "relativeRanking" : 0.2, - "datum" : 1757 - }, { - "relativeRanking" : 0.25, - "datum" : 1757 - }, { - "relativeRanking" : 0.3, - "datum" : 1757 - }, { - "relativeRanking" : 0.35, - "datum" : 1757 - }, { - "relativeRanking" : 0.4, - "datum" : 1757 - }, { - "relativeRanking" : 0.45, - "datum" : 1757 - }, { - "relativeRanking" : 0.5, - "datum" : 1757 - }, { - "relativeRanking" : 0.55, - "datum" : 1757 - }, { - "relativeRanking" : 0.6, - "datum" : 1757 - }, { - "relativeRanking" : 0.65, - "datum" : 1757 - }, { - "relativeRanking" : 0.7, - "datum" : 1757 - }, { - "relativeRanking" : 0.75, - "datum" : 1757 - }, { - "relativeRanking" : 0.8, - "datum" : 1757 - }, { - "relativeRanking" : 0.85, - "datum" : 1757 - }, { - "relativeRanking" : 0.9, - "datum" : 1757 - }, { - "relativeRanking" : 0.95, - "datum" : 1757 - } ], - "numberValues" : 2 - } ], - "failedMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 23008, - "minimum" : 23008, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 23008 - }, { - "relativeRanking" : 0.1, - "datum" : 23008 - }, { - "relativeRanking" : 0.15, - "datum" : 23008 - }, { - "relativeRanking" : 0.2, - "datum" : 23008 - }, { - "relativeRanking" : 0.25, - "datum" : 23008 - }, { - "relativeRanking" : 0.3, - "datum" : 23008 - }, { - "relativeRanking" : 0.35, - "datum" : 23008 - }, { - "relativeRanking" : 0.4, - "datum" : 23008 - }, { - "relativeRanking" : 0.45, - "datum" : 23008 - }, { - "relativeRanking" : 0.5, - "datum" : 23008 - }, { - "relativeRanking" : 0.55, - "datum" : 23008 - }, { - "relativeRanking" : 0.6, - "datum" : 23008 - }, { - "relativeRanking" : 0.65, - "datum" : 23008 - }, { - "relativeRanking" : 0.7, - "datum" : 23008 - }, { - "relativeRanking" : 0.75, - "datum" : 23008 - }, { - "relativeRanking" : 0.8, - "datum" : 23008 - }, { - "relativeRanking" : 0.85, - "datum" : 23008 - }, { - "relativeRanking" : 0.9, - "datum" : 23008 - }, { - "relativeRanking" : 0.95, - "datum" : 23008 - } ], - "numberValues" : 1 - } ], - "successfulReduceAttemptCDF" : { - "maximum" : 83784, - "minimum" : 83784, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 83784 - }, { - "relativeRanking" : 0.1, - "datum" : 83784 - }, { - "relativeRanking" : 0.15, - "datum" : 83784 - }, { - "relativeRanking" : 0.2, - "datum" : 83784 - }, { - "relativeRanking" : 0.25, - "datum" : 83784 - }, { - "relativeRanking" : 0.3, - "datum" : 83784 - }, { - "relativeRanking" : 0.35, - "datum" : 83784 - }, { - "relativeRanking" : 0.4, - "datum" : 83784 - }, { - "relativeRanking" : 0.45, - "datum" : 83784 - }, { - "relativeRanking" : 0.5, - "datum" : 83784 - }, { - "relativeRanking" : 0.55, - "datum" : 83784 - }, { - "relativeRanking" : 0.6, - "datum" : 83784 - }, { - "relativeRanking" : 0.65, - "datum" : 83784 - }, { - "relativeRanking" : 0.7, - "datum" : 83784 - }, { - "relativeRanking" : 0.75, - "datum" : 83784 - }, { - "relativeRanking" : 0.8, - "datum" : 83784 - }, { - "relativeRanking" : 0.85, - "datum" : 83784 - }, { - "relativeRanking" : 0.9, - "datum" : 83784 - }, { - "relativeRanking" : 0.95, - "datum" : 83784 - } ], - "numberValues" : 1 - }, - "failedReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "mapperTriesToSucceed" : [ 0.9565217391304348, 0.043478260869565216 ], - "failedMapperFraction" : 0.0, - "relativeTime" : 0, - "queue" : null, - "clusterMapMB" : -1, - "clusterReduceMB" : -1, - "jobMapMB" : -1, - "jobReduceMB" : -1 -} - { - "priority" : "NORMAL", - "jobID" : "job_200904211745_0003", - "mapTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.192", "cluster50494\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.192/cluster50494\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336870159, - "finishTime" : 1240336876906, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0003_m_000000_0", - "hdfsBytesRead" : 40, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 120, - "mapInputRecords" : 6, - "mapOutputBytes" : 64, - "mapOutputRecords" : 6, - "combineInputRecords" : 6, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 6, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.64", "cluster1823\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.64", "cluster1800\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1979\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336862787, - "finishTime" : 1240336878867, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_m_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.192", "cluster50396\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.192/cluster50396\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336863528, - "finishTime" : 1240336878419, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0003_m_000002_0", - "hdfsBytesRead" : 29, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 100, - "mapInputRecords" : 4, - "mapOutputBytes" : 45, - "mapOutputRecords" : 4, - "combineInputRecords" : 4, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1457\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1463\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50144\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336862891, - "finishTime" : 1240336878973, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_m_000002", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.192", "cluster50416\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.192/cluster50416\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336863725, - "finishTime" : 1240336868610, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0003_m_000001_0", - "hdfsBytesRead" : 36, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 109, - "mapInputRecords" : 5, - "mapOutputBytes" : 56, - "mapOutputRecords" : 5, - "combineInputRecords" : 5, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1346\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.64", "cluster1333\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50476\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336862992, - "finishTime" : 1240336879103, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_m_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - } ], - "reduceTasks" : [ { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.192/cluster50225\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879990, - "finishTime" : 1240336912326, - "shuffleFinished" : 1240336906008, - "sortFinished" : 1240336906089, - "attemptID" : "attempt_200904211745_0003_r_000000_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 61, - "fileBytesRead" : 112, - "fileBytesWritten" : 112, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 0, - "reduceInputRecords" : 15, - "reduceShuffleBytes" : 233, - "reduceOutputRecords" : 0, - "spilledRecords" : 15, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336879303, - "finishTime" : 1240336928223, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_r_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - } ], - "otherTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.0", "cluster50261\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.0/cluster50261\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336846902, - "finishTime" : 1240336858206, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0003_m_000004_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336846190, - "finishTime" : 1240336862349, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_m_000004", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "SETUP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.192", "cluster50225\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.192/cluster50225\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336928353, - "finishTime" : 1240336930147, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0003_m_000003_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336928224, - "finishTime" : 1240336944237, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0003_m_000003", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "CLEANUP" - } ], - "finishTime" : 1240336944238, - "user" : "geek2", - "jobName" : null, - "computonsPerMapInputByte" : -1, - "computonsPerMapOutputByte" : -1, - "computonsPerReduceInputByte" : -1, - "computonsPerReduceOutputByte" : -1, - "submitTime" : 1240336843916, - "launchTime" : 1240336844448, - "heapMegabytes" : 640, - "totalMaps" : 3, - "totalReduces" : 1, - "outcome" : "SUCCESS", - "jobtype" : "JAVA", - "directDependantJobs" : [ ], - "successfulMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 14891, - "minimum" : 4885, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 4885 - }, { - "relativeRanking" : 0.1, - "datum" : 4885 - }, { - "relativeRanking" : 0.15, - "datum" : 4885 - }, { - "relativeRanking" : 0.2, - "datum" : 4885 - }, { - "relativeRanking" : 0.25, - "datum" : 4885 - }, { - "relativeRanking" : 0.3, - "datum" : 4885 - }, { - "relativeRanking" : 0.35, - "datum" : 4885 - }, { - "relativeRanking" : 0.4, - "datum" : 4885 - }, { - "relativeRanking" : 0.45, - "datum" : 4885 - }, { - "relativeRanking" : 0.5, - "datum" : 4885 - }, { - "relativeRanking" : 0.55, - "datum" : 4885 - }, { - "relativeRanking" : 0.6, - "datum" : 4885 - }, { - "relativeRanking" : 0.65, - "datum" : 4885 - }, { - "relativeRanking" : 0.7, - "datum" : 6747 - }, { - "relativeRanking" : 0.75, - "datum" : 6747 - }, { - "relativeRanking" : 0.8, - "datum" : 6747 - }, { - "relativeRanking" : 0.85, - "datum" : 6747 - }, { - "relativeRanking" : 0.9, - "datum" : 6747 - }, { - "relativeRanking" : 0.95, - "datum" : 6747 - } ], - "numberValues" : 3 - }, { - "maximum" : 11304, - "minimum" : 1794, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 1794 - }, { - "relativeRanking" : 0.1, - "datum" : 1794 - }, { - "relativeRanking" : 0.15, - "datum" : 1794 - }, { - "relativeRanking" : 0.2, - "datum" : 1794 - }, { - "relativeRanking" : 0.25, - "datum" : 1794 - }, { - "relativeRanking" : 0.3, - "datum" : 1794 - }, { - "relativeRanking" : 0.35, - "datum" : 1794 - }, { - "relativeRanking" : 0.4, - "datum" : 1794 - }, { - "relativeRanking" : 0.45, - "datum" : 1794 - }, { - "relativeRanking" : 0.5, - "datum" : 1794 - }, { - "relativeRanking" : 0.55, - "datum" : 1794 - }, { - "relativeRanking" : 0.6, - "datum" : 1794 - }, { - "relativeRanking" : 0.65, - "datum" : 1794 - }, { - "relativeRanking" : 0.7, - "datum" : 1794 - }, { - "relativeRanking" : 0.75, - "datum" : 1794 - }, { - "relativeRanking" : 0.8, - "datum" : 1794 - }, { - "relativeRanking" : 0.85, - "datum" : 1794 - }, { - "relativeRanking" : 0.9, - "datum" : 1794 - }, { - "relativeRanking" : 0.95, - "datum" : 1794 - } ], - "numberValues" : 2 - } ], - "failedMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - } ], - "successfulReduceAttemptCDF" : { - "maximum" : 32336, - "minimum" : 32336, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 32336 - }, { - "relativeRanking" : 0.1, - "datum" : 32336 - }, { - "relativeRanking" : 0.15, - "datum" : 32336 - }, { - "relativeRanking" : 0.2, - "datum" : 32336 - }, { - "relativeRanking" : 0.25, - "datum" : 32336 - }, { - "relativeRanking" : 0.3, - "datum" : 32336 - }, { - "relativeRanking" : 0.35, - "datum" : 32336 - }, { - "relativeRanking" : 0.4, - "datum" : 32336 - }, { - "relativeRanking" : 0.45, - "datum" : 32336 - }, { - "relativeRanking" : 0.5, - "datum" : 32336 - }, { - "relativeRanking" : 0.55, - "datum" : 32336 - }, { - "relativeRanking" : 0.6, - "datum" : 32336 - }, { - "relativeRanking" : 0.65, - "datum" : 32336 - }, { - "relativeRanking" : 0.7, - "datum" : 32336 - }, { - "relativeRanking" : 0.75, - "datum" : 32336 - }, { - "relativeRanking" : 0.8, - "datum" : 32336 - }, { - "relativeRanking" : 0.85, - "datum" : 32336 - }, { - "relativeRanking" : 0.9, - "datum" : 32336 - }, { - "relativeRanking" : 0.95, - "datum" : 32336 - } ], - "numberValues" : 1 - }, - "failedReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "mapperTriesToSucceed" : [ 1.0 ], - "failedMapperFraction" : 0.0, - "relativeTime" : 0, - "queue" : null, - "clusterMapMB" : -1, - "clusterReduceMB" : -1, - "jobMapMB" : -1, - "jobReduceMB" : -1 -} - { - "priority" : "NORMAL", - "jobID" : "job_200904211745_0004", - "mapTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.128", "cluster1702\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.128/cluster1702\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873318, - "finishTime" : 1240336916053, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000024_0", - "hdfsBytesRead" : 64208964, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2476698, - "mapInputRecords" : 4024803, - "mapOutputBytes" : 4143839, - "mapOutputRecords" : 64187, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 64187, - "mapInputBytes" : 249623354 - }, { - "location" : null, - "hostName" : "cluster1687\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915251, - "finishTime" : 1240336948507, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000024_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.128", "cluster1683\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1025\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1030\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872259, - "finishTime" : 1240336920373, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000024", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.64", "cluster1185\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.64/cluster1185\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875106, - "finishTime" : 1240336929081, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000007_0", - "hdfsBytesRead" : 67416667, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2601457, - "mapInputRecords" : 4254279, - "mapOutputBytes" : 4350539, - "mapOutputRecords" : 67759, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 67759, - "mapInputBytes" : 263667529 - }, { - "location" : null, - "hostName" : "cluster1195\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336918908, - "finishTime" : 1240336988786, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000007_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.64", "cluster1174\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.64", "cluster1171\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1600\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872260, - "finishTime" : 1240336936418, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000007", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.128", "cluster1209\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.128/cluster1209\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874012, - "finishTime" : 1240336912483, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000037_0", - "hdfsBytesRead" : 62083938, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2407514, - "mapInputRecords" : 3862317, - "mapOutputBytes" : 4057593, - "mapOutputRecords" : 62680, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62680, - "mapInputBytes" : 241895332 - }, { - "location" : null, - "hostName" : "cluster1227\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915459, - "finishTime" : 1240336949436, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000037_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1107\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1215\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1226\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872265, - "finishTime" : 1240336920359, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000037", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.128", "cluster1110\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.128/cluster1110\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871992, - "finishTime" : 1240336912072, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000015_0", - "hdfsBytesRead" : 65491754, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2477099, - "mapInputRecords" : 4163656, - "mapOutputBytes" : 4140319, - "mapOutputRecords" : 64248, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 64248, - "mapInputBytes" : 256585350 - }, { - "location" : null, - "hostName" : "cluster1825\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336921125, - "finishTime" : 1240336954463, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000015_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1104\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1465\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1462\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872268, - "finishTime" : 1240336920434, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000015", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.64", "cluster1755\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.64/cluster1755\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876311, - "finishTime" : 1240336942586, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000041_0", - "hdfsBytesRead" : 33589630, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 35182817, - "mapInputRecords" : 1398039, - "mapOutputBytes" : 74392390, - "mapOutputRecords" : 1217208, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1217208, - "mapInputBytes" : 103351186 - }, { - "location" : null, - "hostName" : "cluster1441\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916237, - "finishTime" : 1240336981644, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000041_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1346\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.64", "cluster1332\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1759\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872270, - "finishTime" : 1240336952476, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000041", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1534\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1534\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873710, - "finishTime" : 1240336886646, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000115_0", - "hdfsBytesRead" : 7802358, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 61768, - "mapInputRecords" : 338528, - "mapOutputBytes" : 117149, - "mapOutputRecords" : 1831, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1831, - "mapInputBytes" : 25534769 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1070\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.64", "cluster1041\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1534\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872271, - "finishTime" : 1240336888399, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000115", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1657\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1657\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872974, - "finishTime" : 1240336912238, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000022_0", - "hdfsBytesRead" : 64534282, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2454984, - "mapInputRecords" : 4070141, - "mapOutputBytes" : 4111780, - "mapOutputRecords" : 64137, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 64137, - "mapInputBytes" : 257286422 - }, { - "location" : null, - "hostName" : "cluster1973\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917218, - "finishTime" : 1240336950542, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000022_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1657\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1782\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1762\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872273, - "finishTime" : 1240336920438, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000022", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1835\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1835\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874481, - "finishTime" : 1240336949882, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000045_0", - "hdfsBytesRead" : 30958862, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2445, - "mapInputRecords" : 2212648, - "mapOutputBytes" : 508, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 134455678 - }, { - "location" : null, - "hostName" : "cluster1804\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917613, - "finishTime" : 1240336951416, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000045_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.64", "cluster1819\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.64", "cluster1809\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50316\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872275, - "finishTime" : 1240336952483, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000045", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.0", "cluster50273\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.0/cluster50273\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872929, - "finishTime" : 1240336917687, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000000_0", - "hdfsBytesRead" : 70551688, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2705106, - "mapInputRecords" : 4456455, - "mapOutputBytes" : 4576941, - "mapOutputRecords" : 71582, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 71582, - "mapInputBytes" : 275178411 - }, { - "location" : null, - "hostName" : "cluster50279\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915037, - "finishTime" : 1240336948454, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000000_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.128", "cluster50191\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50252\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50263\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872276, - "finishTime" : 1240336920429, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "cluster1484\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336873052, - "finishTime" : 1240336891468, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000026_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : { - "layers" : [ "194\\.6\\.133\\.64", "cluster50159\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.64/cluster50159\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336905498, - "finishTime" : 1240336942729, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000026_1", - "hdfsBytesRead" : 63978817, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2435223, - "mapInputRecords" : 4013967, - "mapOutputBytes" : 4114611, - "mapOutputRecords" : 63929, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63929, - "mapInputBytes" : 248841916 - }, { - "location" : null, - "hostName" : "cluster50025\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916380, - "finishTime" : 1240336958665, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000026_2", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1504\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1035\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1008\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872278, - "finishTime" : 1240336952813, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000026", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.64", "cluster50125\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.64/cluster50125\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872976, - "finishTime" : 1240336915820, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000012_0", - "hdfsBytesRead" : 65683422, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2514927, - "mapInputRecords" : 4135021, - "mapOutputBytes" : 4224631, - "mapOutputRecords" : 66548, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 66548, - "mapInputBytes" : 255713201 - }, { - "location" : null, - "hostName" : "cluster1283\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914458, - "finishTime" : 1240336947756, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000012_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1299\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.0", "cluster1315\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50147\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872299, - "finishTime" : 1240336920463, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000012", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50285\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50285\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872964, - "finishTime" : 1240336939111, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000006_0", - "hdfsBytesRead" : 67732698, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2596196, - "mapInputRecords" : 4299703, - "mapOutputBytes" : 4415200, - "mapOutputRecords" : 70202, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 70202, - "mapInputBytes" : 264286110 - }, { - "location" : null, - "hostName" : "cluster50282\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915054, - "finishTime" : 1240336955710, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000006_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.0", "cluster1781\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50307\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50311\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872300, - "finishTime" : 1240336952474, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000006", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.192", "cluster50204\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.192/cluster50204\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872970, - "finishTime" : 1240336919341, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000002_0", - "hdfsBytesRead" : 68694241, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2641160, - "mapInputRecords" : 4318855, - "mapOutputBytes" : 4523968, - "mapOutputRecords" : 71535, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 71535, - "mapInputBytes" : 268342920 - }, { - "location" : null, - "hostName" : "cluster1894\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915343, - "finishTime" : 1240336956992, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000002_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.192", "cluster1896\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50213\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50228\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872301, - "finishTime" : 1240336920455, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000002", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1568\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1568\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336918628, - "finishTime" : 1240336945487, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000060_1", - "hdfsBytesRead" : 30143641, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2581, - "mapInputRecords" : 2183668, - "mapOutputBytes" : 619, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 128698969 - }, { - "location" : null, - "hostName" : "cluster1560\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336873218, - "finishTime" : 1240336938166, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000060_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.0", "cluster1581\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1560\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50276\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872374, - "finishTime" : 1240336946441, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000060", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.128", "cluster1207\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.128/cluster1207\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336867477, - "finishTime" : 1240336891579, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000068_0", - "hdfsBytesRead" : 29914824, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2149, - "mapInputRecords" : 2146323, - "mapOutputBytes" : 176, - "mapOutputRecords" : 3, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 3, - "mapInputBytes" : 127372829 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1239\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1217\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1479\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872374, - "finishTime" : 1240336904545, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000068", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.192", "cluster1911\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.192/cluster1911\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875928, - "finishTime" : 1240336914911, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000008_0", - "hdfsBytesRead" : 67080786, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2606104, - "mapInputRecords" : 4182439, - "mapOutputBytes" : 4393005, - "mapOutputRecords" : 68146, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 68146, - "mapInputBytes" : 260552004 - }, { - "location" : null, - "hostName" : "cluster50001\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914959, - "finishTime" : 1240336959329, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000008_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.192", "cluster1914\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1911\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.128", "cluster50011\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872375, - "finishTime" : 1240336920488, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000008", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1858\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1858\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874685, - "finishTime" : 1240336917858, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000010_0", - "hdfsBytesRead" : 66510009, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2504720, - "mapInputRecords" : 4229507, - "mapOutputBytes" : 4193214, - "mapOutputRecords" : 65150, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65150, - "mapInputBytes" : 261348815 - }, { - "location" : null, - "hostName" : "cluster50030\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914970, - "finishTime" : 1240336948391, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000010_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.64", "cluster1180\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.64", "cluster1171\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1857\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872376, - "finishTime" : 1240336920531, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000010", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.192", "cluster1021\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.192/cluster1021\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872982, - "finishTime" : 1240336912193, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000019_0", - "hdfsBytesRead" : 64941817, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2419356, - "mapInputRecords" : 4085619, - "mapOutputBytes" : 4074644, - "mapOutputRecords" : 63304, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63304, - "mapInputBytes" : 254958753 - }, { - "location" : null, - "hostName" : "cluster1576\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336919809, - "finishTime" : 1240336953416, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000019_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1006\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1011\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1573\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872376, - "finishTime" : 1240336920515, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000019", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.128", "cluster1700\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.128/cluster1700\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336869803, - "finishTime" : 1240336893577, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000047_0", - "hdfsBytesRead" : 30559058, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2349, - "mapInputRecords" : 2184357, - "mapOutputBytes" : 444, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 129670187 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.128", "cluster1717\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1713\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50389\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872377, - "finishTime" : 1240336904552, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000047", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1457\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1457\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871305, - "finishTime" : 1240336910899, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000017_0", - "hdfsBytesRead" : 65262059, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2517094, - "mapInputRecords" : 4042279, - "mapOutputBytes" : 4239279, - "mapOutputRecords" : 66022, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 66022, - "mapInputBytes" : 252581214 - }, { - "location" : null, - "hostName" : "cluster50539\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915026, - "finishTime" : 1240336948436, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000017_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1454\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1466\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.0", "cluster50539\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872377, - "finishTime" : 1240336920534, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000017", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1598\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1598\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336862833, - "finishTime" : 1240336886002, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000081_0", - "hdfsBytesRead" : 29655169, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2492, - "mapInputRecords" : 2160857, - "mapOutputBytes" : 519, - "mapOutputRecords" : 6, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 6, - "mapInputBytes" : 128065621 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.0", "cluster1566\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1598\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1626\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872378, - "finishTime" : 1240336904475, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000081", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1592\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1592\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876693, - "finishTime" : 1240336916107, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000021_0", - "hdfsBytesRead" : 64599955, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2454477, - "mapInputRecords" : 4026318, - "mapOutputBytes" : 4137511, - "mapOutputRecords" : 63818, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63818, - "mapInputBytes" : 251469382 - }, { - "location" : null, - "hostName" : "cluster1564\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917825, - "finishTime" : 1240336951212, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000021_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1127\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1131\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1562\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872379, - "finishTime" : 1240336920487, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000021", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3106\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3106\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879219, - "finishTime" : 1240336913954, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000003_0", - "hdfsBytesRead" : 68427487, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2623342, - "mapInputRecords" : 4252689, - "mapOutputBytes" : 4454505, - "mapOutputRecords" : 70042, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 70042, - "mapInputBytes" : 264536055 - }, { - "location" : null, - "hostName" : "cluster1829\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917410, - "finishTime" : 1240336950698, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000003_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1672\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3089\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3114\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872379, - "finishTime" : 1240336920465, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000003", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.64", "cluster1334\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.64/cluster1334\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874371, - "finishTime" : 1240336916964, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000013_0", - "hdfsBytesRead" : 65647050, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2545518, - "mapInputRecords" : 4090204, - "mapOutputBytes" : 4275432, - "mapOutputRecords" : 66126, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 66126, - "mapInputBytes" : 255910823 - }, { - "location" : null, - "hostName" : "cluster1410\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914216, - "finishTime" : 1240336963861, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000013_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.192", "cluster1415\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.192", "cluster1412\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.64", "cluster1332\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872380, - "finishTime" : 1240336920509, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000013", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1442\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1442\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871928, - "finishTime" : 1240336911159, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000033_0", - "hdfsBytesRead" : 62843496, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2438028, - "mapInputRecords" : 3993110, - "mapOutputBytes" : 4086319, - "mapOutputRecords" : 63785, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63785, - "mapInputBytes" : 246369440 - }, { - "location" : null, - "hostName" : "cluster1941\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336909373, - "finishTime" : 1240336966403, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000033_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1356\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.64", "cluster1350\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1455\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872380, - "finishTime" : 1240336920485, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000033", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "cluster1751\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336873015, - "finishTime" : 1240336891599, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000046_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : { - "layers" : [ "194\\.6\\.134\\.192", "cluster50368\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.192/cluster50368\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336905058, - "finishTime" : 1240336926560, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000046_1", - "hdfsBytesRead" : 30599610, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2489, - "mapInputRecords" : 2280773, - "mapOutputBytes" : 658, - "mapOutputRecords" : 12, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 12, - "mapInputBytes" : 133157243 - }, { - "location" : null, - "hostName" : "cluster1247\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916005, - "finishTime" : 1240336942980, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000046_2", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.64", "cluster1757\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1751\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50292\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872381, - "finishTime" : 1240336936977, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000046", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1777\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1777\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336878797, - "finishTime" : 1240336901414, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000059_0", - "hdfsBytesRead" : 30176905, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2160, - "mapInputRecords" : 2227855, - "mapOutputBytes" : 210, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 130221301 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.192", "cluster50425\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50407\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1774\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872381, - "finishTime" : 1240336904493, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000059", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.192", "cluster1027\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.192/cluster1027\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873401, - "finishTime" : 1240336909982, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000038_0", - "hdfsBytesRead" : 61899686, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2392428, - "mapInputRecords" : 3861737, - "mapOutputBytes" : 3999150, - "mapOutputRecords" : 61718, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 61718, - "mapInputBytes" : 240872637 - }, { - "location" : null, - "hostName" : "cluster3117\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917616, - "finishTime" : 1240336952203, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000038_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1002\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1027\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3093\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872382, - "finishTime" : 1240336920531, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000038", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.128", "cluster1093\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.128/cluster1093\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336877844, - "finishTime" : 1240336902594, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000066_0", - "hdfsBytesRead" : 29972278, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2311, - "mapInputRecords" : 2151755, - "mapOutputBytes" : 476, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 127537268 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1118\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1026\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1031\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872383, - "finishTime" : 1240336904501, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000066", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1632\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1632\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874600, - "finishTime" : 1240336886276, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000098_0", - "hdfsBytesRead" : 8241095, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 53850, - "mapInputRecords" : 359279, - "mapOutputBytes" : 97992, - "mapOutputRecords" : 1544, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1544, - "mapInputBytes" : 26848771 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.64", "cluster1758\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1614\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1632\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872383, - "finishTime" : 1240336888503, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000098", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1785\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1785\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879910, - "finishTime" : 1240336910063, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000062_0", - "hdfsBytesRead" : 30075619, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2239, - "mapInputRecords" : 2131858, - "mapOutputBytes" : 300, - "mapOutputRecords" : 6, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 6, - "mapInputBytes" : 126857339 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1357\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1776\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1773\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872384, - "finishTime" : 1240336904537, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000062", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1626\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1626\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876699, - "finishTime" : 1240336914881, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000025_0", - "hdfsBytesRead" : 64200239, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2529950, - "mapInputRecords" : 3996858, - "mapOutputBytes" : 4239266, - "mapOutputRecords" : 65959, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65959, - "mapInputBytes" : 248767046 - }, { - "location" : null, - "hostName" : "cluster1601\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336918879, - "finishTime" : 1240336952244, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000025_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.64", "cluster1633\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1611\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50275\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872384, - "finishTime" : 1240336920528, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000025", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1628\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1628\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336881912, - "finishTime" : 1240336906234, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000079_0", - "hdfsBytesRead" : 29704868, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2212, - "mapInputRecords" : 2170913, - "mapOutputBytes" : 222, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 127927659 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1490\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.128", "cluster1482\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1636\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872385, - "finishTime" : 1240336904599, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000079", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1470\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1470\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871928, - "finishTime" : 1240336893040, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000088_0", - "hdfsBytesRead" : 8927130, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 63070, - "mapInputRecords" : 416911, - "mapOutputBytes" : 119856, - "mapOutputRecords" : 2051, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2051, - "mapInputBytes" : 29933360 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1474\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1479\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1927\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872385, - "finishTime" : 1240336904691, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000088", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1662\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1662\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336880282, - "finishTime" : 1240336919548, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000011_0", - "hdfsBytesRead" : 66183568, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2597810, - "mapInputRecords" : 4149962, - "mapOutputBytes" : 4367215, - "mapOutputRecords" : 67906, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 67906, - "mapInputBytes" : 256962052 - }, { - "location" : null, - "hostName" : "cluster50234\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915110, - "finishTime" : 1240336948409, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000011_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1652\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3065\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3076\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872386, - "finishTime" : 1240336920520, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000011", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1623\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1623\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879066, - "finishTime" : 1240336901295, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000087_0", - "hdfsBytesRead" : 29290661, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2299, - "mapInputRecords" : 2093812, - "mapOutputBytes" : 307, - "mapOutputRecords" : 5, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5, - "mapInputBytes" : 124249977 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1008\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1020\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1602\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872387, - "finishTime" : 1240336904528, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000087", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1535\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1535\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874347, - "finishTime" : 1240336896706, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000057_0", - "hdfsBytesRead" : 30235345, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2182, - "mapInputRecords" : 2261814, - "mapOutputBytes" : 234, - "mapOutputRecords" : 2, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2, - "mapInputBytes" : 131804880 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.192", "cluster1559\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1594\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1579\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872387, - "finishTime" : 1240336904515, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000057", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1574\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1574\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876540, - "finishTime" : 1240336915909, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000028_0", - "hdfsBytesRead" : 63494931, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2479449, - "mapInputRecords" : 3946537, - "mapOutputBytes" : 4204287, - "mapOutputRecords" : 65600, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65600, - "mapInputBytes" : 246575765 - }, { - "location" : null, - "hostName" : "cluster50205\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915586, - "finishTime" : 1240336949032, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000028_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.0", "cluster1561\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50206\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50217\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872388, - "finishTime" : 1240336920508, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000028", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.0", "cluster1302\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.0/cluster1302\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875716, - "finishTime" : 1240336915224, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000014_0", - "hdfsBytesRead" : 65542507, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2519207, - "mapInputRecords" : 4123667, - "mapOutputBytes" : 4223635, - "mapOutputRecords" : 65390, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65390, - "mapInputBytes" : 256189404 - }, { - "location" : null, - "hostName" : "cluster50154\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915022, - "finishTime" : 1240336954536, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000014_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1299\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.0", "cluster1289\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50142\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872388, - "finishTime" : 1240336920547, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000014", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.128", "cluster1377\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.128/cluster1377\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336870160, - "finishTime" : 1240336910333, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000018_0", - "hdfsBytesRead" : 65179399, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2488791, - "mapInputRecords" : 4131630, - "mapOutputBytes" : 4155682, - "mapOutputRecords" : 64683, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 64683, - "mapInputBytes" : 255327265 - }, { - "location" : null, - "hostName" : "cluster1364\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336911917, - "finishTime" : 1240336945214, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000018_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1383\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.128", "cluster1395\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50441\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872389, - "finishTime" : 1240336920490, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000018", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.64", "cluster1357\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.64/cluster1357\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336866494, - "finishTime" : 1240336887708, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000064_0", - "hdfsBytesRead" : 29993789, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2363, - "mapInputRecords" : 2168531, - "mapOutputBytes" : 454, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 128381652 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1321\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50475\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50473\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872389, - "finishTime" : 1240336904527, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000064", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.128", "cluster1368\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.128/cluster1368\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871607, - "finishTime" : 1240336898707, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000063_0", - "hdfsBytesRead" : 29998454, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2352, - "mapInputRecords" : 2182549, - "mapOutputBytes" : 428, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 128497476 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1371\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.64", "cluster1822\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.64", "cluster1832\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872390, - "finishTime" : 1240336904536, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000063", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.192", "cluster1251\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.192/cluster1251\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872977, - "finishTime" : 1240336910764, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000023_0", - "hdfsBytesRead" : 64457911, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2454975, - "mapInputRecords" : 4073107, - "mapOutputBytes" : 4111625, - "mapOutputRecords" : 63981, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63981, - "mapInputBytes" : 252476783 - }, { - "location" : null, - "hostName" : "cluster1255\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916099, - "finishTime" : 1240336969463, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000023_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1246\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1274\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50423\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872390, - "finishTime" : 1240336920495, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000023", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.0", "cluster1281\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.0/cluster1281\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873567, - "finishTime" : 1240336910205, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000074_0", - "hdfsBytesRead" : 29798610, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 3066, - "mapInputRecords" : 2137289, - "mapOutputBytes" : 1604, - "mapOutputRecords" : 38, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 38, - "mapInputBytes" : 126797079 - }, { - "location" : null, - "hostName" : "cluster1830\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336920116, - "finishTime" : 1240336954303, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000074_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1316\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50493\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50488\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872391, - "finishTime" : 1240336920505, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000074", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1420\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1420\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336870902, - "finishTime" : 1240336911334, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000001_0", - "hdfsBytesRead" : 69761283, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2709488, - "mapInputRecords" : 4346205, - "mapOutputBytes" : 4543664, - "mapOutputRecords" : 70626, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 70626, - "mapInputBytes" : 270462297 - }, { - "location" : null, - "hostName" : "cluster1850\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336917236, - "finishTime" : 1240336950666, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000001_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.192", "cluster1405\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.192", "cluster1427\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50365\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872391, - "finishTime" : 1240336920486, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1540\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1540\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871750, - "finishTime" : 1240336899790, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000061_0", - "hdfsBytesRead" : 30111997, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2348, - "mapInputRecords" : 2160034, - "mapOutputBytes" : 354, - "mapOutputRecords" : 6, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 6, - "mapInputBytes" : 128130156 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1138\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1137\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1520\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872392, - "finishTime" : 1240336904500, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000061", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.128", "cluster1493\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.128/cluster1493\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874481, - "finishTime" : 1240336923543, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000049_0", - "hdfsBytesRead" : 30486394, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2076, - "mapInputRecords" : 2239224, - "mapOutputBytes" : 102, - "mapOutputRecords" : 2, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2, - "mapInputBytes" : 131723681 - }, { - "location" : null, - "hostName" : "cluster1519\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915105, - "finishTime" : 1240336939262, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000049_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1518\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50501\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50484\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872393, - "finishTime" : 1240336936512, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000049", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1843\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1843\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873996, - "finishTime" : 1240336930684, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000029_0", - "hdfsBytesRead" : 63373946, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2388744, - "mapInputRecords" : 4033224, - "mapOutputBytes" : 3991838, - "mapOutputRecords" : 62073, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62073, - "mapInputBytes" : 248134254 - }, { - "location" : null, - "hostName" : "cluster1872\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916333, - "finishTime" : 1240336965543, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000029_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1035\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1018\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1853\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872393, - "finishTime" : 1240336936568, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000029", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1555\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1555\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875187, - "finishTime" : 1240336898191, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000075_0", - "hdfsBytesRead" : 29746839, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2502, - "mapInputRecords" : 2130992, - "mapOutputBytes" : 642, - "mapOutputRecords" : 13, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 13, - "mapInputBytes" : 126084263 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.192", "cluster1557\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50214\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50210\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872394, - "finishTime" : 1240336904493, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000075", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.128", "cluster1228\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.128/cluster1228\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873902, - "finishTime" : 1240336914414, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000085_0", - "hdfsBytesRead" : 29405954, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2264, - "mapInputRecords" : 2125410, - "mapOutputBytes" : 307, - "mapOutputRecords" : 6, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 6, - "mapInputBytes" : 125216183 - }, { - "location" : null, - "hostName" : "cluster1239\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914422, - "finishTime" : 1240336947813, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000085_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1212\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.128", "cluster1205\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1743\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872394, - "finishTime" : 1240336920525, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000085", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.64", "cluster1180\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.64/cluster1180\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873955, - "finishTime" : 1240336917137, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000009_0", - "hdfsBytesRead" : 66872279, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2517067, - "mapInputRecords" : 4251406, - "mapOutputBytes" : 4217383, - "mapOutputRecords" : 65906, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65906, - "mapInputBytes" : 261611387 - }, { - "location" : null, - "hostName" : "cluster50545\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916823, - "finishTime" : 1240336950112, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000009_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.64", "cluster1180\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.64", "cluster1193\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50281\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872395, - "finishTime" : 1240336920563, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000009", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.128", "cluster1690\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.128/cluster1690\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874755, - "finishTime" : 1240336907982, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000078_0", - "hdfsBytesRead" : 29710276, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2244, - "mapInputRecords" : 2103351, - "mapOutputBytes" : 429, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 125317251 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.128", "cluster1703\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1694\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1768\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872395, - "finishTime" : 1240336920560, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000078", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1639\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1639\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879086, - "finishTime" : 1240336891448, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000091_0", - "hdfsBytesRead" : 8632081, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 61030, - "mapInputRecords" : 415389, - "mapOutputBytes" : 125127, - "mapOutputRecords" : 2025, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2025, - "mapInputBytes" : 30313509 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.64", "cluster1635\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50317\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50281\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872396, - "finishTime" : 1240336888496, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000091", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.192", "cluster1250\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.192/cluster1250\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874153, - "finishTime" : 1240336895738, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000083_0", - "hdfsBytesRead" : 29568513, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2278, - "mapInputRecords" : 2122899, - "mapOutputBytes" : 302, - "mapOutputRecords" : 5, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5, - "mapInputBytes" : 125860315 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1250\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1931\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1957\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872396, - "finishTime" : 1240336904518, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000083", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.64", "cluster1727\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.64/cluster1727\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879765, - "finishTime" : 1240336932976, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000043_0", - "hdfsBytesRead" : 32717815, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 34330747, - "mapInputRecords" : 1341156, - "mapOutputBytes" : 71461506, - "mapOutputRecords" : 1155088, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1155088, - "mapInputBytes" : 99883113 - }, { - "location" : null, - "hostName" : "cluster50135\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915113, - "finishTime" : 1240336964459, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000043_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.64", "cluster1752\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1725\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50156\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872397, - "finishTime" : 1240336936550, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000043", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.64", "cluster1356\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.64/cluster1356\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872994, - "finishTime" : 1240336894558, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000069_0", - "hdfsBytesRead" : 29896312, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2490, - "mapInputRecords" : 2161066, - "mapOutputBytes" : 626, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 127337881 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.64", "cluster1358\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.64", "cluster1346\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.128", "cluster1518\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872397, - "finishTime" : 1240336904506, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000069", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1418\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1418\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872526, - "finishTime" : 1240336920970, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000042_0", - "hdfsBytesRead" : 32920579, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 34480517, - "mapInputRecords" : 1354003, - "mapOutputBytes" : 72052350, - "mapOutputRecords" : 1167455, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1167455, - "mapInputBytes" : 100609874 - }, { - "location" : null, - "hostName" : "cluster50192\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914080, - "finishTime" : 1240336963396, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000042_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.192", "cluster1400\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50195\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50179\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872397, - "finishTime" : 1240336936525, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000042", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1807\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1807\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874347, - "finishTime" : 1240336897429, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000052_0", - "hdfsBytesRead" : 30352683, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2466, - "mapInputRecords" : 2223842, - "mapOutputBytes" : 728, - "mapOutputRecords" : 12, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 12, - "mapInputBytes" : 130818373 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.64", "cluster1832\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50226\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50205\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872398, - "finishTime" : 1240336904498, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000052", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.192", "cluster1249\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.192/cluster1249\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873038, - "finishTime" : 1240336937714, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000031_0", - "hdfsBytesRead" : 63310753, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2411979, - "mapInputRecords" : 3945460, - "mapOutputBytes" : 4026410, - "mapOutputRecords" : 62373, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62373, - "mapInputBytes" : 245235770 - }, { - "location" : null, - "hostName" : "cluster50242\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916045, - "finishTime" : 1240336953642, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000031_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1248\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50274\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50260\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872398, - "finishTime" : 1240336952565, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000031", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1561\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1561\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876300, - "finishTime" : 1240336927625, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000032_0", - "hdfsBytesRead" : 63245290, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2440937, - "mapInputRecords" : 3984125, - "mapOutputBytes" : 4070942, - "mapOutputRecords" : 62997, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62997, - "mapInputBytes" : 247088467 - }, { - "location" : null, - "hostName" : "cluster1585\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915947, - "finishTime" : 1240336965244, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000032_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.0", "cluster1585\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3061\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3079\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872399, - "finishTime" : 1240336936618, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000032", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.128", "cluster1395\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.128/cluster1395\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872064, - "finishTime" : 1240336894757, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000077_0", - "hdfsBytesRead" : 29714767, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2099, - "mapInputRecords" : 2121327, - "mapOutputBytes" : 152, - "mapOutputRecords" : 3, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 3, - "mapInputBytes" : 126146844 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1379\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.128", "cluster1373\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3085\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872400, - "finishTime" : 1240336904494, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000077", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.128", "cluster1877\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.128/cluster1877\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872985, - "finishTime" : 1240336894181, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000086_0", - "hdfsBytesRead" : 29331367, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2457, - "mapInputRecords" : 2098982, - "mapOutputBytes" : 513, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 124536660 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.128", "cluster1859\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1871\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50072\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872400, - "finishTime" : 1240336904496, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000086", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1437\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1437\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873019, - "finishTime" : 1240336921372, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000080_0", - "hdfsBytesRead" : 29702894, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2351, - "mapInputRecords" : 2127301, - "mapOutputBytes" : 379, - "mapOutputRecords" : 5, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 5, - "mapInputBytes" : 126316979 - }, { - "location" : null, - "hostName" : "cluster50007\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336924581, - "finishTime" : 1240336946151, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000080_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.192", "cluster1533\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1529\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1036\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872401, - "finishTime" : 1240336936524, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000080", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.128", "cluster1711\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.128/cluster1711\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876373, - "finishTime" : 1240336889533, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000099_0", - "hdfsBytesRead" : 8195626, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 53268, - "mapInputRecords" : 364433, - "mapOutputBytes" : 92731, - "mapOutputRecords" : 1473, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1473, - "mapInputBytes" : 26915776 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.128", "cluster1708\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1714\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.0", "cluster50085\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872401, - "finishTime" : 1240336888509, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000099", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1775\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1775\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874334, - "finishTime" : 1240336895935, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000065_0", - "hdfsBytesRead" : 29991531, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2256, - "mapInputRecords" : 2146107, - "mapOutputBytes" : 291, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 127398601 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.64", "cluster1175\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1762\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1786\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872405, - "finishTime" : 1240336904543, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000065", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.64", "cluster1072\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.64/cluster1072\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876623, - "finishTime" : 1240336920733, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000016_0", - "hdfsBytesRead" : 65311537, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2517895, - "mapInputRecords" : 4137686, - "mapOutputBytes" : 4245719, - "mapOutputRecords" : 65718, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65718, - "mapInputBytes" : 257371314 - }, { - "location" : null, - "hostName" : "cluster1524\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336913870, - "finishTime" : 1240336947267, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000016_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1053\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.64", "cluster1055\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50459\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872405, - "finishTime" : 1240336920528, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000016", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1806\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1806\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875605, - "finishTime" : 1240336887030, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000100_0", - "hdfsBytesRead" : 8167681, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54317, - "mapInputRecords" : 363347, - "mapOutputBytes" : 93315, - "mapOutputRecords" : 1533, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1533, - "mapInputBytes" : 26787874 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1669\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1643\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.64", "cluster1816\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872406, - "finishTime" : 1240336888526, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000100", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.0", "cluster1920\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.0/cluster1920\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875470, - "finishTime" : 1240336912179, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000040_0", - "hdfsBytesRead" : 60683975, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2370660, - "mapInputRecords" : 3782452, - "mapOutputBytes" : 3971981, - "mapOutputRecords" : 61525, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 61525, - "mapInputBytes" : 235446580 - }, { - "location" : null, - "hostName" : "cluster1953\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916862, - "finishTime" : 1240336950159, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000040_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.64", "cluster1167\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1924\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1925\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872407, - "finishTime" : 1240336920542, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000040", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1448\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1448\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336872975, - "finishTime" : 1240336883733, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000097_0", - "hdfsBytesRead" : 8245699, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57116, - "mapInputRecords" : 361754, - "mapOutputBytes" : 109299, - "mapOutputRecords" : 1733, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1733, - "mapInputBytes" : 27052310 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1515\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1449\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1446\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872407, - "finishTime" : 1240336888495, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000097", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.64", "cluster1610\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.64/cluster1610\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874243, - "finishTime" : 1240336886322, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000114_0", - "hdfsBytesRead" : 7815881, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 56450, - "mapInputRecords" : 338147, - "mapOutputBytes" : 101843, - "mapOutputRecords" : 1713, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1713, - "mapInputBytes" : 25352645 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.64", "cluster1629\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.64", "cluster1615\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1989\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872408, - "finishTime" : 1240336888543, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000114", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.64", "cluster3068\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.64/cluster3068\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871157, - "finishTime" : 1240336903920, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000030_0", - "hdfsBytesRead" : 63316983, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2499856, - "mapInputRecords" : 3998244, - "mapOutputBytes" : 4194242, - "mapOutputRecords" : 65558, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65558, - "mapInputBytes" : 247082392 - }, { - "location" : null, - "hostName" : "cluster50114\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915039, - "finishTime" : 1240336948328, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000030_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.0", "cluster50101\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3054\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3070\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872408, - "finishTime" : 1240336920494, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000030", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.64", "cluster3073\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.64/cluster3073\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873209, - "finishTime" : 1240336892920, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000048_0", - "hdfsBytesRead" : 30548276, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2470, - "mapInputRecords" : 2272297, - "mapOutputBytes" : 588, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 133071879 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.192", "cluster50232\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3077\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3055\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872409, - "finishTime" : 1240336904497, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000048", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1788\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1788\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336887602, - "finishTime" : 1240336909157, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000072_0", - "hdfsBytesRead" : 29828627, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2687, - "mapInputRecords" : 2132740, - "mapOutputBytes" : 935, - "mapOutputRecords" : 16, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 16, - "mapInputBytes" : 126719922 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1650\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1666\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1760\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872409, - "finishTime" : 1240336904505, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000072", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1538\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1538\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874182, - "finishTime" : 1240336886198, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000105_0", - "hdfsBytesRead" : 7999674, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 56463, - "mapInputRecords" : 351714, - "mapOutputBytes" : 102216, - "mapOutputRecords" : 1667, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1667, - "mapInputBytes" : 26137712 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.192", "cluster1550\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1557\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50490\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872410, - "finishTime" : 1240336888521, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000105", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.128", "cluster1095\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.128/cluster1095\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873957, - "finishTime" : 1240336885026, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000095_0", - "hdfsBytesRead" : 8324595, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 60582, - "mapInputRecords" : 370323, - "mapOutputBytes" : 112110, - "mapOutputRecords" : 1745, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1745, - "mapInputBytes" : 27631638 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1097\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50058\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50047\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872410, - "finishTime" : 1240336888498, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000095", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3085\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3085\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336883981, - "finishTime" : 1240336903185, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000058_0", - "hdfsBytesRead" : 30204544, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2544, - "mapInputRecords" : 2174809, - "mapOutputBytes" : 619, - "mapOutputRecords" : 12, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 12, - "mapInputBytes" : 128829948 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1029\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3111\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3085\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872411, - "finishTime" : 1240336904451, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000058", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.64", "cluster1736\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.64/cluster1736\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874390, - "finishTime" : 1240336885111, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000129_0", - "hdfsBytesRead" : 7596302, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 56324, - "mapInputRecords" : 329237, - "mapOutputBytes" : 101487, - "mapOutputRecords" : 1619, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1619, - "mapInputBytes" : 24704627 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.0", "cluster1796\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1736\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1757\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872411, - "finishTime" : 1240336888504, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000129", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.64", "cluster3075\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.64/cluster3075\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871519, - "finishTime" : 1240336890113, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000084_0", - "hdfsBytesRead" : 29527352, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2526, - "mapInputRecords" : 2107299, - "mapOutputBytes" : 542, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 125356348 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.134\\.192", "cluster50375\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50382\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3076\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872412, - "finishTime" : 1240336904452, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000084", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.117\\.192", "cluster1276\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.117\\.192/cluster1276\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873160, - "finishTime" : 1240336910195, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000034_0", - "hdfsBytesRead" : 62826152, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2442116, - "mapInputRecords" : 3912487, - "mapOutputBytes" : 4109087, - "mapOutputRecords" : 63793, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63793, - "mapInputBytes" : 243815199 - }, { - "location" : null, - "hostName" : "cluster50061\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336948611, - "finishTime" : 1240336986156, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000034_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1276\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1243\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50046\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872412, - "finishTime" : 1240336920545, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000034", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.64", "cluster1722\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.64/cluster1722\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336886217, - "finishTime" : 1240336899148, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000092_0", - "hdfsBytesRead" : 8571804, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 65830, - "mapInputRecords" : 382509, - "mapOutputBytes" : 153711, - "mapOutputRecords" : 2418, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2418, - "mapInputBytes" : 28713382 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1221\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1731\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.64", "cluster1732\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872413, - "finishTime" : 1240336888549, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000092", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3103\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3103\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871792, - "finishTime" : 1240336904149, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000039_0", - "hdfsBytesRead" : 61493951, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2383004, - "mapInputRecords" : 3826526, - "mapOutputBytes" : 4014747, - "mapOutputRecords" : 62222, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62222, - "mapInputBytes" : 238695445 - }, { - "location" : null, - "hostName" : "cluster50119\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914788, - "finishTime" : 1240336948132, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000039_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.134\\.128", "cluster50348\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.128", "cluster50352\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3094\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872415, - "finishTime" : 1240336920500, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000039", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3099\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3099\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879295, - "finishTime" : 1240336889446, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000094_0", - "hdfsBytesRead" : 8363130, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 61542, - "mapInputRecords" : 362353, - "mapOutputBytes" : 116919, - "mapOutputRecords" : 1812, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1812, - "mapInputBytes" : 27305556 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.0", "cluster1316\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.0", "cluster1283\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.128", "cluster3097\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872416, - "finishTime" : 1240336888457, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000094", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.128", "cluster50185\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.128/cluster50185\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874482, - "finishTime" : 1240336901837, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000112_0", - "hdfsBytesRead" : 7897873, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 59138, - "mapInputRecords" : 348461, - "mapOutputBytes" : 109578, - "mapOutputRecords" : 1815, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1815, - "mapInputBytes" : 25836442 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.128", "cluster50174\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50185\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50120\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872418, - "finishTime" : 1240336904531, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000112", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50294\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50294\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873068, - "finishTime" : 1240336912576, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000020_0", - "hdfsBytesRead" : 64761324, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2512516, - "mapInputRecords" : 4036072, - "mapOutputBytes" : 4220545, - "mapOutputRecords" : 65595, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 65595, - "mapInputBytes" : 251142910 - }, { - "location" : null, - "hostName" : "cluster50045\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915002, - "finishTime" : 1240336950197, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000020_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.132\\.192", "cluster50055\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50316\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.64", "cluster50315\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872419, - "finishTime" : 1240336920577, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000020", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1447\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1447\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336871738, - "finishTime" : 1240336885628, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000120_0", - "hdfsBytesRead" : 7756197, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57141, - "mapInputRecords" : 332797, - "mapOutputBytes" : 105980, - "mapOutputRecords" : 1680, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1680, - "mapInputBytes" : 25063052 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1491\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.128", "cluster1480\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1444\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872419, - "finishTime" : 1240336888501, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000120", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50300\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50300\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873077, - "finishTime" : 1240336890562, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000117_0", - "hdfsBytesRead" : 7779599, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54489, - "mapInputRecords" : 342102, - "mapOutputBytes" : 99769, - "mapOutputRecords" : 1593, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1593, - "mapInputBytes" : 25758088 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1259\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.117\\.192", "cluster1256\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1525\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872420, - "finishTime" : 1240336904516, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000117", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.64", "cluster1969\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.64/cluster1969\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874912, - "finishTime" : 1240336912529, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000027_0", - "hdfsBytesRead" : 63744100, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2482609, - "mapInputRecords" : 3916909, - "mapOutputBytes" : 4146823, - "mapOutputRecords" : 64050, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 64050, - "mapInputBytes" : 245872068 - }, { - "location" : null, - "hostName" : "cluster1990\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336919940, - "finishTime" : 1240336953269, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000027_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1154\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1999\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1967\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872420, - "finishTime" : 1240336920525, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000027", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.128", "cluster1498\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.128/cluster1498\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876312, - "finishTime" : 1240336913198, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000056_0", - "hdfsBytesRead" : 30250367, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2417, - "mapInputRecords" : 2228688, - "mapOutputBytes" : 448, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 130399482 - }, { - "location" : null, - "hostName" : "cluster50040\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915085, - "finishTime" : 1240336948419, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000056_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1517\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50053\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50057\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872421, - "finishTime" : 1240336921375, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000056", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.128", "cluster3108\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.128/cluster3108\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336744382, - "finishTime" : 1240336754327, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000118_0", - "hdfsBytesRead" : 7761361, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54570, - "mapInputRecords" : 333787, - "mapOutputBytes" : 97693, - "mapOutputRecords" : 1550, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1550, - "mapInputBytes" : 25134207 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.192", "cluster1550\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.192", "cluster1533\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1928\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872421, - "finishTime" : 1240336888458, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000118", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50296\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50296\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873049, - "finishTime" : 1240336907869, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000053_0", - "hdfsBytesRead" : 30302479, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2191, - "mapInputRecords" : 2169494, - "mapOutputBytes" : 201, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 129763826 - }, { - "location" : null, - "hostName" : "cluster1937\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915752, - "finishTime" : 1240336948962, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000053_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1075\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.64", "cluster1042\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1938\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872422, - "finishTime" : 1240336920575, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000053", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1768\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1768\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336886240, - "finishTime" : 1240336908239, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000082_0", - "hdfsBytesRead" : 29653562, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2207, - "mapInputRecords" : 2109744, - "mapOutputBytes" : 228, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 125417788 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.0", "cluster1772\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.128", "cluster50025\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.128", "cluster50023\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872423, - "finishTime" : 1240336904530, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000082", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.0", "cluster50108\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.0/cluster50108\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336915063, - "finishTime" : 1240336930514, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000110_1", - "hdfsBytesRead" : 7921493, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54235, - "mapInputRecords" : 349247, - "mapOutputBytes" : 110600, - "mapOutputRecords" : 1804, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1804, - "mapInputBytes" : 26033361 - }, { - "location" : null, - "hostName" : "cluster50097\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336873099, - "finishTime" : 1240336927622, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000110_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1652\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.0", "cluster50108\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.0", "cluster50113\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872423, - "finishTime" : 1240336930526, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000110", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.192", "cluster50382\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.192/cluster50382\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873058, - "finishTime" : 1240336913430, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000005_0", - "hdfsBytesRead" : 68085382, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2636901, - "mapInputRecords" : 4335874, - "mapOutputBytes" : 4439823, - "mapOutputRecords" : 69636, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 69636, - "mapInputBytes" : 266679989 - }, { - "location" : null, - "hostName" : "cluster50186\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915168, - "finishTime" : 1240336948458, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000005_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.128", "cluster50166\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50372\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50377\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872424, - "finishTime" : 1240336920570, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000005", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.192", "cluster50377\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.192/cluster50377\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873119, - "finishTime" : 1240336889361, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000089_0", - "hdfsBytesRead" : 8799897, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 55966, - "mapInputRecords" : 414955, - "mapOutputBytes" : 103189, - "mapOutputRecords" : 1701, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1701, - "mapInputBytes" : 29805669 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.192", "cluster50232\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50365\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.192", "cluster50364\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872425, - "finishTime" : 1240336904514, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000089", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.128", "cluster50348\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.128/cluster50348\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873048, - "finishTime" : 1240336886665, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000107_0", - "hdfsBytesRead" : 7976981, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54786, - "mapInputRecords" : 382253, - "mapOutputBytes" : 99015, - "mapOutputRecords" : 1613, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1613, - "mapInputBytes" : 27687269 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.132\\.192", "cluster50043\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.128", "cluster50326\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.128", "cluster50325\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872425, - "finishTime" : 1240336888490, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000107", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.128", "cluster50477\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.128/cluster50477\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336877770, - "finishTime" : 1240336917699, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000004_0", - "hdfsBytesRead" : 68248954, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2553348, - "mapInputRecords" : 4274813, - "mapOutputBytes" : 4305896, - "mapOutputRecords" : 66582, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 66582, - "mapInputBytes" : 265135117 - }, { - "location" : null, - "hostName" : "cluster1164\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336916017, - "finishTime" : 1240336949262, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000004_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.128", "cluster50447\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50464\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50491\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872426, - "finishTime" : 1240336920537, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000004", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.64", "cluster1078\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.64/cluster1078\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873042, - "finishTime" : 1240336905504, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000073_0", - "hdfsBytesRead" : 29810727, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2406, - "mapInputRecords" : 2120499, - "mapOutputBytes" : 442, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 126971404 - }, { - "location" : null, - "hostName" : "cluster1301\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915497, - "finishTime" : 1240336948840, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000073_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1061\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.64", "cluster1040\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50245\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872427, - "finishTime" : 1240336920518, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000073", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.0", "cluster50520\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.0/cluster50520\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874158, - "finishTime" : 1240336896172, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000054_0", - "hdfsBytesRead" : 30291933, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2467, - "mapInputRecords" : 2197164, - "mapOutputBytes" : 537, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 130029656 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.0", "cluster50530\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.0", "cluster50526\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1970\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872427, - "finishTime" : 1240336904495, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000054", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50319\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50319\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873105, - "finishTime" : 1240336899922, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000071_0", - "hdfsBytesRead" : 29867527, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2529, - "mapInputRecords" : 2168718, - "mapOutputBytes" : 639, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 128006719 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1032\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1026\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1936\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872428, - "finishTime" : 1240336904508, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000071", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.0", "cluster1583\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.0/cluster1583\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336878767, - "finishTime" : 1240336889771, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000116_0", - "hdfsBytesRead" : 7788625, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 55129, - "mapInputRecords" : 334841, - "mapOutputBytes" : 105424, - "mapOutputRecords" : 1675, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1675, - "mapInputBytes" : 25214728 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.0", "cluster1567\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.0", "cluster1594\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1915\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872429, - "finishTime" : 1240336888563, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000116", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.192", "cluster1883\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.192/cluster1883\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875263, - "finishTime" : 1240336916121, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000035_0", - "hdfsBytesRead" : 62277748, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2394254, - "mapInputRecords" : 3851918, - "mapOutputBytes" : 4034362, - "mapOutputRecords" : 62292, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 62292, - "mapInputBytes" : 241639956 - }, { - "location" : null, - "hostName" : "cluster1891\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914750, - "finishTime" : 1240336948061, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000035_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1017\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1019\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1900\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872430, - "finishTime" : 1240336920630, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000035", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50297\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50297\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873087, - "finishTime" : 1240336887207, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000103_0", - "hdfsBytesRead" : 8054880, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57203, - "mapInputRecords" : 352581, - "mapOutputBytes" : 105232, - "mapOutputRecords" : 1677, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1677, - "mapInputBytes" : 26276597 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.192", "cluster50490\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1959\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1933\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872430, - "finishTime" : 1240336888512, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000103", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.192", "cluster50203\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.192/cluster50203\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873054, - "finishTime" : 1240336940061, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000036_0", - "hdfsBytesRead" : 62256077, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2425449, - "mapInputRecords" : 3896871, - "mapOutputBytes" : 4079715, - "mapOutputRecords" : 63812, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 63812, - "mapInputBytes" : 242202529 - }, { - "location" : null, - "hostName" : "cluster50170\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915065, - "finishTime" : 1240336955862, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000036_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.133\\.128", "cluster50170\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50208\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50210\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872431, - "finishTime" : 1240336952548, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000036", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50291\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50291\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873125, - "finishTime" : 1240336888240, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000125_0", - "hdfsBytesRead" : 7644012, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 56569, - "mapInputRecords" : 330028, - "mapOutputBytes" : 107007, - "mapOutputRecords" : 1684, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1684, - "mapInputBytes" : 24814596 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1144\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1141\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.0", "cluster1953\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872431, - "finishTime" : 1240336888528, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000125", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.0", "cluster1936\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.0/cluster1936\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873757, - "finishTime" : 1240336886908, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000101_0", - "hdfsBytesRead" : 8100684, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 64050, - "mapInputRecords" : 353384, - "mapOutputBytes" : 148877, - "mapOutputRecords" : 2652, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 2652, - "mapInputBytes" : 26767647 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.128", "cluster1503\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1961\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1963\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872433, - "finishTime" : 1240336888492, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000101", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.63\\.0", "cluster1778\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.63\\.0/cluster1778\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873042, - "finishTime" : 1240336884233, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000119_0", - "hdfsBytesRead" : 7760781, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54166, - "mapInputRecords" : 342134, - "mapOutputBytes" : 100849, - "mapOutputRecords" : 1633, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1633, - "mapInputBytes" : 25522379 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1107\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1798\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1789\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872435, - "finishTime" : 1240336888500, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000119", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.128", "cluster50342\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.128/cluster50342\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873123, - "finishTime" : 1240336886459, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000130_0", - "hdfsBytesRead" : 7567997, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 58226, - "mapInputRecords" : 330387, - "mapOutputBytes" : 106187, - "mapOutputRecords" : 1681, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1681, - "mapInputBytes" : 24831142 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.192", "cluster1272\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1683\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.128", "cluster1707\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872440, - "finishTime" : 1240336888519, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000130", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.64", "cluster1813\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.64/cluster1813\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336875527, - "finishTime" : 1240336911619, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000050_0", - "hdfsBytesRead" : 30483937, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2986, - "mapInputRecords" : 2181690, - "mapOutputBytes" : 2645, - "mapOutputRecords" : 59, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 59, - "mapInputBytes" : 129508563 - }, { - "location" : null, - "hostName" : "cluster50099\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915651, - "finishTime" : 1240336949203, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000050_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1151\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1679\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1678\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872447, - "finishTime" : 1240336920621, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000050", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.192", "cluster1906\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.192/cluster1906\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336877854, - "finishTime" : 1240336915858, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000090_0", - "hdfsBytesRead" : 8744991, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 59096, - "mapInputRecords" : 400419, - "mapOutputBytes" : 114548, - "mapOutputRecords" : 1879, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1879, - "mapInputBytes" : 28968801 - }, { - "location" : null, - "hostName" : "cluster50041\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915097, - "finishTime" : 1240336966677, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000090_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.192", "cluster1891\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1913\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50051\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872493, - "finishTime" : 1240336920834, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000090", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.64", "cluster50124\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.64/cluster50124\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336915274, - "finishTime" : 1240336938869, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000067_1", - "hdfsBytesRead" : 29917139, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2179, - "mapInputRecords" : 2151889, - "mapOutputBytes" : 189, - "mapOutputRecords" : 4, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 4, - "mapInputBytes" : 127323156 - }, { - "location" : null, - "hostName" : "cluster50475\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336877497, - "finishTime" : 1240336951548, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000067_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.128", "cluster50444\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50124\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50156\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872506, - "finishTime" : 1240336946437, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000067", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.129\\.192", "cluster1903\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.129\\.192/cluster1903\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336877965, - "finishTime" : 1240336889156, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000109_0", - "hdfsBytesRead" : 7954082, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 59357, - "mapInputRecords" : 344387, - "mapOutputBytes" : 107739, - "mapOutputRecords" : 1775, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1775, - "mapInputBytes" : 25743781 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.129\\.128", "cluster1841\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.128", "cluster1842\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.129\\.192", "cluster1907\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872544, - "finishTime" : 1240336888614, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000109", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.192", "cluster50231\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.192/cluster50231\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874559, - "finishTime" : 1240336902529, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000051_0", - "hdfsBytesRead" : 30393039, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2309, - "mapInputRecords" : 2170141, - "mapOutputBytes" : 418, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 128768431 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1130\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1158\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.192", "cluster50205\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872547, - "finishTime" : 1240336904655, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000051", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.64", "cluster50159\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.64/cluster50159\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873969, - "finishTime" : 1240336897513, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000070_0", - "hdfsBytesRead" : 29891233, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2372, - "mapInputRecords" : 2181020, - "mapOutputBytes" : 492, - "mapOutputRecords" : 9, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 9, - "mapInputBytes" : 128287582 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.192", "cluster50484\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50509\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50133\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872651, - "finishTime" : 1240336904748, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000070", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.128", "cluster1102\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.128/cluster1102\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336876290, - "finishTime" : 1240336900685, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000126_0", - "hdfsBytesRead" : 7643780, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 50624, - "mapInputRecords" : 332992, - "mapOutputBytes" : 89335, - "mapOutputRecords" : 1413, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1413, - "mapInputBytes" : 24892336 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.128", "cluster1087\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50420\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50429\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872665, - "finishTime" : 1240336905377, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000126", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.130\\.64", "cluster1976\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.130\\.64/cluster1976\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336916201, - "finishTime" : 1240336948646, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000055_1", - "hdfsBytesRead" : 30263792, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2383, - "mapInputRecords" : 2169198, - "mapOutputBytes" : 547, - "mapOutputRecords" : 7, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 7, - "mapInputBytes" : 128672563 - }, { - "location" : null, - "hostName" : "cluster1735\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336884134, - "finishTime" : 1240336962153, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000055_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1669\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50076\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50077\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872669, - "finishTime" : 1240336962561, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000055", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.0", "cluster50252\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.0/cluster50252\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873488, - "finishTime" : 1240336913959, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000108_0", - "hdfsBytesRead" : 7959315, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 55584, - "mapInputRecords" : 358754, - "mapOutputBytes" : 100438, - "mapOutputRecords" : 1658, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1658, - "mapInputBytes" : 26607263 - }, { - "location" : null, - "hostName" : "cluster1738\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336925060, - "finishTime" : 1240336958446, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000108_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.128", "cluster50460\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.128", "cluster50440\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50272\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872723, - "finishTime" : 1240336920897, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000108", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.132\\.128", "cluster50034\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.132\\.128/cluster50034\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336915103, - "finishTime" : 1240336927032, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000102_1", - "hdfsBytesRead" : 8093114, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57157, - "mapInputRecords" : 352241, - "mapOutputBytes" : 105550, - "mapOutputRecords" : 1675, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1675, - "mapInputBytes" : 26466064 - }, { - "location" : null, - "hostName" : "cluster50303\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336873423, - "finishTime" : 1240336926231, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000102_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1655\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1648\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50059\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872762, - "finishTime" : 1240336930552, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000102", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.192", "cluster50368\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.192/cluster50368\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873439, - "finishTime" : 1240336902317, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000106_0", - "hdfsBytesRead" : 7984742, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 61170, - "mapInputRecords" : 347188, - "mapOutputBytes" : 117181, - "mapOutputRecords" : 1839, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1839, - "mapInputBytes" : 26617003 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.64", "cluster1678\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1654\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.128\\.192", "cluster50485\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872831, - "finishTime" : 1240336904966, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000106", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.133\\.64", "cluster50145\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.133\\.64/cluster50145\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874931, - "finishTime" : 1240336890399, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000121_0", - "hdfsBytesRead" : 7753496, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54465, - "mapInputRecords" : 335167, - "mapOutputBytes" : 104220, - "mapOutputRecords" : 1650, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1650, - "mapInputBytes" : 25170333 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1388\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.128", "cluster1383\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.64", "cluster50133\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872869, - "finishTime" : 1240336904994, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000121", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.128\\.192", "cluster50513\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.128\\.192/cluster50513\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874944, - "finishTime" : 1240336885951, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000123_0", - "hdfsBytesRead" : 7711498, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57707, - "mapInputRecords" : 336739, - "mapOutputBytes" : 111288, - "mapOutputRecords" : 1762, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1762, - "mapInputBytes" : 25234362 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.127\\.64", "cluster1474\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1445\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.64", "cluster1654\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872873, - "finishTime" : 1240336888962, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000123", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "cluster50458\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336873610, - "finishTime" : 1240336892202, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000113_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : { - "layers" : [ "192\\.30\\.116\\.192", "cluster1150\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.192/cluster1150\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336905590, - "finishTime" : 1240336916127, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000113_1", - "hdfsBytesRead" : 7838525, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 54305, - "mapInputRecords" : 342277, - "mapOutputBytes" : 100228, - "mapOutputRecords" : 1564, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1564, - "mapInputBytes" : 25563399 - }, { - "location" : null, - "hostName" : "cluster3063\\.secondleveldomain\\.com", - "result" : "FAILED", - "startTime" : 1240336937806, - "finishTime" : 1240336971856, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000113_2", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - }, { - "location" : null, - "hostName" : "cluster3063\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336937806, - "finishTime" : 1240336971856, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000113_3", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.128\\.128", "cluster50468\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3058\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3063\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336872938, - "finishTime" : 1240336921233, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000113", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.135\\.64", "cluster3077\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.135\\.64/cluster3077\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336883475, - "finishTime" : 1240336893425, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000128_0", - "hdfsBytesRead" : 7614230, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57139, - "mapInputRecords" : 327184, - "mapOutputBytes" : 98814, - "mapOutputRecords" : 1560, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1560, - "mapInputBytes" : 24696916 - } ], - "preferredLocations" : [ { - "layers" : [ "194\\.6\\.134\\.0", "cluster50243\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.134\\.0", "cluster50279\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.135\\.64", "cluster3057\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873047, - "finishTime" : 1240336889130, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000128", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.192", "cluster1429\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.192/cluster1429\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336869816, - "finishTime" : 1240336880564, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000124_0", - "hdfsBytesRead" : 7699474, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57466, - "mapInputRecords" : 335634, - "mapOutputBytes" : 114714, - "mapOutputRecords" : 1809, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1809, - "mapInputBytes" : 25212522 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1139\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1467\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.127\\.64", "cluster1479\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873048, - "finishTime" : 1240336889146, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000124", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1466\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1466\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336869967, - "finishTime" : 1240336905157, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000093_0", - "hdfsBytesRead" : 8500796, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 61110, - "mapInputRecords" : 381639, - "mapOutputBytes" : 111387, - "mapOutputRecords" : 1871, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1871, - "mapInputBytes" : 28132206 - }, { - "location" : null, - "hostName" : "cluster50174\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336914921, - "finishTime" : 1240336926260, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000093_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1157\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50174\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50177\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873049, - "finishTime" : 1240336921197, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000093", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.116\\.192", "cluster1150\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.116\\.192/cluster1150\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336873948, - "finishTime" : 1240336901907, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000044_0", - "hdfsBytesRead" : 31080028, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2324, - "mapInputRecords" : 2317829, - "mapOutputBytes" : 406, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 136117012 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1140\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1124\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50058\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873050, - "finishTime" : 1240336905227, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000044", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.192", "cluster1531\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.192/cluster1531\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874990, - "finishTime" : 1240336911553, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000104_0", - "hdfsBytesRead" : 8040935, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 55143, - "mapInputRecords" : 350537, - "mapOutputBytes" : 100554, - "mapOutputRecords" : 1616, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1616, - "mapInputBytes" : 26367904 - }, { - "location" : null, - "hostName" : "cluster3092\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336961643, - "finishTime" : 1240336971382, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000104_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.117\\.128", "cluster1235\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50171\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.128", "cluster50161\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873051, - "finishTime" : 1240336921172, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000104", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.0", "cluster1305\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.0/cluster1305\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874698, - "finishTime" : 1240336924089, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000076_0", - "hdfsBytesRead" : 29718365, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 2434, - "mapInputRecords" : 2122176, - "mapOutputBytes" : 514, - "mapOutputRecords" : 8, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 8, - "mapInputBytes" : 125964532 - }, { - "location" : null, - "hostName" : "cluster1964\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915897, - "finishTime" : 1240336940807, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000076_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.64", "cluster1053\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.64", "cluster1056\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.130\\.64", "cluster1968\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873052, - "finishTime" : 1240336937226, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000076", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.127\\.64", "cluster1454\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.127\\.64/cluster1454\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874635, - "finishTime" : 1240336886965, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000096_0", - "hdfsBytesRead" : 8294959, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 57076, - "mapInputRecords" : 373384, - "mapOutputBytes" : 115997, - "mapOutputRecords" : 1854, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1854, - "mapInputBytes" : 27404774 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.63\\.192", "cluster1028\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.192", "cluster1030\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.132\\.192", "cluster50044\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873053, - "finishTime" : 1240336889184, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000096", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.0", "cluster50254\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.0/cluster50254\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874658, - "finishTime" : 1240336888484, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000127_0", - "hdfsBytesRead" : 7627825, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 60903, - "mapInputRecords" : 330989, - "mapOutputBytes" : 120185, - "mapOutputRecords" : 1985, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1985, - "mapInputBytes" : 24806195 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.116\\.192", "cluster1157\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.116\\.192", "cluster1136\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.62\\.192", "cluster50409\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873054, - "finishTime" : 1240336889174, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000127", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.126\\.64", "cluster1328\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.126\\.64/cluster1328\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336874488, - "finishTime" : 1240336913539, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000122_0", - "hdfsBytesRead" : 7746591, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 55261, - "mapInputRecords" : 337548, - "mapOutputBytes" : 110325, - "mapOutputRecords" : 1750, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1750, - "mapInputBytes" : 25159067 - }, { - "location" : null, - "hostName" : "cluster1060\\.secondleveldomain\\.com", - "result" : "KILLED", - "startTime" : 1240336915247, - "finishTime" : 1240336927337, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000122_1", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : -1, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.126\\.128", "cluster1370\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.126\\.128", "cluster1378\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "192\\.30\\.63\\.0", "cluster1793\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873055, - "finishTime" : 1240336921185, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000122", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - }, { - "attempts" : [ { - "location" : { - "layers" : [ "192\\.30\\.62\\.64", "cluster1641\\.secondleveldomain\\.com" ] - }, - "hostName" : "/192\\.30\\.62\\.64/cluster1641\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336879658, - "finishTime" : 1240336890529, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000111_0", - "hdfsBytesRead" : 7914429, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : 58343, - "mapInputRecords" : 345958, - "mapOutputBytes" : 107374, - "mapOutputRecords" : 1748, - "combineInputRecords" : 0, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 1748, - "mapInputBytes" : 25806139 - } ], - "preferredLocations" : [ { - "layers" : [ "192\\.30\\.62\\.192", "cluster50418\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.0", "cluster50090\\.secondleveldomain\\.com" ] - }, { - "layers" : [ "194\\.6\\.133\\.0", "cluster50117\\.secondleveldomain\\.com" ] - } ], - "startTime" : 1240336873056, - "finishTime" : 1240336889145, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000111", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "MAP" - } ], - "reduceTasks" : [ { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.134\\.64/cluster50303\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336888934, - "finishTime" : 1240336985264, - "shuffleFinished" : 1240336974089, - "sortFinished" : 1240336974613, - "attemptID" : "attempt_200904211745_0004_r_000000_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1514383, - "fileBytesRead" : 2632927, - "fileBytesWritten" : 2632927, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 135445, - "reduceInputRecords" : 139391, - "reduceShuffleBytes" : 4405338, - "reduceOutputRecords" : 139391, - "spilledRecords" : 139391, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336888851, - "finishTime" : 1240337000993, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000000", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.134\\.0/cluster50252\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336888952, - "finishTime" : 1240336985501, - "shuffleFinished" : 1240336971396, - "sortFinished" : 1240336971892, - "attemptID" : "attempt_200904211745_0004_r_000001_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1499786, - "fileBytesRead" : 2600635, - "fileBytesWritten" : 2600635, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 127806, - "reduceInputRecords" : 130935, - "reduceShuffleBytes" : 4415977, - "reduceOutputRecords" : 130935, - "spilledRecords" : 130935, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336888869, - "finishTime" : 1240337000960, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000001", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.134\\.192/cluster50368\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889019, - "finishTime" : 1240336985309, - "shuffleFinished" : 1240336975164, - "sortFinished" : 1240336975677, - "attemptID" : "attempt_200904211745_0004_r_000002_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1530537, - "fileBytesRead" : 2654430, - "fileBytesWritten" : 2654430, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 133326, - "reduceInputRecords" : 136708, - "reduceShuffleBytes" : 4499527, - "reduceOutputRecords" : 136708, - "spilledRecords" : 136708, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336888941, - "finishTime" : 1240337001041, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000002", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.128\\.192/cluster50513\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890410, - "finishTime" : 1240336987320, - "shuffleFinished" : 1240336974158, - "sortFinished" : 1240336974678, - "attemptID" : "attempt_200904211745_0004_r_000003_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1499750, - "fileBytesRead" : 2604086, - "fileBytesWritten" : 2604086, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 129054, - "reduceInputRecords" : 132376, - "reduceShuffleBytes" : 4422289, - "reduceOutputRecords" : 132376, - "spilledRecords" : 132376, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336888963, - "finishTime" : 1240337001095, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000003", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.64/cluster50145\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890439, - "finishTime" : 1240336986603, - "shuffleFinished" : 1240336973537, - "sortFinished" : 1240336974021, - "attemptID" : "attempt_200904211745_0004_r_000004_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1463089, - "fileBytesRead" : 2534927, - "fileBytesWritten" : 2534927, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 125284, - "reduceInputRecords" : 128270, - "reduceShuffleBytes" : 4292451, - "reduceOutputRecords" : 128270, - "spilledRecords" : 128270, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336888984, - "finishTime" : 1240337001109, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000004", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.63\\.64/cluster1735\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336898197, - "finishTime" : 1240336994614, - "shuffleFinished" : 1240336982632, - "sortFinished" : 1240336983604, - "attemptID" : "attempt_200904211745_0004_r_000005_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1524925, - "fileBytesRead" : 2651643, - "fileBytesWritten" : 2651643, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 133361, - "reduceInputRecords" : 136557, - "reduceShuffleBytes" : 4483041, - "reduceOutputRecords" : 136557, - "spilledRecords" : 136557, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889084, - "finishTime" : 1240337001321, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000005", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.64/cluster50132\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889699, - "finishTime" : 1240336985779, - "shuffleFinished" : 1240336974653, - "sortFinished" : 1240336975212, - "attemptID" : "attempt_200904211745_0004_r_000006_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1490233, - "fileBytesRead" : 2582328, - "fileBytesWritten" : 2582328, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 127906, - "reduceInputRecords" : 131571, - "reduceShuffleBytes" : 4383017, - "reduceOutputRecords" : 131571, - "spilledRecords" : 131571, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889089, - "finishTime" : 1240337001250, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000006", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.192/cluster50235\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889794, - "finishTime" : 1240336986114, - "shuffleFinished" : 1240336973868, - "sortFinished" : 1240336974400, - "attemptID" : "attempt_200904211745_0004_r_000007_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1505343, - "fileBytesRead" : 2610391, - "fileBytesWritten" : 2610391, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 129022, - "reduceInputRecords" : 132145, - "reduceShuffleBytes" : 4444172, - "reduceOutputRecords" : 132145, - "spilledRecords" : 132145, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889090, - "finishTime" : 1240337001307, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000007", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.128\\.192/cluster50491\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336892799, - "finishTime" : 1240336989074, - "shuffleFinished" : 1240336977913, - "sortFinished" : 1240336978491, - "attemptID" : "attempt_200904211745_0004_r_000008_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1504684, - "fileBytesRead" : 2608073, - "fileBytesWritten" : 2608073, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 127971, - "reduceInputRecords" : 131200, - "reduceShuffleBytes" : 4441998, - "reduceOutputRecords" : 131200, - "spilledRecords" : 131200, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889091, - "finishTime" : 1240337001271, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000008", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.62\\.64/cluster1679\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889759, - "finishTime" : 1240336986187, - "shuffleFinished" : 1240336974015, - "sortFinished" : 1240336974545, - "attemptID" : "attempt_200904211745_0004_r_000009_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1496159, - "fileBytesRead" : 2593399, - "fileBytesWritten" : 2593399, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 128913, - "reduceInputRecords" : 131980, - "reduceShuffleBytes" : 4397570, - "reduceOutputRecords" : 131980, - "spilledRecords" : 131980, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889094, - "finishTime" : 1240337001265, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000009", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.132\\.128/cluster50024\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889731, - "finishTime" : 1240336985667, - "shuffleFinished" : 1240336975405, - "sortFinished" : 1240336975928, - "attemptID" : "attempt_200904211745_0004_r_000010_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1516034, - "fileBytesRead" : 2633863, - "fileBytesWritten" : 2633863, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 130278, - "reduceInputRecords" : 133696, - "reduceShuffleBytes" : 4454003, - "reduceOutputRecords" : 133696, - "spilledRecords" : 133696, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889095, - "finishTime" : 1240337001270, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000010", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.192/cluster50223\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889705, - "finishTime" : 1240336985456, - "shuffleFinished" : 1240336972242, - "sortFinished" : 1240336972740, - "attemptID" : "attempt_200904211745_0004_r_000011_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1493749, - "fileBytesRead" : 2585694, - "fileBytesWritten" : 2585694, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 128462, - "reduceInputRecords" : 131413, - "reduceShuffleBytes" : 4380350, - "reduceOutputRecords" : 131413, - "spilledRecords" : 131413, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889096, - "finishTime" : 1240337001272, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000011", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.126\\.0/cluster1314\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890592, - "finishTime" : 1240336986175, - "shuffleFinished" : 1240336973921, - "sortFinished" : 1240336974467, - "attemptID" : "attempt_200904211745_0004_r_000012_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1542372, - "fileBytesRead" : 2681618, - "fileBytesWritten" : 2681618, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 136804, - "reduceInputRecords" : 139932, - "reduceShuffleBytes" : 4537451, - "reduceOutputRecords" : 139932, - "spilledRecords" : 139932, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889100, - "finishTime" : 1240337001225, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000012", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.127\\.64/cluster1440\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890397, - "finishTime" : 1240336986407, - "shuffleFinished" : 1240336974415, - "sortFinished" : 1240336974913, - "attemptID" : "attempt_200904211745_0004_r_000013_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1518799, - "fileBytesRead" : 2631693, - "fileBytesWritten" : 2631693, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 129644, - "reduceInputRecords" : 132582, - "reduceShuffleBytes" : 4464832, - "reduceOutputRecords" : 132582, - "spilledRecords" : 132582, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889101, - "finishTime" : 1240337001247, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000013", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.62\\.64/cluster1647\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889674, - "finishTime" : 1240336985701, - "shuffleFinished" : 1240336974501, - "sortFinished" : 1240336975018, - "attemptID" : "attempt_200904211745_0004_r_000014_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1529493, - "fileBytesRead" : 2660190, - "fileBytesWritten" : 2660190, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 130930, - "reduceInputRecords" : 134318, - "reduceShuffleBytes" : 4508491, - "reduceOutputRecords" : 134318, - "spilledRecords" : 134318, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889106, - "finishTime" : 1240337001250, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000014", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.126\\.0/cluster1295\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890630, - "finishTime" : 1240336986643, - "shuffleFinished" : 1240336974515, - "sortFinished" : 1240336975047, - "attemptID" : "attempt_200904211745_0004_r_000015_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1488785, - "fileBytesRead" : 2587120, - "fileBytesWritten" : 2587120, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 129374, - "reduceInputRecords" : 132757, - "reduceShuffleBytes" : 4397939, - "reduceOutputRecords" : 132757, - "spilledRecords" : 132757, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889107, - "finishTime" : 1240337001221, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000015", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.135\\.64/cluster3077\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336899098, - "finishTime" : 1240336995768, - "shuffleFinished" : 1240336982067, - "sortFinished" : 1240336982475, - "attemptID" : "attempt_200904211745_0004_r_000016_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1482497, - "fileBytesRead" : 2568924, - "fileBytesWritten" : 2568924, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 125907, - "reduceInputRecords" : 128797, - "reduceShuffleBytes" : 4350055, - "reduceOutputRecords" : 128797, - "spilledRecords" : 128797, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889131, - "finishTime" : 1240337001197, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000016", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.62\\.64/cluster1641\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336895225, - "finishTime" : 1240336991501, - "shuffleFinished" : 1240336978367, - "sortFinished" : 1240336978877, - "attemptID" : "attempt_200904211745_0004_r_000017_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1501337, - "fileBytesRead" : 2604597, - "fileBytesWritten" : 2604597, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 131288, - "reduceInputRecords" : 134365, - "reduceShuffleBytes" : 4400916, - "reduceOutputRecords" : 134365, - "spilledRecords" : 134365, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889145, - "finishTime" : 1240337001222, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000017", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.126\\.192/cluster1429\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336885349, - "finishTime" : 1240336981576, - "shuffleFinished" : 1240336968527, - "sortFinished" : 1240336969054, - "attemptID" : "attempt_200904211745_0004_r_000018_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1512739, - "fileBytesRead" : 2623583, - "fileBytesWritten" : 2623583, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 131750, - "reduceInputRecords" : 134780, - "reduceShuffleBytes" : 4448997, - "reduceOutputRecords" : 134780, - "spilledRecords" : 134780, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889147, - "finishTime" : 1240337001223, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000018", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.127\\.192/cluster1531\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890514, - "finishTime" : 1240336987216, - "shuffleFinished" : 1240336974189, - "sortFinished" : 1240336974682, - "attemptID" : "attempt_200904211745_0004_r_000019_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1497900, - "fileBytesRead" : 2595945, - "fileBytesWritten" : 2595945, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 127094, - "reduceInputRecords" : 130020, - "reduceShuffleBytes" : 4421936, - "reduceOutputRecords" : 130020, - "spilledRecords" : 130020, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889152, - "finishTime" : 1240337001238, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000019", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.126\\.64/cluster1328\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890066, - "finishTime" : 1240336986478, - "shuffleFinished" : 1240336976422, - "sortFinished" : 1240336976934, - "attemptID" : "attempt_200904211745_0004_r_000020_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1527353, - "fileBytesRead" : 2649256, - "fileBytesWritten" : 2649256, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 133593, - "reduceInputRecords" : 136882, - "reduceShuffleBytes" : 4474494, - "reduceOutputRecords" : 136882, - "spilledRecords" : 136882, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889157, - "finishTime" : 1240337001251, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000020", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.134\\.0/cluster50254\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890210, - "finishTime" : 1240336986556, - "shuffleFinished" : 1240336973579, - "sortFinished" : 1240336974104, - "attemptID" : "attempt_200904211745_0004_r_000021_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1513275, - "fileBytesRead" : 2630609, - "fileBytesWritten" : 2630609, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 131516, - "reduceInputRecords" : 135124, - "reduceShuffleBytes" : 4466234, - "reduceOutputRecords" : 135124, - "spilledRecords" : 135124, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889174, - "finishTime" : 1240337001247, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000021", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.127\\.64/cluster1466\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336885559, - "finishTime" : 1240336982206, - "shuffleFinished" : 1240336972182, - "sortFinished" : 1240336972682, - "attemptID" : "attempt_200904211745_0004_r_000022_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1502055, - "fileBytesRead" : 2608484, - "fileBytesWritten" : 2608484, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 129784, - "reduceInputRecords" : 132724, - "reduceShuffleBytes" : 4414804, - "reduceOutputRecords" : 132724, - "spilledRecords" : 132724, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889179, - "finishTime" : 1240337001263, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000022", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.127\\.64/cluster1454\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890222, - "finishTime" : 1240336986523, - "shuffleFinished" : 1240336973420, - "sortFinished" : 1240336973931, - "attemptID" : "attempt_200904211745_0004_r_000023_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1468760, - "fileBytesRead" : 2543252, - "fileBytesWritten" : 2543252, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 126859, - "reduceInputRecords" : 129871, - "reduceShuffleBytes" : 4296232, - "reduceOutputRecords" : 129871, - "spilledRecords" : 129871, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889185, - "finishTime" : 1240337001262, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000023", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.126\\.0/cluster1305\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890266, - "finishTime" : 1240336986394, - "shuffleFinished" : 1240336975908, - "sortFinished" : 1240336976433, - "attemptID" : "attempt_200904211745_0004_r_000024_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1527960, - "fileBytesRead" : 2649290, - "fileBytesWritten" : 2649290, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 133015, - "reduceInputRecords" : 136281, - "reduceShuffleBytes" : 4485423, - "reduceOutputRecords" : 136281, - "spilledRecords" : 136281, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889196, - "finishTime" : 1240337001291, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000024", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889205, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000025", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/192\\.30\\.116\\.192/cluster1150\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889529, - "finishTime" : 1240336986187, - "shuffleFinished" : 1240336973131, - "sortFinished" : 1240336973659, - "attemptID" : "attempt_200904211745_0004_r_000026_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1482330, - "fileBytesRead" : 2569201, - "fileBytesWritten" : 2569201, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 125873, - "reduceInputRecords" : 129080, - "reduceShuffleBytes" : 4363921, - "reduceOutputRecords" : 129080, - "spilledRecords" : 129080, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889208, - "finishTime" : 1240337001309, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000026", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.128\\.192/cluster50511\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336890681, - "finishTime" : 1240336986867, - "shuffleFinished" : 1240336975793, - "sortFinished" : 1240336976337, - "attemptID" : "attempt_200904211745_0004_r_000027_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1443497, - "fileBytesRead" : 2503238, - "fileBytesWritten" : 2503238, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 122459, - "reduceInputRecords" : 125626, - "reduceShuffleBytes" : 4238108, - "reduceOutputRecords" : 125626, - "spilledRecords" : 125626, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889209, - "finishTime" : 1240337001366, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000027", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "REDUCE" - }, { - "attempts" : [ { - "location" : null, - "hostName" : "/194\\.6\\.133\\.128/cluster50189\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336889839, - "finishTime" : 1240336986369, - "shuffleFinished" : 1240336976232, - "sortFinished" : 1240336976742, - "attemptID" : "attempt_200904211745_0004_r_000028_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : 1495322, - "fileBytesRead" : 2590588, - "fileBytesWritten" : 2590588, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : 0, - "reduceInputGroups" : 126998, - "reduceInputRecords" : 130037, - "reduceShuffleBytes" : 4405490, - "reduceOutputRecords" : 130037, - "spilledRecords" : 130037, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336889210, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000028", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889318, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000029", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889364, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000030", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889419, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000031", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889425, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000032", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889426, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000033", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889427, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000034", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889429, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000035", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889429, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000036", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889430, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000037", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889430, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000038", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889431, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000039", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889431, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000040", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889432, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000041", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889432, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000042", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889433, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000043", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889433, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000044", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889434, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000045", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - }, { - "attempts" : [ ], - "preferredLocations" : [ ], - "startTime" : 1240336889434, - "finishTime" : -1, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_r_000046", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : null, - "taskType" : "REDUCE" - } ], - "otherTasks" : [ { - "attempts" : [ { - "location" : { - "layers" : [ "194\\.6\\.134\\.64", "cluster50316\\.secondleveldomain\\.com" ] - }, - "hostName" : "/194\\.6\\.134\\.64/cluster50316\\.secondleveldomain\\.com", - "result" : "SUCCESS", - "startTime" : 1240336856225, - "finishTime" : 1240336858461, - "shuffleFinished" : -1, - "sortFinished" : -1, - "attemptID" : "attempt_200904211745_0004_m_000132_0", - "hdfsBytesRead" : -1, - "hdfsBytesWritten" : -1, - "fileBytesRead" : -1, - "fileBytesWritten" : -1, - "mapInputRecords" : -1, - "mapOutputBytes" : -1, - "mapOutputRecords" : -1, - "combineInputRecords" : -1, - "reduceInputGroups" : -1, - "reduceInputRecords" : -1, - "reduceShuffleBytes" : -1, - "reduceOutputRecords" : -1, - "spilledRecords" : 0, - "mapInputBytes" : -1 - } ], - "preferredLocations" : [ ], - "startTime" : 1240336855651, - "finishTime" : 1240336871747, - "inputBytes" : -1, - "inputRecords" : -1, - "outputBytes" : -1, - "outputRecords" : -1, - "taskID" : "task_200904211745_0004_m_000132", - "numberMaps" : -1, - "numberReduces" : -1, - "taskStatus" : "SUCCESS", - "taskType" : "SETUP" - } ], - "finishTime" : -1, - "user" : "geek3", - "jobName" : null, - "computonsPerMapInputByte" : -1, - "computonsPerMapOutputByte" : -1, - "computonsPerReduceInputByte" : -1, - "computonsPerReduceOutputByte" : -1, - "submitTime" : 1240336853354, - "launchTime" : 1240336854289, - "heapMegabytes" : 1234, - "totalMaps" : 131, - "totalReduces" : 47, - "outcome" : "SUCCESS", - "jobtype" : "JAVA", - "directDependantJobs" : [ ], - "successfulMapAttemptCDFs" : [ { - "maximum" : 43182, - "minimum" : 10721, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 10721 - }, { - "relativeRanking" : 0.1, - "datum" : 10721 - }, { - "relativeRanking" : 0.15, - "datum" : 11676 - }, { - "relativeRanking" : 0.2, - "datum" : 11676 - }, { - "relativeRanking" : 0.25, - "datum" : 12936 - }, { - "relativeRanking" : 0.3, - "datum" : 15451 - }, { - "relativeRanking" : 0.35, - "datum" : 15451 - }, { - "relativeRanking" : 0.4, - "datum" : 19204 - }, { - "relativeRanking" : 0.45, - "datum" : 21585 - }, { - "relativeRanking" : 0.5, - "datum" : 23169 - }, { - "relativeRanking" : 0.55, - "datum" : 23169 - }, { - "relativeRanking" : 0.6, - "datum" : 23595 - }, { - "relativeRanking" : 0.65, - "datum" : 27355 - }, { - "relativeRanking" : 0.7, - "datum" : 27355 - }, { - "relativeRanking" : 0.75, - "datum" : 36581 - }, { - "relativeRanking" : 0.8, - "datum" : 37035 - }, { - "relativeRanking" : 0.85, - "datum" : 37035 - }, { - "relativeRanking" : 0.9, - "datum" : 38983 - }, { - "relativeRanking" : 0.95, - "datum" : 39264 - } ], - "numberValues" : 14 - }, { - "maximum" : 75401, - "minimum" : 9950, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 11004 - }, { - "relativeRanking" : 0.1, - "datum" : 12016 - }, { - "relativeRanking" : 0.15, - "datum" : 13160 - }, { - "relativeRanking" : 0.2, - "datum" : 18594 - }, { - "relativeRanking" : 0.25, - "datum" : 21555 - }, { - "relativeRanking" : 0.3, - "datum" : 22014 - }, { - "relativeRanking" : 0.35, - "datum" : 23004 - }, { - "relativeRanking" : 0.4, - "datum" : 24102 - }, { - "relativeRanking" : 0.45, - "datum" : 27100 - }, { - "relativeRanking" : 0.5, - "datum" : 32357 - }, { - "relativeRanking" : 0.55, - "datum" : 34735 - }, { - "relativeRanking" : 0.6, - "datum" : 37787 - }, { - "relativeRanking" : 0.65, - "datum" : 39211 - }, { - "relativeRanking" : 0.7, - "datum" : 39508 - }, { - "relativeRanking" : 0.75, - "datum" : 40173 - }, { - "relativeRanking" : 0.8, - "datum" : 40512 - }, { - "relativeRanking" : 0.85, - "datum" : 43173 - }, { - "relativeRanking" : 0.9, - "datum" : 48444 - }, { - "relativeRanking" : 0.95, - "datum" : 56688 - } ], - "numberValues" : 92 - }, { - "maximum" : 49391, - "minimum" : 9945, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 9945 - }, { - "relativeRanking" : 0.1, - "datum" : 10537 - }, { - "relativeRanking" : 0.15, - "datum" : 10748 - }, { - "relativeRanking" : 0.2, - "datum" : 11007 - }, { - "relativeRanking" : 0.25, - "datum" : 11929 - }, { - "relativeRanking" : 0.3, - "datum" : 12330 - }, { - "relativeRanking" : 0.35, - "datum" : 13151 - }, { - "relativeRanking" : 0.4, - "datum" : 13826 - }, { - "relativeRanking" : 0.45, - "datum" : 14120 - }, { - "relativeRanking" : 0.5, - "datum" : 15115 - }, { - "relativeRanking" : 0.55, - "datum" : 17485 - }, { - "relativeRanking" : 0.6, - "datum" : 26817 - }, { - "relativeRanking" : 0.65, - "datum" : 28878 - }, { - "relativeRanking" : 0.7, - "datum" : 32445 - }, { - "relativeRanking" : 0.75, - "datum" : 34820 - }, { - "relativeRanking" : 0.8, - "datum" : 36092 - }, { - "relativeRanking" : 0.85, - "datum" : 36563 - }, { - "relativeRanking" : 0.9, - "datum" : 37231 - }, { - "relativeRanking" : 0.95, - "datum" : 39051 - } ], - "numberValues" : 25 - }, { - "maximum" : 2236, - "minimum" : 2236, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 2236 - }, { - "relativeRanking" : 0.1, - "datum" : 2236 - }, { - "relativeRanking" : 0.15, - "datum" : 2236 - }, { - "relativeRanking" : 0.2, - "datum" : 2236 - }, { - "relativeRanking" : 0.25, - "datum" : 2236 - }, { - "relativeRanking" : 0.3, - "datum" : 2236 - }, { - "relativeRanking" : 0.35, - "datum" : 2236 - }, { - "relativeRanking" : 0.4, - "datum" : 2236 - }, { - "relativeRanking" : 0.45, - "datum" : 2236 - }, { - "relativeRanking" : 0.5, - "datum" : 2236 - }, { - "relativeRanking" : 0.55, - "datum" : 2236 - }, { - "relativeRanking" : 0.6, - "datum" : 2236 - }, { - "relativeRanking" : 0.65, - "datum" : 2236 - }, { - "relativeRanking" : 0.7, - "datum" : 2236 - }, { - "relativeRanking" : 0.75, - "datum" : 2236 - }, { - "relativeRanking" : 0.8, - "datum" : 2236 - }, { - "relativeRanking" : 0.85, - "datum" : 2236 - }, { - "relativeRanking" : 0.9, - "datum" : 2236 - }, { - "relativeRanking" : 0.95, - "datum" : 2236 - } ], - "numberValues" : 1 - } ], - "failedMapAttemptCDFs" : [ { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, { - "maximum" : 18592, - "minimum" : 18416, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 18416 - }, { - "relativeRanking" : 0.1, - "datum" : 18416 - }, { - "relativeRanking" : 0.15, - "datum" : 18416 - }, { - "relativeRanking" : 0.2, - "datum" : 18416 - }, { - "relativeRanking" : 0.25, - "datum" : 18416 - }, { - "relativeRanking" : 0.3, - "datum" : 18416 - }, { - "relativeRanking" : 0.35, - "datum" : 18416 - }, { - "relativeRanking" : 0.4, - "datum" : 18416 - }, { - "relativeRanking" : 0.45, - "datum" : 18416 - }, { - "relativeRanking" : 0.5, - "datum" : 18416 - }, { - "relativeRanking" : 0.55, - "datum" : 18416 - }, { - "relativeRanking" : 0.6, - "datum" : 18416 - }, { - "relativeRanking" : 0.65, - "datum" : 18416 - }, { - "relativeRanking" : 0.7, - "datum" : 18584 - }, { - "relativeRanking" : 0.75, - "datum" : 18584 - }, { - "relativeRanking" : 0.8, - "datum" : 18584 - }, { - "relativeRanking" : 0.85, - "datum" : 18584 - }, { - "relativeRanking" : 0.9, - "datum" : 18584 - }, { - "relativeRanking" : 0.95, - "datum" : 18584 - } ], - "numberValues" : 3 - } ], - "successfulReduceAttemptCDF" : { - "maximum" : 96910, - "minimum" : 95583, - "rankings" : [ { - "relativeRanking" : 0.05, - "datum" : 95583 - }, { - "relativeRanking" : 0.1, - "datum" : 95751 - }, { - "relativeRanking" : 0.15, - "datum" : 96010 - }, { - "relativeRanking" : 0.2, - "datum" : 96013 - }, { - "relativeRanking" : 0.25, - "datum" : 96080 - }, { - "relativeRanking" : 0.3, - "datum" : 96128 - }, { - "relativeRanking" : 0.35, - "datum" : 96164 - }, { - "relativeRanking" : 0.4, - "datum" : 96227 - }, { - "relativeRanking" : 0.45, - "datum" : 96275 - }, { - "relativeRanking" : 0.5, - "datum" : 96290 - }, { - "relativeRanking" : 0.55, - "datum" : 96301 - }, { - "relativeRanking" : 0.6, - "datum" : 96320 - }, { - "relativeRanking" : 0.65, - "datum" : 96346 - }, { - "relativeRanking" : 0.7, - "datum" : 96412 - }, { - "relativeRanking" : 0.75, - "datum" : 96428 - }, { - "relativeRanking" : 0.8, - "datum" : 96530 - }, { - "relativeRanking" : 0.85, - "datum" : 96549 - }, { - "relativeRanking" : 0.9, - "datum" : 96658 - }, { - "relativeRanking" : 0.95, - "datum" : 96670 - } ], - "numberValues" : 28 - }, - "failedReduceAttemptCDF" : { - "maximum" : 9223372036854775807, - "minimum" : -9223372036854775808, - "rankings" : [ ], - "numberValues" : 0 - }, - "mapperTriesToSucceed" : [ 0.6567164179104478, 0.3283582089552239, 0.014925373134328358 ], - "failedMapperFraction" : 0.0, - "relativeTime" : 0, - "queue" : null, - "clusterMapMB" : -1, - "clusterReduceMB" : -1, - "jobMapMB" : -1, - "jobReduceMB" : -1 -} diff --git a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestJobTrackerPlugins.java b/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestJobTrackerPlugins.java deleted file mode 100644 index 7a86a0c2357..00000000000 --- a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestJobTrackerPlugins.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.util.ServicePlugin; -import org.junit.Test; - -public class TestJobTrackerPlugins extends TestCase { - - static class FakeServicePlugin implements ServicePlugin { - - private static FakeServicePlugin instance; - - public static FakeServicePlugin getInstance() { - return instance; - } - - private Object service; - private boolean stopped; - - public Object getService() { - return service; - } - - public boolean isStopped() { - return stopped; - } - - public FakeServicePlugin() { - // store static reference to instance so we can retrieve it in the test - instance = this; - } - - @Override - public void start(Object service) { - this.service = service; - } - - @Override - public void stop() { - stopped = true; - } - - @Override - public void close() throws IOException { - } - } - - @Test - public void test() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setClass(JTConfig.JT_PLUGINS, FakeServicePlugin.class, - ServicePlugin.class); - - assertNull("Plugin not created", FakeServicePlugin.getInstance()); - - JobTracker jobTracker = JobTracker.startTracker(conf); - assertNotNull("Plugin created", FakeServicePlugin.getInstance()); - assertSame("Service is jobTracker", - FakeServicePlugin.getInstance().getService(), jobTracker); - assertFalse("Plugin not stopped", - FakeServicePlugin.getInstance().isStopped()); - - jobTracker.close(); - assertTrue("Plugin stopped", FakeServicePlugin.getInstance().isStopped()); - } - -} diff --git a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java b/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java deleted file mode 100644 index 99035b808ce..00000000000 --- a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.mockito.Mockito.*; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.hadoop.mapred.UtilsForTests.FakeClock; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; -import org.hamcrest.Matcher; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatcher; - -/** - * Tests that trackers that don't heartbeat within a given time are considered - * lost. Note that this test is not a direct replacement for - * {@link TestLostTracker} since it doesn't test that a task - * running on a lost tracker is retried on another tracker. - */ -@SuppressWarnings("deprecation") -public class TestLostTaskTracker extends TestCase { - - private JobTracker jobTracker; - - private FakeClock clock; - - @Override - protected void setUp() throws Exception { - JobConf conf = new JobConf(); - conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); - conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); - conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); - clock = new FakeClock(); - // We use a "partial mock" of JobTracker which lets us see when certain - // methods are called. If we were writing JobTracker from scratch then - // we would make it call another object which we would mock out instead - // (and use a real JobTracker) so we could perform assertions on the mock. - // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#16 - jobTracker = spy(new JobTracker(conf, clock)); - } - - public void testLostTaskTrackerCalledAfterExpiryTime() throws IOException { - - String tracker1 = "tracker_tracker1:1000"; - String tracker2 = "tracker_tracker2:1000"; - - establishFirstContact(tracker1); - - // Wait long enough for tracker1 to be considered lost - // We could have used a Mockito stub here, except we don't know how many - // times JobTracker calls getTime() on the clock, so a static mock - // is appropriate. - clock.advance(8 * 1000); - - establishFirstContact(tracker2); - - jobTracker.checkExpiredTrackers(); - - // Now we check that JobTracker's lostTaskTracker() was called for tracker1 - // but not for tracker2. - - // We use an ArgumentCaptor to capture the task tracker object - // in the lostTaskTracker() call, so we can perform an assertion on its - // name. (We could also have used a custom matcher, see below.) - // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#15 - ArgumentCaptor argument = - ArgumentCaptor.forClass(TaskTracker.class); - - verify(jobTracker).lostTaskTracker(argument.capture()); - assertEquals(tracker1, argument.getValue().getTrackerName()); - - // Check tracker2 was not lost by using the never() construct - // We use a custom Hamcrest matcher to check that it was indeed tracker2 - // that didn't match (since tracker1 did match). - // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#3 - verify(jobTracker, never()).lostTaskTracker( - argThat(taskTrackerWithName(tracker2))); - } - - private Matcher taskTrackerWithName(final String name) { - return new ArgumentMatcher() { - public boolean matches(Object taskTracker) { - return name.equals(((TaskTracker) taskTracker).getTrackerName()); - } - }; - } - - private void establishFirstContact(String tracker) throws IOException { - TaskTrackerStatus status = new TaskTrackerStatus(tracker, - JobInProgress.convertTrackerNameToHostName(tracker)); - jobTracker.heartbeat(status, false, true, false, (short) 0); - } - -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java b/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java deleted file mode 100644 index 7e7235c1a85..00000000000 --- a/hadoop-mapreduce-project/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred; - -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.hadoop.mapreduce.MRConfig; -import org.junit.Test; -import org.junit.Before; - -/** - * Tests for the correct behavior of the TaskTracker starting up with - * respect to its local-disk directories. - */ -public class TestTaskTrackerDirectories { - private final String TEST_DIR = new File("build/test/testmapredlocaldir") - .getAbsolutePath(); - - @Before - public void deleteTestDir() throws IOException { - FileUtil.fullyDelete(new File(TEST_DIR)); - assertFalse("Could not delete " + TEST_DIR, - new File(TEST_DIR).exists()); - } - - @Test - public void testCreatesLocalDirs() throws Exception { - Configuration conf = new Configuration(); - String[] dirs = new String[] { - TEST_DIR + "/local1", - TEST_DIR + "/local2" - }; - - conf.setStrings(MRConfig.LOCAL_DIR, dirs); - setupTaskController(conf); - - for (String dir : dirs) { - checkDir(dir); - } - } - - @Test - public void testFixesLocalDirPermissions() throws Exception { - Configuration conf = new Configuration(); - String[] dirs = new String[] { - TEST_DIR + "/badperms" - }; - - new File(dirs[0]).mkdirs(); - FileUtil.chmod(dirs[0], "000"); - - conf.setStrings(MRConfig.LOCAL_DIR, dirs); - setupTaskController(conf); - - for (String dir : dirs) { - checkDir(dir); - } - } - - @Test - public void testCreatesLogDir() throws Exception { - File dir = TaskLog.getUserLogDir(); - FileUtil.fullyDelete(dir); - - setupTaskController(new Configuration()); - - checkDir(dir.getAbsolutePath()); - } - - /** - * If the log dir can't be created, the TT should fail to start since - * it will be unable to localize or run tasks. - */ - @Test - public void testCantCreateLogDir() throws Exception { - File dir = TaskLog.getUserLogDir(); - FileUtil.fullyDelete(dir); - assertTrue("Making file in place of log dir", - dir.createNewFile()); - - try { - setupTaskController(new Configuration()); - fail("Didn't throw!"); - } catch (IOException ioe) { - System.err.println("Got expected exception"); - ioe.printStackTrace(System.out); - } - } - - @Test - public void testFixesLogDirPermissions() throws Exception { - File dir = TaskLog.getUserLogDir(); - FileUtil.fullyDelete(dir); - dir.mkdirs(); - FileUtil.chmod(dir.getAbsolutePath(), "000"); - - setupTaskController(new Configuration()); - - checkDir(dir.getAbsolutePath()); - } - - private void setupTaskController(Configuration conf) throws IOException { - TaskController tc = new DefaultTaskController(); - tc.setConf(conf); - tc.setup(); - } - - private void checkDir(String dir) throws IOException { - FileSystem fs = RawLocalFileSystem.get(new Configuration()); - File f = new File(dir); - assertTrue(dir + "should exist", f.exists()); - FileStatus stat = fs.getFileStatus(new Path(dir)); - assertEquals(dir + " has correct permissions", - 0755, stat.getPermission().toShort()); - } -} diff --git a/hadoop-mapreduce-project/src/tools/org/apache/hadoop/fs/package-info.java b/hadoop-mapreduce-project/src/tools/org/apache/hadoop/fs/package-info.java deleted file mode 100644 index 67ca9584e8d..00000000000 --- a/hadoop-mapreduce-project/src/tools/org/apache/hadoop/fs/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Hadoop filesystem classes for MapReduce. - */ -package org.apache.hadoop.fs; diff --git a/hadoop-mapreduce-project/src/webapps/job/analysejobhistory.jsp b/hadoop-mapreduce-project/src/webapps/job/analysejobhistory.jsp deleted file mode 100644 index 33a5828e9a3..00000000000 --- a/hadoop-mapreduce-project/src/webapps/job/analysejobhistory.jsp +++ /dev/null @@ -1,254 +0,0 @@ -<% -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -%> -<%@ page - contentType="text/html; charset=UTF-8" - import="javax.servlet.http.*" - import="java.io.*" - import="java.util.*" - import="org.apache.hadoop.http.HtmlQuoting" - import="org.apache.hadoop.mapred.*" - import="org.apache.hadoop.fs.*" - import="org.apache.hadoop.util.*" - import="java.text.SimpleDateFormat" - import="org.apache.hadoop.mapreduce.jobhistory.*" -%> - -<%! private static SimpleDateFormat dateFormat - = new SimpleDateFormat("d/MM HH:mm:ss") ; -%> -<%! private static final long serialVersionUID = 1L; -%> - - -<% - String logFile = request.getParameter("logFile"); - String numTasks = request.getParameter("numTasks"); - int showTasks = 10 ; - if (numTasks != null) { - showTasks = Integer.parseInt(numTasks); - } - FileSystem fs = (FileSystem) application.getAttribute("fileSys"); - JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker"); - JobHistoryParser.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, - response, jobTracker, fs, new Path(logFile)); - if (job == null) { - return; - } -%> -