From 28c308d5e81432b79f9c9e14df316a52cc7ba48f Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 31 Jan 2013 00:28:10 +0000 Subject: [PATCH 01/69] MAPREDUCE-4893. Fixed MR ApplicationMaster to do optimal assignment of containers to get maximum locality. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1440749 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/rm/RMContainerAllocator.java | 250 ++++++++++-------- .../v2/app/TestRMContainerAllocator.java | 88 +++++- 3 files changed, 236 insertions(+), 105 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index efdd1071d61..643006bbb18 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -217,6 +217,9 @@ Release 2.0.3-alpha - Unreleased OPTIMIZATIONS + MAPREDUCE-4893. Fixed MR ApplicationMaster to do optimal assignment of + containers to get maximum locality. (Bikas Saha via vinodkv) + BUG FIXES MAPREDUCE-4607. Race condition in ReduceTask completion can result in Task diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 010ebc983f6..d21528aa376 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -747,7 +747,7 @@ public class RMContainerAllocator extends RMContainerRequestor addContainerReq(req); } - @SuppressWarnings("unchecked") + // this method will change the list of allocatedContainers. private void assign(List allocatedContainers) { Iterator it = allocatedContainers.iterator(); LOG.info("Got allocated containers " + allocatedContainers.size()); @@ -788,84 +788,97 @@ public class RMContainerAllocator extends RMContainerRequestor + reduces.isEmpty()); isAssignable = false; } - } + } else { + LOG.warn("Container allocated at unwanted priority: " + priority + + ". Returning to RM..."); + isAssignable = false; + } - boolean blackListed = false; - ContainerRequest assigned = null; + if(!isAssignable) { + // release container if we could not assign it + containerNotAssigned(allocated); + it.remove(); + continue; + } - if (isAssignable) { - // do not assign if allocated container is on a - // blacklisted host - String allocatedHost = allocated.getNodeId().getHost(); - blackListed = isNodeBlacklisted(allocatedHost); - if (blackListed) { - // we need to request for a new container - // and release the current one - LOG.info("Got allocated container on a blacklisted " - + " host "+allocatedHost - +". Releasing container " + allocated); + // do not assign if allocated container is on a + // blacklisted host + String allocatedHost = allocated.getNodeId().getHost(); + if (isNodeBlacklisted(allocatedHost)) { + // we need to request for a new container + // and release the current one + LOG.info("Got allocated container on a blacklisted " + + " host "+allocatedHost + +". Releasing container " + allocated); - // find the request matching this allocated container - // and replace it with a new one - ContainerRequest toBeReplacedReq = - getContainerReqToReplace(allocated); - if (toBeReplacedReq != null) { - LOG.info("Placing a new container request for task attempt " - + toBeReplacedReq.attemptID); - ContainerRequest newReq = - getFilteredContainerRequest(toBeReplacedReq); - decContainerReq(toBeReplacedReq); - if (toBeReplacedReq.attemptID.getTaskId().getTaskType() == - TaskType.MAP) { - maps.put(newReq.attemptID, newReq); - } - else { - reduces.put(newReq.attemptID, newReq); - } - addContainerReq(newReq); + // find the request matching this allocated container + // and replace it with a new one + ContainerRequest toBeReplacedReq = + getContainerReqToReplace(allocated); + if (toBeReplacedReq != null) { + LOG.info("Placing a new container request for task attempt " + + toBeReplacedReq.attemptID); + ContainerRequest newReq = + getFilteredContainerRequest(toBeReplacedReq); + decContainerReq(toBeReplacedReq); + if (toBeReplacedReq.attemptID.getTaskId().getTaskType() == + TaskType.MAP) { + maps.put(newReq.attemptID, newReq); } else { - LOG.info("Could not map allocated container to a valid request." - + " Releasing allocated container " + allocated); + reduces.put(newReq.attemptID, newReq); } + addContainerReq(newReq); } else { - assigned = assign(allocated); - if (assigned != null) { - // Update resource requests - decContainerReq(assigned); - - // send the container-assigned event to task attempt - eventHandler.handle(new TaskAttemptContainerAssignedEvent( - assigned.attemptID, allocated, applicationACLs)); - - assignedRequests.add(allocated, assigned.attemptID); - - if (LOG.isDebugEnabled()) { - LOG.info("Assigned container (" + allocated + ") " - + " to task " + assigned.attemptID + " on node " - + allocated.getNodeId().toString()); - } - } - else { - //not assigned to any request, release the container - LOG.info("Releasing unassigned and invalid container " - + allocated + ". RM has gone crazy, someone go look!" - + " Hey RM, if you are so rich, go donate to non-profits!"); - } + LOG.info("Could not map allocated container to a valid request." + + " Releasing allocated container " + allocated); } + + // release container if we could not assign it + containerNotAssigned(allocated); + it.remove(); + continue; } - - // release container if it was blacklisted - // or if we could not assign it - if (blackListed || assigned == null) { - containersReleased++; - release(allocated.getId()); - } + } + + assignContainers(allocatedContainers); + + // release container if we could not assign it + it = allocatedContainers.iterator(); + while (it.hasNext()) { + Container allocated = it.next(); + LOG.info("Releasing unassigned and invalid container " + + allocated + ". RM may have assignment issues"); + containerNotAssigned(allocated); } } - private ContainerRequest assign(Container allocated) { + @SuppressWarnings("unchecked") + private void containerAssigned(Container allocated, + ContainerRequest assigned) { + // Update resource requests + decContainerReq(assigned); + + // send the container-assigned event to task attempt + eventHandler.handle(new TaskAttemptContainerAssignedEvent( + assigned.attemptID, allocated, applicationACLs)); + + assignedRequests.add(allocated, assigned.attemptID); + + if (LOG.isDebugEnabled()) { + LOG.info("Assigned container (" + allocated + ") " + + " to task " + assigned.attemptID + " on node " + + allocated.getNodeId().toString()); + } + } + + private void containerNotAssigned(Container allocated) { + containersReleased++; + release(allocated.getId()); + } + + private ContainerRequest assignWithoutLocality(Container allocated) { ContainerRequest assigned = null; Priority priority = allocated.getPriority(); @@ -877,18 +890,24 @@ public class RMContainerAllocator extends RMContainerRequestor LOG.debug("Assigning container " + allocated + " to reduce"); } assigned = assignToReduce(allocated); - } else if (PRIORITY_MAP.equals(priority)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Assigning container " + allocated + " to map"); - } - assigned = assignToMap(allocated); - } else { - LOG.warn("Container allocated at unwanted priority: " + priority + - ". Returning to RM..."); } return assigned; } + + private void assignContainers(List allocatedContainers) { + Iterator it = allocatedContainers.iterator(); + while (it.hasNext()) { + Container allocated = it.next(); + ContainerRequest assigned = assignWithoutLocality(allocated); + if (assigned != null) { + containerAssigned(allocated, assigned); + it.remove(); + } + } + + assignMapsWithLocality(allocatedContainers); + } private ContainerRequest getContainerReqToReplace(Container allocated) { LOG.info("Finding containerReq for allocated container: " + allocated); @@ -959,11 +978,15 @@ public class RMContainerAllocator extends RMContainerRequestor } @SuppressWarnings("unchecked") - private ContainerRequest assignToMap(Container allocated) { - //try to assign to maps if present - //first by host, then by rack, followed by * - ContainerRequest assigned = null; - while (assigned == null && maps.size() > 0) { + private void assignMapsWithLocality(List allocatedContainers) { + // try to assign to all nodes first to match node local + Iterator it = allocatedContainers.iterator(); + while(it.hasNext() && maps.size() > 0){ + Container allocated = it.next(); + Priority priority = allocated.getPriority(); + assert PRIORITY_MAP.equals(priority); + // "if (maps.containsKey(tId))" below should be almost always true. + // hence this while loop would almost always have O(1) complexity String host = allocated.getNodeId().getHost(); LinkedList list = mapsHostMapping.get(host); while (list != null && list.size() > 0) { @@ -972,7 +995,9 @@ public class RMContainerAllocator extends RMContainerRequestor } TaskAttemptId tId = list.removeFirst(); if (maps.containsKey(tId)) { - assigned = maps.remove(tId); + ContainerRequest assigned = maps.remove(tId); + containerAssigned(allocated, assigned); + it.remove(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); jce.addCounterUpdate(JobCounter.DATA_LOCAL_MAPS, 1); @@ -984,39 +1009,56 @@ public class RMContainerAllocator extends RMContainerRequestor break; } } - if (assigned == null) { - String rack = RackResolver.resolve(host).getNetworkLocation(); - list = mapsRackMapping.get(rack); - while (list != null && list.size() > 0) { - TaskAttemptId tId = list.removeFirst(); - if (maps.containsKey(tId)) { - assigned = maps.remove(tId); - JobCounterUpdateEvent jce = - new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); - jce.addCounterUpdate(JobCounter.RACK_LOCAL_MAPS, 1); - eventHandler.handle(jce); - rackLocalAssigned++; - if (LOG.isDebugEnabled()) { - LOG.debug("Assigned based on rack match " + rack); - } - break; - } - } - if (assigned == null && maps.size() > 0) { - TaskAttemptId tId = maps.keySet().iterator().next(); - assigned = maps.remove(tId); + } + + // try to match all rack local + it = allocatedContainers.iterator(); + while(it.hasNext() && maps.size() > 0){ + Container allocated = it.next(); + Priority priority = allocated.getPriority(); + assert PRIORITY_MAP.equals(priority); + // "if (maps.containsKey(tId))" below should be almost always true. + // hence this while loop would almost always have O(1) complexity + String host = allocated.getNodeId().getHost(); + String rack = RackResolver.resolve(host).getNetworkLocation(); + LinkedList list = mapsRackMapping.get(rack); + while (list != null && list.size() > 0) { + TaskAttemptId tId = list.removeFirst(); + if (maps.containsKey(tId)) { + ContainerRequest assigned = maps.remove(tId); + containerAssigned(allocated, assigned); + it.remove(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); - jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1); + jce.addCounterUpdate(JobCounter.RACK_LOCAL_MAPS, 1); eventHandler.handle(jce); + rackLocalAssigned++; if (LOG.isDebugEnabled()) { - LOG.debug("Assigned based on * match"); + LOG.debug("Assigned based on rack match " + rack); } break; } } } - return assigned; + + // assign remaining + it = allocatedContainers.iterator(); + while(it.hasNext() && maps.size() > 0){ + Container allocated = it.next(); + Priority priority = allocated.getPriority(); + assert PRIORITY_MAP.equals(priority); + TaskAttemptId tId = maps.keySet().iterator().next(); + ContainerRequest assigned = maps.remove(tId); + containerAssigned(allocated, assigned); + it.remove(); + JobCounterUpdateEvent jce = + new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); + jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1); + eventHandler.handle(jce); + if (LOG.isDebugEnabled()) { + LOG.debug("Assigned based on * match"); + } + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 68c7f00fcaa..bd00e1b1608 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -190,6 +190,92 @@ public class TestRMContainerAllocator { checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 }, assigned, false); } + + @Test + public void testMapNodeLocality() throws Exception { + // test checks that ordering of allocated containers list from the RM does + // not affect the map->container assignment done by the AM. If there is a + // node local container available for a map then it should be assigned to + // that container and not a rack-local container that happened to be seen + // earlier in the allocated containers list from the RM. + // Regression test for MAPREDUCE-4893 + LOG.info("Running testMapNodeLocality"); + + Configuration conf = new Configuration(); + MyResourceManager rm = new MyResourceManager(conf); + rm.start(); + DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() + .getDispatcher(); + + // Submit the application + RMApp app = rm.submitApp(1024); + dispatcher.await(); + + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + amNodeManager.nodeHeartbeat(true); + dispatcher.await(); + + ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() + .getAppAttemptId(); + rm.sendAMLaunched(appAttemptId); + dispatcher.await(); + + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); + Job mockJob = mock(Job.class); + when(mockJob.getReport()).thenReturn( + MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, + 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); + MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, + appAttemptId, mockJob); + + // add resources to scheduler + MockNM nodeManager1 = rm.registerNode("h1:1234", 3072); // can assign 2 maps + rm.registerNode("h2:1234", 10240); // wont heartbeat on node local node + MockNM nodeManager3 = rm.registerNode("h3:1234", 1536); // assign 1 map + dispatcher.await(); + + // create the container requests for maps + ContainerRequestEvent event1 = createReq(jobId, 1, 1024, + new String[] { "h1" }); + allocator.sendRequest(event1); + ContainerRequestEvent event2 = createReq(jobId, 2, 1024, + new String[] { "h1" }); + allocator.sendRequest(event2); + ContainerRequestEvent event3 = createReq(jobId, 3, 1024, + new String[] { "h2" }); + allocator.sendRequest(event3); + + // this tells the scheduler about the requests + // as nodes are not added, no allocations + List assigned = allocator.schedule(); + dispatcher.await(); + Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); + + // update resources in scheduler + // Node heartbeat from rack-local first. This makes node h3 the first in the + // list of allocated containers but it should not be assigned to task1. + nodeManager3.nodeHeartbeat(true); + // Node heartbeat from node-local next. This allocates 2 node local + // containers for task1 and task2. These should be matched with those tasks. + nodeManager1.nodeHeartbeat(true); + dispatcher.await(); + + assigned = allocator.schedule(); + dispatcher.await(); + checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 }, + assigned, false); + // remove the rack-local assignment that should have happened for task3 + for(TaskAttemptContainerAssignedEvent event : assigned) { + if(event.getTaskAttemptID().equals(event3.getAttemptID())) { + assigned.remove(event); + Assert.assertTrue( + event.getContainer().getNodeId().getHost().equals("h3")); + break; + } + } + checkAssignments(new ContainerRequestEvent[] { event1, event2}, + assigned, true); + } @Test public void testResource() throws Exception { @@ -1202,7 +1288,7 @@ public class TestRMContainerAllocator { if (checkHostMatch) { Assert.assertTrue("Not assigned to requested host", Arrays.asList( request.getHosts()).contains( - assigned.getContainer().getNodeId().toString())); + assigned.getContainer().getNodeId().getHost())); } } From e668c16975d90f673c3cfdfaeae5d24a1918d586 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 01:37:16 +0000 Subject: [PATCH 02/69] HADOOP-8981. Merge r1422279 from branch-trunk-win to trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1440780 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../metrics2/impl/TestMetricsSystemImpl.java | 56 +++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 87cc2496c46..855a77fbd6b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -588,6 +588,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-9221. Convert remaining xdocs to APT. (Andy Isaacson via atm) + HADOOP-8981. TestMetricsSystemImpl fails on Windows. (Xuan Gong via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java index 0aa19032cdc..7f53a7da41d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java @@ -56,6 +56,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; /** * Test the MetricsSystemImpl class @@ -80,7 +81,7 @@ public class TestMetricsSystemImpl { } } - @Test public void testInitFirst() throws Exception { + @Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception { new ConfigBuilder().add("*.period", 8) //.add("test.sink.plugin.urls", getPluginUrlsAsString()) .add("test.sink.test.class", TestSink.class.getName()) @@ -106,14 +107,61 @@ public class TestMetricsSystemImpl { ms.stop(); ms.shutdown(); - verify(sink1, times(2)).putMetrics(r1.capture()); + //When we call stop, at most two sources will be consumed by each sink thread. + verify(sink1, atMost(2)).putMetrics(r1.capture()); + List mr1 = r1.getAllValues(); + verify(sink2, atMost(2)).putMetrics(r2.capture()); + List mr2 = r2.getAllValues(); + if (mr1.size() != 0 && mr2.size() != 0) { + checkMetricsRecords(mr1); + assertEquals("output", mr1, mr2); + } else if (mr1.size() != 0) { + checkMetricsRecords(mr1); + } else if (mr2.size() != 0) { + checkMetricsRecords(mr2); + } + } + + @Test public void testInitFirstVerifyCallBacks() throws Exception { + DefaultMetricsSystem.shutdown(); + new ConfigBuilder().add("*.period", 8) + //.add("test.sink.plugin.urls", getPluginUrlsAsString()) + .add("test.sink.test.class", TestSink.class.getName()) + .add("test.*.source.filter.exclude", "s0") + .add("test.source.s1.metric.filter.exclude", "X*") + .add("test.sink.sink1.metric.filter.exclude", "Y*") + .add("test.sink.sink2.metric.filter.exclude", "Y*") + .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test")); + MetricsSystemImpl ms = new MetricsSystemImpl("Test"); + ms.start(); + ms.register("s0", "s0 desc", new TestSource("s0rec")); + TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec")); + s1.c1.incr(); + s1.xxx.incr(); + s1.g1.set(2); + s1.yyy.incr(2); + s1.s1.add(0); + MetricsSink sink1 = mock(MetricsSink.class); + MetricsSink sink2 = mock(MetricsSink.class); + ms.registerSink("sink1", "sink1 desc", sink1); + ms.registerSink("sink2", "sink2 desc", sink2); + ms.publishMetricsNow(); // publish the metrics + + try { + verify(sink1, timeout(200).times(2)).putMetrics(r1.capture()); + verify(sink2, timeout(200).times(2)).putMetrics(r2.capture()); + } finally { + ms.stop(); + ms.shutdown(); + } + //When we call stop, at most two sources will be consumed by each sink thread. List mr1 = r1.getAllValues(); - verify(sink2, times(2)).putMetrics(r2.capture()); List mr2 = r2.getAllValues(); checkMetricsRecords(mr1); assertEquals("output", mr1, mr2); - } + } + @Test public void testMultiThreadedPublish() throws Exception { new ConfigBuilder().add("*.period", 80) .add("test.sink.Collector.queue.capacity", "20") From e7380b4f9c4f29bf5f0c07b95ddba88b0ef3765d Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 31 Jan 2013 07:29:46 +0000 Subject: [PATCH 03/69] HDFS-4428. FsDatasetImpl should disclose what the error is when a rename fails. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1440865 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/io/nativeio/Errno.java | 3 ++ .../apache/hadoop/io/nativeio/NativeIO.java | 32 +++++++++++++++ .../org/apache/hadoop/io/nativeio/NativeIO.c | 23 ++++++++++- .../apache/hadoop/io/nativeio/errno_enum.c | 3 ++ .../hadoop/io/nativeio/TestNativeIO.java | 39 +++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../fsdataset/impl/FsDatasetImpl.java | 37 ++++++++++++------ 7 files changed, 127 insertions(+), 13 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java index b48f76da18e..f823978bd71 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/Errno.java @@ -55,6 +55,9 @@ public enum Errno { EPIPE, EDOM, ERANGE, + ELOOP, + ENAMETOOLONG, + ENOTEMPTY, UNKNOWN; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 94ff5f6057f..c8649c6a2e9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.io.nativeio; +import java.io.File; import java.io.FileDescriptor; import java.io.IOException; import java.util.Map; @@ -293,4 +294,35 @@ public class NativeIO { stat.group = getName(IdCache.GROUP, stat.groupId); return stat; } + + /** + * A version of renameTo that throws a descriptive exception when it fails. + * + * @param src The source path + * @param dst The destination path + * + * @throws NativeIOException On failure. + */ + public static void renameTo(File src, File dst) + throws IOException { + if (!nativeLoaded) { + if (!src.renameTo(dst)) { + throw new IOException("renameTo(src=" + src + ", dst=" + + dst + ") failed."); + } + } else { + renameTo0(src.getAbsolutePath(), dst.getAbsolutePath()); + } + } + + /** + * A version of renameTo that throws a descriptive exception when it fails. + * + * @param src The source path + * @param dst The destination path + * + * @throws NativeIOException On failure. + */ + private static native void renameTo0(String src, String dst) + throws NativeIOException; } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 7e82152c1fe..be957b447ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -24,11 +24,12 @@ #include #include #include +#include #include #include #include -#include #include +#include #include #include "config.h" @@ -502,6 +503,26 @@ ssize_t get_pw_buflen() { #endif return (ret > 512) ? ret : 512; } + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_renameTo0(JNIEnv *env, +jclass clazz, jstring jsrc, jstring jdst) +{ + const char *src = NULL, *dst = NULL; + + src = (*env)->GetStringUTFChars(env, jsrc, NULL); + if (!src) goto done; // exception was thrown + dst = (*env)->GetStringUTFChars(env, jdst, NULL); + if (!dst) goto done; // exception was thrown + if (rename(src, dst)) { + throw_ioe(env, errno); + } + +done: + if (src) (*env)->ReleaseStringUTFChars(env, jsrc, src); + if (dst) (*env)->ReleaseStringUTFChars(env, jdst, dst); +} + /** * vim: sw=2: ts=2: et: */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c index 76d1ff17252..4d07c31394a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/errno_enum.c @@ -63,6 +63,9 @@ static errno_mapping_t ERRNO_MAPPINGS[] = { MAPPING(EPIPE), MAPPING(EDOM), MAPPING(ERANGE), + MAPPING(ELOOP), + MAPPING(ENAMETOOLONG), + MAPPING(ENOTEMPTY), {-1, NULL} }; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java index 9ee1516863b..f77e7288d62 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java @@ -25,11 +25,14 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicReference; import java.util.ArrayList; import java.util.List; + +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import static org.junit.Assume.*; import static org.junit.Assert.*; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -293,4 +296,40 @@ public class TestNativeIO { assertFalse(NativeIO.getGroupName(0).isEmpty()); } + @Test + public void testRenameTo() throws Exception { + final File TEST_DIR = new File(new File( + System.getProperty("test.build.data","build/test/data")), "renameTest"); + assumeTrue(TEST_DIR.mkdirs()); + File nonExistentFile = new File(TEST_DIR, "nonexistent"); + File targetFile = new File(TEST_DIR, "target"); + // Test attempting to rename a nonexistent file. + try { + NativeIO.renameTo(nonExistentFile, targetFile); + Assert.fail(); + } catch (NativeIOException e) { + Assert.assertEquals(e.getErrno(), Errno.ENOENT); + } + + // Test renaming a file to itself. It should succeed and do nothing. + File sourceFile = new File(TEST_DIR, "source"); + Assert.assertTrue(sourceFile.createNewFile()); + NativeIO.renameTo(sourceFile, sourceFile); + + // Test renaming a source to a destination. + NativeIO.renameTo(sourceFile, targetFile); + + // Test renaming a source to a path which uses a file as a directory. + sourceFile = new File(TEST_DIR, "source"); + Assert.assertTrue(sourceFile.createNewFile()); + File badTarget = new File(targetFile, "subdir"); + try { + NativeIO.renameTo(sourceFile, badTarget); + Assert.fail(); + } catch (NativeIOException e) { + Assert.assertEquals(e.getErrno(), Errno.ENOTDIR); + } + + FileUtils.deleteQuietly(TEST_DIR); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9512f26e7cb..a314d0f7e75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -735,6 +735,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4444. Add space between total transaction time and number of transactions in FSEditLog#printStatistics. (Stephen Chu via suresh) + HDFS-4428. FsDatasetImpl should disclose what the error is when a rename + fails. (Colin Patrick McCabe via atm) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index c65e5faa405..caf970de6a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -75,6 +75,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -398,13 +399,17 @@ class FsDatasetImpl implements FsDatasetSpi { final File dstfile = new File(destdir, b.getBlockName()); final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp()); final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp()); - if (!srcmeta.renameTo(dstmeta)) { + try { + NativeIO.renameTo(srcmeta, dstmeta); + } catch (IOException e) { throw new IOException("Failed to move meta file for " + b - + " from " + srcmeta + " to " + dstmeta); + + " from " + srcmeta + " to " + dstmeta, e); } - if (!srcfile.renameTo(dstfile)) { + try { + NativeIO.renameTo(srcfile, dstfile); + } catch (IOException e) { throw new IOException("Failed to move block file for " + b - + " from " + srcfile + " to " + dstfile.getAbsolutePath()); + + " from " + srcfile + " to " + dstfile.getAbsolutePath(), e); } if (LOG.isDebugEnabled()) { LOG.debug("addBlock: Moved " + srcmeta + " to " + dstmeta @@ -531,10 +536,12 @@ class FsDatasetImpl implements FsDatasetSpi { if (LOG.isDebugEnabled()) { LOG.debug("Renaming " + oldmeta + " to " + newmeta); } - if (!oldmeta.renameTo(newmeta)) { + try { + NativeIO.renameTo(oldmeta, newmeta); + } catch (IOException e) { throw new IOException("Block " + replicaInfo + " reopen failed. " + " Unable to move meta file " + oldmeta + - " to rbw dir " + newmeta); + " to rbw dir " + newmeta, e); } // rename block file to rbw directory @@ -542,14 +549,18 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.debug("Renaming " + blkfile + " to " + newBlkFile + ", file length=" + blkfile.length()); } - if (!blkfile.renameTo(newBlkFile)) { - if (!newmeta.renameTo(oldmeta)) { // restore the meta file + try { + NativeIO.renameTo(blkfile, newBlkFile); + } catch (IOException e) { + try { + NativeIO.renameTo(newmeta, oldmeta); + } catch (IOException ex) { LOG.warn("Cannot move meta file " + newmeta + - "back to the finalized directory " + oldmeta); + "back to the finalized directory " + oldmeta, ex); } throw new IOException("Block " + replicaInfo + " reopen failed. " + " Unable to move block file " + blkfile + - " to rbw dir " + newBlkFile); + " to rbw dir " + newBlkFile, e); } // Replace finalized replica by a RBW replica in replicas map @@ -656,11 +667,13 @@ class FsDatasetImpl implements FsDatasetSpi { if (LOG.isDebugEnabled()) { LOG.debug("Renaming " + oldmeta + " to " + newmeta); } - if (!oldmeta.renameTo(newmeta)) { + try { + NativeIO.renameTo(oldmeta, newmeta); + } catch (IOException e) { replicaInfo.setGenerationStamp(oldGS); // restore old GS throw new IOException("Block " + replicaInfo + " reopen failed. " + " Unable to move meta file " + oldmeta + - " to " + newmeta); + " to " + newmeta, e); } } From ac21e15a3920dbd222ad7648d49c4c763ff3a5e5 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 17:11:36 +0000 Subject: [PATCH 04/69] MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7. Contributed by Arpit Agarwal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441069 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/TestKeyValueTextInputFormat.java | 84 ++++++++++++------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 643006bbb18..e405fde57dc 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -281,6 +281,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-2264. Job status exceeds 100% in some cases. (devaraj.k and sandyr via tucu) + MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7. + (Arpit Agarwal via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java index 3846bbedc0f..27070783e14 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java @@ -136,32 +136,47 @@ public class TestKeyValueTextInputFormat extends TestCase { } public void testUTF8() throws Exception { - LineReader in = makeStream("abcd\u20acbdcd\u20ac"); - Text line = new Text(); - in.readLine(line); - assertEquals("readLine changed utf8 characters", - "abcd\u20acbdcd\u20ac", line.toString()); - in = makeStream("abc\u200axyz"); - in.readLine(line); - assertEquals("split on fake newline", "abc\u200axyz", line.toString()); + LineReader in = null; + + try { + in = makeStream("abcd\u20acbdcd\u20ac"); + Text line = new Text(); + in.readLine(line); + assertEquals("readLine changed utf8 characters", + "abcd\u20acbdcd\u20ac", line.toString()); + in = makeStream("abc\u200axyz"); + in.readLine(line); + assertEquals("split on fake newline", "abc\u200axyz", line.toString()); + } finally { + if (in != null) { + in.close(); + } + } } public void testNewLines() throws Exception { - LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee"); - Text out = new Text(); - in.readLine(out); - assertEquals("line1 length", 1, out.getLength()); - in.readLine(out); - assertEquals("line2 length", 2, out.getLength()); - in.readLine(out); - assertEquals("line3 length", 0, out.getLength()); - in.readLine(out); - assertEquals("line4 length", 3, out.getLength()); - in.readLine(out); - assertEquals("line5 length", 4, out.getLength()); - in.readLine(out); - assertEquals("line5 length", 5, out.getLength()); - assertEquals("end of file", 0, in.readLine(out)); + LineReader in = null; + try { + in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee"); + Text out = new Text(); + in.readLine(out); + assertEquals("line1 length", 1, out.getLength()); + in.readLine(out); + assertEquals("line2 length", 2, out.getLength()); + in.readLine(out); + assertEquals("line3 length", 0, out.getLength()); + in.readLine(out); + assertEquals("line4 length", 3, out.getLength()); + in.readLine(out); + assertEquals("line5 length", 4, out.getLength()); + in.readLine(out); + assertEquals("line5 length", 5, out.getLength()); + assertEquals("end of file", 0, in.readLine(out)); + } finally { + if (in != null) { + in.close(); + } + } } private static void writeFile(FileSystem fs, Path name, @@ -183,14 +198,21 @@ public class TestKeyValueTextInputFormat extends TestCase { InputSplit split, JobConf job) throws IOException { List result = new ArrayList(); - RecordReader reader = format.getRecordReader(split, job, - voidReporter); - Text key = reader.createKey(); - Text value = reader.createValue(); - while (reader.next(key, value)) { - result.add(value); - value = reader.createValue(); - } + RecordReader reader = null; + + try { + reader = format.getRecordReader(split, job, voidReporter); + Text key = reader.createKey(); + Text value = reader.createValue(); + while (reader.next(key, value)) { + result.add(value); + value = (Text) reader.createValue(); + } + } finally { + if (reader != null) { + reader.close(); + } + } return result; } From 6ecbb350193dceefa7c9155d1687f1548358c430 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 18:44:37 +0000 Subject: [PATCH 05/69] HDFS-4151. hdfs balancer command returns exit code 1 on success instead of 0. Contributed by Joshua Blatt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441113 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/balancer/Balancer.java | 12 +++-- .../hdfs/server/balancer/TestBalancer.java | 52 +++++++++++++++---- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a314d0f7e75..079fac57794 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -738,6 +738,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4428. FsDatasetImpl should disclose what the error is when a rename fails. (Colin Patrick McCabe via atm) + HDFS-4151. hdfs balancer command returns exit code 1 on success instead + of 0. (Joshua Blatt via suresh) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index b64844ba523..109ba719734 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -1333,8 +1333,9 @@ public class Balancer { // Exit status enum ReturnStatus { - SUCCESS(1), - IN_PROGRESS(0), + // These int values will map directly to the balancer process's exit code. + SUCCESS(0), + IN_PROGRESS(1), ALREADY_RUNNING(-1), NO_MOVE_BLOCK(-2), NO_MOVE_PROGRESS(-3), @@ -1507,7 +1508,12 @@ public class Balancer { } static class Cli extends Configured implements Tool { - /** Parse arguments and then run Balancer */ + /** + * Parse arguments and then run Balancer. + * + * @param args command specific arguments. + * @return exit code. 0 indicates success, non-zero indicates failure. + */ @Override public int run(String[] args) { final long startTime = Time.now(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index b8450f1a661..86dd3ab7589 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -46,8 +46,10 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Tool; import org.junit.Test; /** @@ -95,7 +97,6 @@ public class TestBalancer { DFSTestUtil.waitReplication(fs, filePath, replicationFactor); } - /* fill up a cluster with numNodes datanodes * whose used space to be size */ @@ -301,10 +302,12 @@ public class TestBalancer { * @param racks - array of racks for original nodes in cluster * @param newCapacity - new node's capacity * @param newRack - new node's rack + * @param useTool - if true run test via Cli with command-line argument + * parsing, etc. Otherwise invoke balancer API directly. * @throws Exception */ private void doTest(Configuration conf, long[] capacities, String[] racks, - long newCapacity, String newRack) throws Exception { + long newCapacity, String newRack, boolean useTool) throws Exception { assertEquals(capacities.length, racks.length); int numOfDatanodes = capacities.length; cluster = new MiniDFSCluster.Builder(conf) @@ -330,7 +333,11 @@ public class TestBalancer { totalCapacity += newCapacity; // run balancer and validate results - runBalancer(conf, totalUsedSpace, totalCapacity); + if (useTool) { + runBalancerCli(conf, totalUsedSpace, totalCapacity); + } else { + runBalancer(conf, totalUsedSpace, totalCapacity); + } } finally { cluster.shutdown(); } @@ -350,22 +357,38 @@ public class TestBalancer { waitForBalancer(totalUsedSpace, totalCapacity, client, cluster); } + private void runBalancerCli(Configuration conf, + long totalUsedSpace, long totalCapacity) throws Exception { + waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster); + + final String[] args = { "-policy", "datanode" }; + final Tool tool = new Cli(); + tool.setConf(conf); + final int r = tool.run(args); // start rebalancing + + assertEquals("Tools should exit 0 on success", 0, r); + waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster); + LOG.info("Rebalancing with default ctor."); + waitForBalancer(totalUsedSpace, totalCapacity, client, cluster); + } + /** one-node cluster test*/ - private void oneNodeTest(Configuration conf) throws Exception { + private void oneNodeTest(Configuration conf, boolean useTool) throws Exception { // add an empty node with half of the CAPACITY & the same rack - doTest(conf, new long[]{CAPACITY}, new String[]{RACK0}, CAPACITY/2, RACK0); + doTest(conf, new long[]{CAPACITY}, new String[]{RACK0}, CAPACITY/2, + RACK0, useTool); } /** two-node cluster test */ private void twoNodeTest(Configuration conf) throws Exception { doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, - CAPACITY, RACK2); + CAPACITY, RACK2, false); } /** test using a user-supplied conf */ public void integrationTest(Configuration conf) throws Exception { initConf(conf); - oneNodeTest(conf); + oneNodeTest(conf, false); } /** @@ -401,7 +424,7 @@ public class TestBalancer { void testBalancer0Internal(Configuration conf) throws Exception { initConf(conf); - oneNodeTest(conf); + oneNodeTest(conf, false); twoNodeTest(conf); } @@ -495,7 +518,18 @@ public class TestBalancer { } - + /** + * Verify balancer exits 0 on success. + */ + @Test(timeout=100000) + public void testExitZeroOnSuccess() throws Exception { + final Configuration conf = new HdfsConfiguration(); + + initConf(conf); + + oneNodeTest(conf, true); + } + /** * @param args */ From 01591b8b9f21ea773d1997e7d4b883d0a460d67a Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 18:59:14 +0000 Subject: [PATCH 06/69] Change incorrect jira number HDFS-4151 to HDFS-4451 and move it to incompatible section. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441123 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 079fac57794..7008d74ce22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -314,6 +314,10 @@ Release 2.0.3-alpha - Unreleased HDFS-4369. GetBlockKeysResponseProto does not handle null response. (suresh) + HDFS-4451. hdfs balancer command returns exit code 1 on success instead + of 0. (Joshua Blatt via suresh) + + NEW FEATURES HDFS-2656. Add libwebhdfs, a pure C client based on WebHDFS. @@ -738,9 +742,6 @@ Release 2.0.3-alpha - Unreleased HDFS-4428. FsDatasetImpl should disclose what the error is when a rename fails. (Colin Patrick McCabe via atm) - HDFS-4151. hdfs balancer command returns exit code 1 on success instead - of 0. (Joshua Blatt via suresh) - BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. From e6ef76999f0c560bab619ad002a2758e19176fc2 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 19:45:28 +0000 Subject: [PATCH 07/69] HADOOP-9264. Port change to use Java untar API on Windows from branch-1-win to trunk. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441164 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-common/pom.xml | 23 ++++++ .../java/org/apache/hadoop/fs/FileUtil.java | 76 +++++++++++++++++- .../org/apache/hadoop/fs/TestFileUtil.java | 40 +++++++++ .../java/org/apache/hadoop/fs/test-untar.tar | Bin 0 -> 20480 bytes .../java/org/apache/hadoop/fs/test-untar.tgz | Bin 0 -> 2024 bytes 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 855a77fbd6b..43a8e483345 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,6 +325,9 @@ Trunk (Unreleased) HADOOP-9249. hadoop-maven-plugins version-info goal causes build failure when running with Clover. (Chris Nauroth via suresh) + HADOOP-9264. Port change to use Java untar API on Windows from + branch-1-win to trunk. (Chris Nauroth via suresh) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index e154d4a8202..545c0cb5da2 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -241,6 +241,11 @@ test-jar test + + org.apache.commons + commons-compress + 1.4 + @@ -381,6 +386,23 @@ + + copy-test-tarballs + process-test-resources + + run + + + + + + + + + + + + pre-site @@ -485,6 +507,7 @@ src/test/all-tests src/test/resources/kdc/ldif/users.ldif src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c + src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 4593eedb9fb..19c19cd2b6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -21,9 +21,12 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; import java.util.Enumeration; +import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -624,14 +627,28 @@ public class FileUtil { * @throws IOException */ public static void unTar(File inFile, File untarDir) throws IOException { - if (!untarDir.mkdirs()) { + if (!untarDir.mkdirs()) { if (!untarDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + untarDir); } } - StringBuilder untarCommand = new StringBuilder(); boolean gzipped = inFile.toString().endsWith("gz"); + if(Shell.WINDOWS) { + // Tar is not native to Windows. Use simple Java based implementation for + // tests and simple tar archives + unTarUsingJava(inFile, untarDir, gzipped); + } + else { + // spawn tar utility to untar archive for full fledged unix behavior such + // as resolving symlinks in tar archives + unTarUsingTar(inFile, untarDir, gzipped); + } + } + + private static void unTarUsingTar(File inFile, File untarDir, + boolean gzipped) throws IOException { + StringBuffer untarCommand = new StringBuffer(); if (gzipped) { untarCommand.append(" gzip -dc '"); untarCommand.append(FileUtil.makeShellPath(inFile)); @@ -656,7 +673,62 @@ public class FileUtil { ". Tar process exited with exit code " + exitcode); } } + + private static void unTarUsingJava(File inFile, File untarDir, + boolean gzipped) throws IOException { + InputStream inputStream = null; + if (gzipped) { + inputStream = new BufferedInputStream(new GZIPInputStream( + new FileInputStream(inFile))); + } else { + inputStream = new BufferedInputStream(new FileInputStream(inFile)); + } + TarArchiveInputStream tis = new TarArchiveInputStream(inputStream); + + for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { + unpackEntries(tis, entry, untarDir); + entry = tis.getNextTarEntry(); + } + } + + private static void unpackEntries(TarArchiveInputStream tis, + TarArchiveEntry entry, File outputDir) throws IOException { + if (entry.isDirectory()) { + File subDir = new File(outputDir, entry.getName()); + if (!subDir.mkdir() && !subDir.isDirectory()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + + for (TarArchiveEntry e : entry.getDirectoryEntries()) { + unpackEntries(tis, e, subDir); + } + + return; + } + + File outputFile = new File(outputDir, entry.getName()); + if (!outputDir.exists()) { + if (!outputDir.mkdirs()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + } + + int count; + byte data[] = new byte[2048]; + BufferedOutputStream outputStream = new BufferedOutputStream( + new FileOutputStream(outputFile)); + + while ((count = tis.read(data)) != -1) { + outputStream.write(data, 0, count); + } + + outputStream.flush(); + outputStream.close(); + } + /** * Class for creating hardlinks. * Supports Unix, Cygwin, WindXP. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index a64b45d80ff..e73c644fb08 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -546,4 +546,44 @@ public class TestFileUtil { long expected = 2 * (3 + System.getProperty("line.separator").length()); Assert.assertEquals(expected, du); } + + private void doUntarAndVerify(File tarFile, File untarDir) + throws IOException { + if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { + throw new IOException("Could not delete directory '" + untarDir + "'"); + } + FileUtil.unTar(tarFile, untarDir); + + String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; + File testFile = new File(parentDir + Path.SEPARATOR + "version"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 0); + String imageDir = parentDir + Path.SEPARATOR + "image"; + testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 157); + String currentDir = parentDir + Path.SEPARATOR + "current"; + testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 4331); + testFile = new File(currentDir + Path.SEPARATOR + "edits"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 1033); + testFile = new File(currentDir + Path.SEPARATOR + "fstime"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 8); + } + + @Test + public void testUntar() throws IOException { + String tarGzFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tgz"; + String tarFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tar"; + String dataDir = System.getProperty("test.build.data", "build/test/data"); + File untarDir = new File(dataDir, "untarDir"); + + doUntarAndVerify(new File(tarGzFileName), untarDir); + doUntarAndVerify(new File(tarFileName), untarDir); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar new file mode 100644 index 0000000000000000000000000000000000000000..949e985c731aa2e4da7d1871fd1e7c114d656021 GIT binary patch literal 20480 zcmeHNe^gXe9)AP)BO+H=@&~N()U%4gzY35g)Yh}v;$W*C?!!@ObU4TwoC zT9PhmX-4L;W10stHhyU2kG9paG%~3Z*lxHUPUVkX^PCFXM6=m9@6MZf-uHNb2kPv0 z=a0E>KJVVo_ukL<-tWEdJI`S%u_j}~`ytqmUQb}YlKpU{UZ>@Ma~P>2R631DrO~P| zf+PtIf#KTW?G|A7xEYfR$1xX8GjjII%=Aikp+y|Iaa@Zt=ghfMVQ3Z?zx)tJw99`PV zV*6^fFtn?JOx6+vW1~H8lcPY=xIt$CJZgC!-(Z#H6k}6bXqU_5WRw<{<<`*~2tw^2 zNk5$mk~wHuo~iPkT4i<7q^J|T-=tjNYhkxK`fM{XXpj|NORDa*$xG%r2QLUnJm{KN$;}r zxi8uK@GaXy92pVvD^TE;{lCynm6(dGLp^Ny{huV%T6JLmSL@a6{?A5k9oqkg+7D8E zzWx8wjrafIzc&Gtp_f3Zm{>dQ@nvv3wvE?^J>k3Xe@2+8Vv{@E?QvRNMK0RoR9qR@ zi6Rtc_y3;5mSZP%YuvdTyEY3j?();JXByg{Qho;G`*j~byUu&04#r0-mwj<~F+CT? zcajrp=6611gz@IkGqpJ{HY#BJdSTd!{a3VMFdjP&zdUXI?maNRFP(|#nsNLbjPH+6 zoH_dYm;MRkkClA9I=6nsAsBbWpPBA8{m1VeY>xjv_QdPmN5{uKPQvxes@r<%@0-Ox z&;{zBem85)v9EI?;rb14w2gWD6}tz<#a?a2qJsMof7J6egJxxwG3Kp~{!ucatukEGJGPSJ6cHFL-NZ}-leUb-GU>;sIm{=(_Tsz0^eoX!0Z z;9ElMadgJFAZ`lqFGpq0zfc?o){g)uybakD^`TToE|VNm{{xT#4h3o!S?k@#M=Ky+A^y=inFZ)WQxJ@0^kzv-z)!=E1nm^-aRb zM*AKZZ;MQ;%-^;fbYH;#R^5g*vDXY>@(J)$yY{wpUHaW_nCJ9gYT6S=;cH=h%$_wV%Rxoe{IRjU%&R{Ol*kpZ|PJt^>vkGn(S=EZ_1k7;m2Z=Dqht z9M}%y*toUZYqE`7U_9x)d+h&SJvs!&UAD5XdUu#F!+69!sj=0y@ntYhJaZ<&=+Li* z@eFg;>n$n9128_XNBQ}8s?=35{#@_Ys?z9-nJ`|J*LyzhX3bO>-#oH&an*wl#KCxd zN798G%gQIgxb1wz0rQy;eh=ektvNf-_PzrGw!rUOGpbiar^c*+>!159@&(I`Sqov@ zmQlc1HrSTH_(L%jA&qZd`~!?{TN>T8ru(IRFn%EKyUvYY6^Z6x7RC4f1(ASdNt15X8zT`I(VJYVKVpAmettC#T!k7O5IR&0lTnfqg zYbhgAahZR33l-C17$28+#r}5tw6z!K!*#?)TCsjo5Td+@q+x%k9+7}uE$w*rEpSE= zxbDagvznroM1coP0sh>G!#DQUH5PFl72rjYA0B)Dy%SHt_?7auE<9|>pTWSga8OE5 zf!j*dNREz|^{QfyqNw#jFu}2S-nirh>_=A7lfSIts+)~H4;1Y6`%~WVfKoxS*3Rkw zo|j(SF)lDZ|Ao}-5vOPZ89!MY-&|XI>ecA|G)EmgI7mv_cj8Fi#0j@&dea*VK<|S# z1~!a-jwXO^Yk%y>g7F)dABO9j3!a`y#TR@J zv?pP_rMF?--K8g&b2t_a3iexSY=VQAKUEh9QQR{F7YMfoDEt?EToRUI zb}xnp1EHUN$itx6{{sqwvD;S`<=wwd6yn9BXvJi+bOQ^4e2}>L#~mwI{)flc{1Av@&3g{NJKhoYA~k`E51{lvuqWHRN4IQ!Am zvdYQe(9CPgmQr37+(wr&3PAZkhAOej)V{r2^7%jZE=fT7Keb+^NBO_oQ+u$j6o&3? zdxzfXgY5>Y|BEQVt$!}-pFC{#`(6L+zfb)8KWf64|5uTu7QO#5Yz`3+3;aJ!|Nda$ zeJ`2!zf@{1?|%~f|D)FEG${TH+%#;L>>0ls!@~1ylpFV*{BX9+$jY0Sm6ws7H8(SJ zDV~;{n`>N@H}8JjP2-Hs#NcICmz$y;?r^#gPd63Nv=eu_DB6WHG)^RuYCOK&Mp#%nV2bAy$-* z)$TIOjx0ON>@0{$g_fkDmS$ugJEnOcqd+wCqYo`hBa=FUePA9=B`4)-rBbd` z$uW|mNF|Bksy@Ciw;Xm}&*3=6u`Dn3f5irNp54?Rchs-4y7F&ya2##r`)VHaJpXc) zT#oYJAG#|4bRNUodp|}u`IigN`{n#oO1WCdzd|KU1Dx#rTk+QaJpWFov+V}q8i)@G45~Bh!M3TH!H?@G=q2v({DleFYKE@)ag= zUM7C$SeVF)OOuk4^5R#9m5E9}Zm7Hbc6u2lR&-P<)vnX=Ms3zbCr##8v97D zh>g>Aebm{d0S2Z(Zx6ORY&4h8u@0N;jDI_dKp!`Dr|>g$WSi2SwXAi;70j)}#ZT8Z zJwbd9`BwF(&n$NCu7-Rtk^9ZAne1f9*HI%XlUwf7LB1vMNLA(wbu!4$n}YUlJ);VO zJZczzdhC)78zJAE!iThu-*?RGK4ao04F2`SuOWZ9@Y4lZHFI}DZjCuI&Z++`0`f;+ zZ`(6G`Vk89;)=%hn!6?@Kz```^o4tW%nXIR_Kn6N@4RerK<=H@?O2%imcjL}?y2JS z1UW_I?bL}4g)3F3%ca-qQgTZwKI>%7dxYQDefUntgmFbnOjj_so_K-3%MmDAvYfn*=9KM(QS|)r8CzZ?Rf7PCf>e=D-3)^1Y4EeU~Us{%Z zm+zIAeYNMm9avF*9P*q$rhRh0K;~5v0$OLhGJJV74|(!w!=A>PG3AiYSf~4=AwHl5 z@`jpEDt9g~t%rQ&j$PyB2eyYnUPSM(#ZF#u4)SU5m67cao*NDM&l65m#RhKP33=kt z>UQVo>Lw4*>EvS7W~AgBVz&d=!cv9-`Ma=;7V%FP+W1nUGYWLw^B_`h%q8pc`ly@M zj9qjx8NMzyQawEvkt)>#VzIMD&Ck3Q0aLg8_lfmkv%*NopB=F4!lvrFe8}@dKi>P? z2m7CZ{7gw>D;_lKDNidMm+IIx9n?q;>si^Ivi$A$dL}p)^LK{T>z%)3;PQ)g?f2wu zO7KcLV3&&aSB0GUrFwjg8AgD@`HMD#)82-aT#jvN^jTH{?Azfr-ic z74nxu<6ix9$CSq*Z|JC9d~?x(Ii6OyWZ$AB&W_vz(BRoy% zI?=O(N$KS#ZfAJzgE>}z!uoNNDc!|M48g%~GKay$NR&VP4q$B+&dZtO-xi=dr8}w{Z;1;XI3z@st9ODKRrfGp^?ZcUB`?Xw&nI z!9oYSc9nt)Aw=iK54Y$`SqDEJFJUYe+@xnLE-$W}@bSU_^x*stBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxDg6aIL-i*B GPyhg8PZ7ib literal 0 HcmV?d00001 From b04586e7ebfad28034931739b3980756c63e54d1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 19:49:06 +0000 Subject: [PATCH 08/69] HADOOP-9264. Reverting r1441164 and wait for Jenkins results before committing the patch. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441166 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 - hadoop-common-project/hadoop-common/pom.xml | 23 ------ .../java/org/apache/hadoop/fs/FileUtil.java | 76 +----------------- .../org/apache/hadoop/fs/TestFileUtil.java | 40 --------- .../java/org/apache/hadoop/fs/test-untar.tar | Bin 20480 -> 0 bytes .../java/org/apache/hadoop/fs/test-untar.tgz | Bin 2024 -> 0 bytes 6 files changed, 2 insertions(+), 140 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 43a8e483345..855a77fbd6b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,9 +325,6 @@ Trunk (Unreleased) HADOOP-9249. hadoop-maven-plugins version-info goal causes build failure when running with Clover. (Chris Nauroth via suresh) - HADOOP-9264. Port change to use Java untar API on Windows from - branch-1-win to trunk. (Chris Nauroth via suresh) - OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 545c0cb5da2..e154d4a8202 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -241,11 +241,6 @@ test-jar test - - org.apache.commons - commons-compress - 1.4 - @@ -386,23 +381,6 @@ - - copy-test-tarballs - process-test-resources - - run - - - - - - - - - - - - pre-site @@ -507,7 +485,6 @@ src/test/all-tests src/test/resources/kdc/ldif/users.ldif src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c - src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 19c19cd2b6a..4593eedb9fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -21,12 +21,9 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; import java.util.Enumeration; -import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -627,28 +624,14 @@ public class FileUtil { * @throws IOException */ public static void unTar(File inFile, File untarDir) throws IOException { - if (!untarDir.mkdirs()) { + if (!untarDir.mkdirs()) { if (!untarDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + untarDir); } } + StringBuilder untarCommand = new StringBuilder(); boolean gzipped = inFile.toString().endsWith("gz"); - if(Shell.WINDOWS) { - // Tar is not native to Windows. Use simple Java based implementation for - // tests and simple tar archives - unTarUsingJava(inFile, untarDir, gzipped); - } - else { - // spawn tar utility to untar archive for full fledged unix behavior such - // as resolving symlinks in tar archives - unTarUsingTar(inFile, untarDir, gzipped); - } - } - - private static void unTarUsingTar(File inFile, File untarDir, - boolean gzipped) throws IOException { - StringBuffer untarCommand = new StringBuffer(); if (gzipped) { untarCommand.append(" gzip -dc '"); untarCommand.append(FileUtil.makeShellPath(inFile)); @@ -673,62 +656,7 @@ public class FileUtil { ". Tar process exited with exit code " + exitcode); } } - - private static void unTarUsingJava(File inFile, File untarDir, - boolean gzipped) throws IOException { - InputStream inputStream = null; - if (gzipped) { - inputStream = new BufferedInputStream(new GZIPInputStream( - new FileInputStream(inFile))); - } else { - inputStream = new BufferedInputStream(new FileInputStream(inFile)); - } - TarArchiveInputStream tis = new TarArchiveInputStream(inputStream); - - for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { - unpackEntries(tis, entry, untarDir); - entry = tis.getNextTarEntry(); - } - } - - private static void unpackEntries(TarArchiveInputStream tis, - TarArchiveEntry entry, File outputDir) throws IOException { - if (entry.isDirectory()) { - File subDir = new File(outputDir, entry.getName()); - if (!subDir.mkdir() && !subDir.isDirectory()) { - throw new IOException("Mkdirs failed to create tar internal dir " - + outputDir); - } - - for (TarArchiveEntry e : entry.getDirectoryEntries()) { - unpackEntries(tis, e, subDir); - } - - return; - } - - File outputFile = new File(outputDir, entry.getName()); - if (!outputDir.exists()) { - if (!outputDir.mkdirs()) { - throw new IOException("Mkdirs failed to create tar internal dir " - + outputDir); - } - } - - int count; - byte data[] = new byte[2048]; - BufferedOutputStream outputStream = new BufferedOutputStream( - new FileOutputStream(outputFile)); - - while ((count = tis.read(data)) != -1) { - outputStream.write(data, 0, count); - } - - outputStream.flush(); - outputStream.close(); - } - /** * Class for creating hardlinks. * Supports Unix, Cygwin, WindXP. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index e73c644fb08..a64b45d80ff 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -546,44 +546,4 @@ public class TestFileUtil { long expected = 2 * (3 + System.getProperty("line.separator").length()); Assert.assertEquals(expected, du); } - - private void doUntarAndVerify(File tarFile, File untarDir) - throws IOException { - if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { - throw new IOException("Could not delete directory '" + untarDir + "'"); - } - FileUtil.unTar(tarFile, untarDir); - - String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; - File testFile = new File(parentDir + Path.SEPARATOR + "version"); - Assert.assertTrue(testFile.exists()); - Assert.assertTrue(testFile.length() == 0); - String imageDir = parentDir + Path.SEPARATOR + "image"; - testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); - Assert.assertTrue(testFile.exists()); - Assert.assertTrue(testFile.length() == 157); - String currentDir = parentDir + Path.SEPARATOR + "current"; - testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); - Assert.assertTrue(testFile.exists()); - Assert.assertTrue(testFile.length() == 4331); - testFile = new File(currentDir + Path.SEPARATOR + "edits"); - Assert.assertTrue(testFile.exists()); - Assert.assertTrue(testFile.length() == 1033); - testFile = new File(currentDir + Path.SEPARATOR + "fstime"); - Assert.assertTrue(testFile.exists()); - Assert.assertTrue(testFile.length() == 8); - } - - @Test - public void testUntar() throws IOException { - String tarGzFileName = System.getProperty("test.cache.data", - "build/test/cache") + "/test-untar.tgz"; - String tarFileName = System.getProperty("test.cache.data", - "build/test/cache") + "/test-untar.tar"; - String dataDir = System.getProperty("test.build.data", "build/test/data"); - File untarDir = new File(dataDir, "untarDir"); - - doUntarAndVerify(new File(tarGzFileName), untarDir); - doUntarAndVerify(new File(tarFileName), untarDir); - } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar deleted file mode 100644 index 949e985c731aa2e4da7d1871fd1e7c114d656021..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeHNe^gXe9)AP)BO+H=@&~N()U%4gzY35g)Yh}v;$W*C?!!@ObU4TwoC zT9PhmX-4L;W10stHhyU2kG9paG%~3Z*lxHUPUVkX^PCFXM6=m9@6MZf-uHNb2kPv0 z=a0E>KJVVo_ukL<-tWEdJI`S%u_j}~`ytqmUQb}YlKpU{UZ>@Ma~P>2R631DrO~P| zf+PtIf#KTW?G|A7xEYfR$1xX8GjjII%=Aikp+y|Iaa@Zt=ghfMVQ3Z?zx)tJw99`PV zV*6^fFtn?JOx6+vW1~H8lcPY=xIt$CJZgC!-(Z#H6k}6bXqU_5WRw<{<<`*~2tw^2 zNk5$mk~wHuo~iPkT4i<7q^J|T-=tjNYhkxK`fM{XXpj|NORDa*$xG%r2QLUnJm{KN$;}r zxi8uK@GaXy92pVvD^TE;{lCynm6(dGLp^Ny{huV%T6JLmSL@a6{?A5k9oqkg+7D8E zzWx8wjrafIzc&Gtp_f3Zm{>dQ@nvv3wvE?^J>k3Xe@2+8Vv{@E?QvRNMK0RoR9qR@ zi6Rtc_y3;5mSZP%YuvdTyEY3j?();JXByg{Qho;G`*j~byUu&04#r0-mwj<~F+CT? zcajrp=6611gz@IkGqpJ{HY#BJdSTd!{a3VMFdjP&zdUXI?maNRFP(|#nsNLbjPH+6 zoH_dYm;MRkkClA9I=6nsAsBbWpPBA8{m1VeY>xjv_QdPmN5{uKPQvxes@r<%@0-Ox z&;{zBem85)v9EI?;rb14w2gWD6}tz<#a?a2qJsMof7J6egJxxwG3Kp~{!ucatukEGJGPSJ6cHFL-NZ}-leUb-GU>;sIm{=(_Tsz0^eoX!0Z z;9ElMadgJFAZ`lqFGpq0zfc?o){g)uybakD^`TToE|VNm{{xT#4h3o!S?k@#M=Ky+A^y=inFZ)WQxJ@0^kzv-z)!=E1nm^-aRb zM*AKZZ;MQ;%-^;fbYH;#R^5g*vDXY>@(J)$yY{wpUHaW_nCJ9gYT6S=;cH=h%$_wV%Rxoe{IRjU%&R{Ol*kpZ|PJt^>vkGn(S=EZ_1k7;m2Z=Dqht z9M}%y*toUZYqE`7U_9x)d+h&SJvs!&UAD5XdUu#F!+69!sj=0y@ntYhJaZ<&=+Li* z@eFg;>n$n9128_XNBQ}8s?=35{#@_Ys?z9-nJ`|J*LyzhX3bO>-#oH&an*wl#KCxd zN798G%gQIgxb1wz0rQy;eh=ektvNf-_PzrGw!rUOGpbiar^c*+>!159@&(I`Sqov@ zmQlc1HrSTH_(L%jA&qZd`~!?{TN>T8ru(IRFn%EKyUvYY6^Z6x7RC4f1(ASdNt15X8zT`I(VJYVKVpAmettC#T!k7O5IR&0lTnfqg zYbhgAahZR33l-C17$28+#r}5tw6z!K!*#?)TCsjo5Td+@q+x%k9+7}uE$w*rEpSE= zxbDagvznroM1coP0sh>G!#DQUH5PFl72rjYA0B)Dy%SHt_?7auE<9|>pTWSga8OE5 zf!j*dNREz|^{QfyqNw#jFu}2S-nirh>_=A7lfSIts+)~H4;1Y6`%~WVfKoxS*3Rkw zo|j(SF)lDZ|Ao}-5vOPZ89!MY-&|XI>ecA|G)EmgI7mv_cj8Fi#0j@&dea*VK<|S# z1~!a-jwXO^Yk%y>g7F)dABO9j3!a`y#TR@J zv?pP_rMF?--K8g&b2t_a3iexSY=VQAKUEh9QQR{F7YMfoDEt?EToRUI zb}xnp1EHUN$itx6{{sqwvD;S`<=wwd6yn9BXvJi+bOQ^4e2}>L#~mwI{)flc{1Av@&3g{NJKhoYA~k`E51{lvuqWHRN4IQ!Am zvdYQe(9CPgmQr37+(wr&3PAZkhAOej)V{r2^7%jZE=fT7Keb+^NBO_oQ+u$j6o&3? zdxzfXgY5>Y|BEQVt$!}-pFC{#`(6L+zfb)8KWf64|5uTu7QO#5Yz`3+3;aJ!|Nda$ zeJ`2!zf@{1?|%~f|D)FEG${TH+%#;L>>0ls!@~1ylpFV*{BX9+$jY0Sm6ws7H8(SJ zDV~;{n`>N@H}8JjP2-Hs#NcICmz$y;?r^#gPd63Nv=eu_DB6WHG)^RuYCOK&Mp#%nV2bAy$-* z)$TIOjx0ON>@0{$g_fkDmS$ugJEnOcqd+wCqYo`hBa=FUePA9=B`4)-rBbd` z$uW|mNF|Bksy@Ciw;Xm}&*3=6u`Dn3f5irNp54?Rchs-4y7F&ya2##r`)VHaJpXc) zT#oYJAG#|4bRNUodp|}u`IigN`{n#oO1WCdzd|KU1Dx#rTk+QaJpWFov+V}q8i)@G45~Bh!M3TH!H?@G=q2v({DleFYKE@)ag= zUM7C$SeVF)OOuk4^5R#9m5E9}Zm7Hbc6u2lR&-P<)vnX=Ms3zbCr##8v97D zh>g>Aebm{d0S2Z(Zx6ORY&4h8u@0N;jDI_dKp!`Dr|>g$WSi2SwXAi;70j)}#ZT8Z zJwbd9`BwF(&n$NCu7-Rtk^9ZAne1f9*HI%XlUwf7LB1vMNLA(wbu!4$n}YUlJ);VO zJZczzdhC)78zJAE!iThu-*?RGK4ao04F2`SuOWZ9@Y4lZHFI}DZjCuI&Z++`0`f;+ zZ`(6G`Vk89;)=%hn!6?@Kz```^o4tW%nXIR_Kn6N@4RerK<=H@?O2%imcjL}?y2JS z1UW_I?bL}4g)3F3%ca-qQgTZwKI>%7dxYQDefUntgmFbnOjj_so_K-3%MmDAvYfn*=9KM(QS|)r8CzZ?Rf7PCf>e=D-3)^1Y4EeU~Us{%Z zm+zIAeYNMm9avF*9P*q$rhRh0K;~5v0$OLhGJJV74|(!w!=A>PG3AiYSf~4=AwHl5 z@`jpEDt9g~t%rQ&j$PyB2eyYnUPSM(#ZF#u4)SU5m67cao*NDM&l65m#RhKP33=kt z>UQVo>Lw4*>EvS7W~AgBVz&d=!cv9-`Ma=;7V%FP+W1nUGYWLw^B_`h%q8pc`ly@M zj9qjx8NMzyQawEvkt)>#VzIMD&Ck3Q0aLg8_lfmkv%*NopB=F4!lvrFe8}@dKi>P? z2m7CZ{7gw>D;_lKDNidMm+IIx9n?q;>si^Ivi$A$dL}p)^LK{T>z%)3;PQ)g?f2wu zO7KcLV3&&aSB0GUrFwjg8AgD@`HMD#)82-aT#jvN^jTH{?Azfr-ic z74nxu<6ix9$CSq*Z|JC9d~?x(Ii6OyWZ$AB&W_vz(BRoy% zI?=O(N$KS#ZfAJzgE>}z!uoNNDc!|M48g%~GKay$NR&VP4q$B+&dZtO-xi=dr8}w{Z;1;XI3z@st9ODKRrfGp^?ZcUB`?Xw&nI z!9oYSc9nt)Aw=iK54Y$`SqDEJFJUYe+@xnLE-$W}@bSU_^x*stBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxDg6aIL-i*B GPyhg8PZ7ib From 71a57ded39a605166d616fe68f36017cdb0abe3e Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 20:13:01 +0000 Subject: [PATCH 09/69] HADOOP-9264. Port change to use Java untar API on Windows from branch-1-win to trunk. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441172 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-common/pom.xml | 23 ++++++ .../java/org/apache/hadoop/fs/FileUtil.java | 76 +++++++++++++++++- .../org/apache/hadoop/fs/TestFileUtil.java | 40 +++++++++ .../java/org/apache/hadoop/fs/test-untar.tar | Bin 0 -> 20480 bytes .../java/org/apache/hadoop/fs/test-untar.tgz | Bin 0 -> 2024 bytes 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 855a77fbd6b..43a8e483345 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,6 +325,9 @@ Trunk (Unreleased) HADOOP-9249. hadoop-maven-plugins version-info goal causes build failure when running with Clover. (Chris Nauroth via suresh) + HADOOP-9264. Port change to use Java untar API on Windows from + branch-1-win to trunk. (Chris Nauroth via suresh) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index e154d4a8202..545c0cb5da2 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -241,6 +241,11 @@ test-jar test + + org.apache.commons + commons-compress + 1.4 + @@ -381,6 +386,23 @@ + + copy-test-tarballs + process-test-resources + + run + + + + + + + + + + + + pre-site @@ -485,6 +507,7 @@ src/test/all-tests src/test/resources/kdc/ldif/users.ldif src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c + src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 4593eedb9fb..19c19cd2b6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -21,9 +21,12 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; import java.util.Enumeration; +import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -624,14 +627,28 @@ public class FileUtil { * @throws IOException */ public static void unTar(File inFile, File untarDir) throws IOException { - if (!untarDir.mkdirs()) { + if (!untarDir.mkdirs()) { if (!untarDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + untarDir); } } - StringBuilder untarCommand = new StringBuilder(); boolean gzipped = inFile.toString().endsWith("gz"); + if(Shell.WINDOWS) { + // Tar is not native to Windows. Use simple Java based implementation for + // tests and simple tar archives + unTarUsingJava(inFile, untarDir, gzipped); + } + else { + // spawn tar utility to untar archive for full fledged unix behavior such + // as resolving symlinks in tar archives + unTarUsingTar(inFile, untarDir, gzipped); + } + } + + private static void unTarUsingTar(File inFile, File untarDir, + boolean gzipped) throws IOException { + StringBuffer untarCommand = new StringBuffer(); if (gzipped) { untarCommand.append(" gzip -dc '"); untarCommand.append(FileUtil.makeShellPath(inFile)); @@ -656,7 +673,62 @@ public class FileUtil { ". Tar process exited with exit code " + exitcode); } } + + private static void unTarUsingJava(File inFile, File untarDir, + boolean gzipped) throws IOException { + InputStream inputStream = null; + if (gzipped) { + inputStream = new BufferedInputStream(new GZIPInputStream( + new FileInputStream(inFile))); + } else { + inputStream = new BufferedInputStream(new FileInputStream(inFile)); + } + TarArchiveInputStream tis = new TarArchiveInputStream(inputStream); + + for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { + unpackEntries(tis, entry, untarDir); + entry = tis.getNextTarEntry(); + } + } + + private static void unpackEntries(TarArchiveInputStream tis, + TarArchiveEntry entry, File outputDir) throws IOException { + if (entry.isDirectory()) { + File subDir = new File(outputDir, entry.getName()); + if (!subDir.mkdir() && !subDir.isDirectory()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + + for (TarArchiveEntry e : entry.getDirectoryEntries()) { + unpackEntries(tis, e, subDir); + } + + return; + } + + File outputFile = new File(outputDir, entry.getName()); + if (!outputDir.exists()) { + if (!outputDir.mkdirs()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + } + + int count; + byte data[] = new byte[2048]; + BufferedOutputStream outputStream = new BufferedOutputStream( + new FileOutputStream(outputFile)); + + while ((count = tis.read(data)) != -1) { + outputStream.write(data, 0, count); + } + + outputStream.flush(); + outputStream.close(); + } + /** * Class for creating hardlinks. * Supports Unix, Cygwin, WindXP. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index a64b45d80ff..e73c644fb08 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -546,4 +546,44 @@ public class TestFileUtil { long expected = 2 * (3 + System.getProperty("line.separator").length()); Assert.assertEquals(expected, du); } + + private void doUntarAndVerify(File tarFile, File untarDir) + throws IOException { + if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { + throw new IOException("Could not delete directory '" + untarDir + "'"); + } + FileUtil.unTar(tarFile, untarDir); + + String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; + File testFile = new File(parentDir + Path.SEPARATOR + "version"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 0); + String imageDir = parentDir + Path.SEPARATOR + "image"; + testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 157); + String currentDir = parentDir + Path.SEPARATOR + "current"; + testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 4331); + testFile = new File(currentDir + Path.SEPARATOR + "edits"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 1033); + testFile = new File(currentDir + Path.SEPARATOR + "fstime"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 8); + } + + @Test + public void testUntar() throws IOException { + String tarGzFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tgz"; + String tarFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tar"; + String dataDir = System.getProperty("test.build.data", "build/test/data"); + File untarDir = new File(dataDir, "untarDir"); + + doUntarAndVerify(new File(tarGzFileName), untarDir); + doUntarAndVerify(new File(tarFileName), untarDir); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar new file mode 100644 index 0000000000000000000000000000000000000000..949e985c731aa2e4da7d1871fd1e7c114d656021 GIT binary patch literal 20480 zcmeHNe^gXe9)AP)BO+H=@&~N()U%4gzY35g)Yh}v;$W*C?!!@ObU4TwoC zT9PhmX-4L;W10stHhyU2kG9paG%~3Z*lxHUPUVkX^PCFXM6=m9@6MZf-uHNb2kPv0 z=a0E>KJVVo_ukL<-tWEdJI`S%u_j}~`ytqmUQb}YlKpU{UZ>@Ma~P>2R631DrO~P| zf+PtIf#KTW?G|A7xEYfR$1xX8GjjII%=Aikp+y|Iaa@Zt=ghfMVQ3Z?zx)tJw99`PV zV*6^fFtn?JOx6+vW1~H8lcPY=xIt$CJZgC!-(Z#H6k}6bXqU_5WRw<{<<`*~2tw^2 zNk5$mk~wHuo~iPkT4i<7q^J|T-=tjNYhkxK`fM{XXpj|NORDa*$xG%r2QLUnJm{KN$;}r zxi8uK@GaXy92pVvD^TE;{lCynm6(dGLp^Ny{huV%T6JLmSL@a6{?A5k9oqkg+7D8E zzWx8wjrafIzc&Gtp_f3Zm{>dQ@nvv3wvE?^J>k3Xe@2+8Vv{@E?QvRNMK0RoR9qR@ zi6Rtc_y3;5mSZP%YuvdTyEY3j?();JXByg{Qho;G`*j~byUu&04#r0-mwj<~F+CT? zcajrp=6611gz@IkGqpJ{HY#BJdSTd!{a3VMFdjP&zdUXI?maNRFP(|#nsNLbjPH+6 zoH_dYm;MRkkClA9I=6nsAsBbWpPBA8{m1VeY>xjv_QdPmN5{uKPQvxes@r<%@0-Ox z&;{zBem85)v9EI?;rb14w2gWD6}tz<#a?a2qJsMof7J6egJxxwG3Kp~{!ucatukEGJGPSJ6cHFL-NZ}-leUb-GU>;sIm{=(_Tsz0^eoX!0Z z;9ElMadgJFAZ`lqFGpq0zfc?o){g)uybakD^`TToE|VNm{{xT#4h3o!S?k@#M=Ky+A^y=inFZ)WQxJ@0^kzv-z)!=E1nm^-aRb zM*AKZZ;MQ;%-^;fbYH;#R^5g*vDXY>@(J)$yY{wpUHaW_nCJ9gYT6S=;cH=h%$_wV%Rxoe{IRjU%&R{Ol*kpZ|PJt^>vkGn(S=EZ_1k7;m2Z=Dqht z9M}%y*toUZYqE`7U_9x)d+h&SJvs!&UAD5XdUu#F!+69!sj=0y@ntYhJaZ<&=+Li* z@eFg;>n$n9128_XNBQ}8s?=35{#@_Ys?z9-nJ`|J*LyzhX3bO>-#oH&an*wl#KCxd zN798G%gQIgxb1wz0rQy;eh=ektvNf-_PzrGw!rUOGpbiar^c*+>!159@&(I`Sqov@ zmQlc1HrSTH_(L%jA&qZd`~!?{TN>T8ru(IRFn%EKyUvYY6^Z6x7RC4f1(ASdNt15X8zT`I(VJYVKVpAmettC#T!k7O5IR&0lTnfqg zYbhgAahZR33l-C17$28+#r}5tw6z!K!*#?)TCsjo5Td+@q+x%k9+7}uE$w*rEpSE= zxbDagvznroM1coP0sh>G!#DQUH5PFl72rjYA0B)Dy%SHt_?7auE<9|>pTWSga8OE5 zf!j*dNREz|^{QfyqNw#jFu}2S-nirh>_=A7lfSIts+)~H4;1Y6`%~WVfKoxS*3Rkw zo|j(SF)lDZ|Ao}-5vOPZ89!MY-&|XI>ecA|G)EmgI7mv_cj8Fi#0j@&dea*VK<|S# z1~!a-jwXO^Yk%y>g7F)dABO9j3!a`y#TR@J zv?pP_rMF?--K8g&b2t_a3iexSY=VQAKUEh9QQR{F7YMfoDEt?EToRUI zb}xnp1EHUN$itx6{{sqwvD;S`<=wwd6yn9BXvJi+bOQ^4e2}>L#~mwI{)flc{1Av@&3g{NJKhoYA~k`E51{lvuqWHRN4IQ!Am zvdYQe(9CPgmQr37+(wr&3PAZkhAOej)V{r2^7%jZE=fT7Keb+^NBO_oQ+u$j6o&3? zdxzfXgY5>Y|BEQVt$!}-pFC{#`(6L+zfb)8KWf64|5uTu7QO#5Yz`3+3;aJ!|Nda$ zeJ`2!zf@{1?|%~f|D)FEG${TH+%#;L>>0ls!@~1ylpFV*{BX9+$jY0Sm6ws7H8(SJ zDV~;{n`>N@H}8JjP2-Hs#NcICmz$y;?r^#gPd63Nv=eu_DB6WHG)^RuYCOK&Mp#%nV2bAy$-* z)$TIOjx0ON>@0{$g_fkDmS$ugJEnOcqd+wCqYo`hBa=FUePA9=B`4)-rBbd` z$uW|mNF|Bksy@Ciw;Xm}&*3=6u`Dn3f5irNp54?Rchs-4y7F&ya2##r`)VHaJpXc) zT#oYJAG#|4bRNUodp|}u`IigN`{n#oO1WCdzd|KU1Dx#rTk+QaJpWFov+V}q8i)@G45~Bh!M3TH!H?@G=q2v({DleFYKE@)ag= zUM7C$SeVF)OOuk4^5R#9m5E9}Zm7Hbc6u2lR&-P<)vnX=Ms3zbCr##8v97D zh>g>Aebm{d0S2Z(Zx6ORY&4h8u@0N;jDI_dKp!`Dr|>g$WSi2SwXAi;70j)}#ZT8Z zJwbd9`BwF(&n$NCu7-Rtk^9ZAne1f9*HI%XlUwf7LB1vMNLA(wbu!4$n}YUlJ);VO zJZczzdhC)78zJAE!iThu-*?RGK4ao04F2`SuOWZ9@Y4lZHFI}DZjCuI&Z++`0`f;+ zZ`(6G`Vk89;)=%hn!6?@Kz```^o4tW%nXIR_Kn6N@4RerK<=H@?O2%imcjL}?y2JS z1UW_I?bL}4g)3F3%ca-qQgTZwKI>%7dxYQDefUntgmFbnOjj_so_K-3%MmDAvYfn*=9KM(QS|)r8CzZ?Rf7PCf>e=D-3)^1Y4EeU~Us{%Z zm+zIAeYNMm9avF*9P*q$rhRh0K;~5v0$OLhGJJV74|(!w!=A>PG3AiYSf~4=AwHl5 z@`jpEDt9g~t%rQ&j$PyB2eyYnUPSM(#ZF#u4)SU5m67cao*NDM&l65m#RhKP33=kt z>UQVo>Lw4*>EvS7W~AgBVz&d=!cv9-`Ma=;7V%FP+W1nUGYWLw^B_`h%q8pc`ly@M zj9qjx8NMzyQawEvkt)>#VzIMD&Ck3Q0aLg8_lfmkv%*NopB=F4!lvrFe8}@dKi>P? z2m7CZ{7gw>D;_lKDNidMm+IIx9n?q;>si^Ivi$A$dL}p)^LK{T>z%)3;PQ)g?f2wu zO7KcLV3&&aSB0GUrFwjg8AgD@`HMD#)82-aT#jvN^jTH{?Azfr-ic z74nxu<6ix9$CSq*Z|JC9d~?x(Ii6OyWZ$AB&W_vz(BRoy% zI?=O(N$KS#ZfAJzgE>}z!uoNNDc!|M48g%~GKay$NR&VP4q$B+&dZtO-xi=dr8}w{Z;1;XI3z@st9ODKRrfGp^?ZcUB`?Xw&nI z!9oYSc9nt)Aw=iK54Y$`SqDEJFJUYe+@xnLE-$W}@bSU_^x*stBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxDg6aIL-i*B GPyhg8PZ7ib literal 0 HcmV?d00001 From f81119816416064f915860ee9db0a6d1893d73b0 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 31 Jan 2013 22:46:05 +0000 Subject: [PATCH 10/69] YARN-364. AggregatedLogDeletionService can take too long to delete logs. Contributed by Jason Lowe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441239 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/site/apt/ClusterSetup.apt.vm | 7 ++ hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 9 +++ .../AggregatedLogDeletionService.java | 9 ++- .../src/main/resources/yarn-default.xml | 9 +++ .../TestAggregatedLogDeletionService.java | 71 +++++++++++++++++++ 6 files changed, 107 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm index 5f28d7c3072..da69b303b37 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm @@ -325,6 +325,13 @@ Hadoop MapReduce Next Generation - Cluster Setup | | | How long to keep aggregation logs before deleting them. -1 disables. | | | | Be careful, set this too small and you will spam the name node. | *-------------------------+-------------------------+------------------------+ +| <<>> | | | +| | <-1> | | +| | | Time between checks for aggregated log retention. If set to 0 or a | +| | | negative value then the value is computed as one-tenth of the | +| | | aggregated log retention time. | +| | | Be careful, set this too small and you will spam the name node. | +*-------------------------+-------------------------+------------------------+ diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1030fa6f33e..0813c172794 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -284,6 +284,9 @@ Release 0.23.7 - UNRELEASED YARN-343. Capacity Scheduler maximum-capacity value -1 is invalid (Xuan Gong via tgraves) + YARN-364. AggregatedLogDeletionService can take too long to delete logs + (jlowe) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 81c1fe933c2..5c22a7d2a4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -379,6 +379,15 @@ public class YarnConfiguration extends Configuration { + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + /** + * How long to wait between aggregated log retention checks. If set to + * a value <= 0 then the value is computed as one-tenth of the log retention + * setting. Be careful set this too small and you will spam the name node. + */ + public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = + YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; + public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; + /** * Number of seconds to retain logs on the NodeManager. Only applicable if Log * aggregation is disabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java index 9fbcae9989d..c8603ab7c10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java @@ -140,9 +140,16 @@ public class AggregatedLogDeletionService extends AbstractService { " too small (" + retentionSecs + ")"); return; } + long checkIntervalMsecs = 1000 * conf.getLong( + YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS); + if (checkIntervalMsecs <= 0) { + // when unspecified compute check interval as 1/10th of retention + checkIntervalMsecs = (retentionSecs * 1000) / 10; + } TimerTask task = new LogDeletionTask(conf, retentionSecs); timer = new Timer(); - timer.scheduleAtFixedRate(task, 0, retentionSecs * 1000); + timer.scheduleAtFixedRate(task, 0, checkIntervalMsecs); super.start(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e28ac43e855..588bb1bccfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -410,6 +410,15 @@ -1 + + How long to wait between aggregated log retention checks. + If set to 0 or a negative value then the value is computed as one-tenth + of the aggregated log retention time. Be careful set this too small and + you will spam the name node. + yarn.log-aggregation.retain-check-interval-seconds + -1 + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java index c1cf9af3602..035cd9515c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java @@ -28,12 +28,19 @@ import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Before; import org.junit.Test; import static org.mockito.Mockito.*; public class TestAggregatedLogDeletionService { + @Before + public void closeFilesystems() throws IOException { + // prevent the same mockfs instance from being reused due to FS cache + FileSystem.closeAll(); + } + @Test public void testDeletion() throws Exception { long now = System.currentTimeMillis(); @@ -121,6 +128,70 @@ public class TestAggregatedLogDeletionService { verify(mockFs).delete(app4Dir, true); } + @Test + public void testCheckInterval() throws Exception { + long RETENTION_SECS = 10 * 24 * 3600; + long now = System.currentTimeMillis(); + long toDeleteTime = now - RETENTION_SECS*1000; + + String root = "mockfs://foo/"; + String remoteRootLogDir = root+"tmp/logs"; + String suffix = "logs"; + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true"); + conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "864000"); + conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, "1"); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + + // prevent us from picking up the same mockfs instance from another test + FileSystem.closeAll(); + Path rootPath = new Path(root); + FileSystem rootFs = rootPath.getFileSystem(conf); + FileSystem mockFs = ((FilterFileSystem)rootFs).getRawFileSystem(); + + Path remoteRootLogPath = new Path(remoteRootLogDir); + + Path userDir = new Path(remoteRootLogPath, "me"); + FileStatus userDirStatus = new FileStatus(0, true, 0, 0, now, userDir); + + when(mockFs.listStatus(remoteRootLogPath)).thenReturn( + new FileStatus[]{userDirStatus}); + + Path userLogDir = new Path(userDir, suffix); + Path app1Dir = new Path(userLogDir, "application_1_1"); + FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, now, app1Dir); + + when(mockFs.listStatus(userLogDir)).thenReturn( + new FileStatus[]{app1DirStatus}); + + Path app1Log1 = new Path(app1Dir, "host1"); + FileStatus app1Log1Status = new FileStatus(10, false, 1, 1, now, app1Log1); + + when(mockFs.listStatus(app1Dir)).thenReturn( + new FileStatus[]{app1Log1Status}); + + AggregatedLogDeletionService deletionSvc = + new AggregatedLogDeletionService(); + deletionSvc.init(conf); + deletionSvc.start(); + + verify(mockFs, timeout(10000).atLeast(4)).listStatus(any(Path.class)); + verify(mockFs, never()).delete(app1Dir, true); + + // modify the timestamp of the logs and verify it's picked up quickly + app1DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app1Dir); + app1Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app1Log1); + when(mockFs.listStatus(userLogDir)).thenReturn( + new FileStatus[]{app1DirStatus}); + when(mockFs.listStatus(app1Dir)).thenReturn( + new FileStatus[]{app1Log1Status}); + + verify(mockFs, timeout(10000)).delete(app1Dir, true); + + deletionSvc.stop(); + } static class MockFileSystem extends FilterFileSystem { MockFileSystem() { From af8553b9feb9fca2f9624512d046d383d45a2584 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Fri, 1 Feb 2013 15:03:35 +0000 Subject: [PATCH 11/69] HADOOP-9124. SortedMapWritable violates contract of Map interface for equals() and hashCode(). Contributed by Surenkumar Nihalani git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441475 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/io/SortedMapWritable.java | 23 ++++++ .../hadoop/io/TestSortedMapWritable.java | 71 ++++++++++++++++++- 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 43a8e483345..b1cc950a747 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -592,6 +592,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9221. Convert remaining xdocs to APT. (Andy Isaacson via atm) HADOOP-8981. TestMetricsSystemImpl fails on Windows. (Xuan Gong via suresh) + + HADOOP-9124. SortedMapWritable violates contract of Map interface for + equals() and hashCode(). (Surenkumar Nihalani via tomwhite) Release 2.0.2-alpha - 2012-09-07 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java index eee744ec6a2..c80af15c9e2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java @@ -203,4 +203,27 @@ public class SortedMapWritable extends AbstractMapWritable e.getValue().write(out); } } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof SortedMapWritable) { + Map map = (Map) obj; + if (size() != map.size()) { + return false; + } + + return entrySet().equals(map.entrySet()); + } + + return false; + } + + @Override + public int hashCode() { + return instance.hashCode(); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java index 927bfc1f42d..1fbfcad7627 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java @@ -17,15 +17,20 @@ */ package org.apache.hadoop.io; -import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; -import junit.framework.TestCase; +import java.util.Map; +import org.junit.Test; /** * Tests SortedMapWritable */ -public class TestSortedMapWritable extends TestCase { +public class TestSortedMapWritable { /** the test */ + @Test @SuppressWarnings("unchecked") public void testSortedMapWritable() { Text[] keys = { @@ -90,6 +95,7 @@ public class TestSortedMapWritable extends TestCase { /** * Test that number of "unknown" classes is propagated across multiple copies. */ + @Test @SuppressWarnings("deprecation") public void testForeignClass() { SortedMapWritable inMap = new SortedMapWritable(); @@ -99,4 +105,63 @@ public class TestSortedMapWritable extends TestCase { SortedMapWritable copyOfCopy = new SortedMapWritable(outMap); assertEquals(1, copyOfCopy.getNewClasses()); } + + /** + * Tests if equal and hashCode method still hold the contract. + */ + @Test + public void testEqualsAndHashCode() { + String failureReason; + SortedMapWritable mapA = new SortedMapWritable(); + SortedMapWritable mapB = new SortedMapWritable(); + + // Sanity checks + failureReason = "SortedMapWritable couldn't be initialized. Got null reference"; + assertNotNull(failureReason, mapA); + assertNotNull(failureReason, mapB); + + // Basic null check + assertFalse("equals method returns true when passed null", mapA.equals(null)); + + // When entry set is empty, they should be equal + assertTrue("Two empty SortedMapWritables are no longer equal", mapA.equals(mapB)); + + // Setup + Text[] keys = { + new Text("key1"), + new Text("key2") + }; + + BytesWritable[] values = { + new BytesWritable("value1".getBytes()), + new BytesWritable("value2".getBytes()) + }; + + mapA.put(keys[0], values[0]); + mapB.put(keys[1], values[1]); + + // entrySets are different + failureReason = "Two SortedMapWritables with different data are now equal"; + assertTrue(failureReason, mapA.hashCode() != mapB.hashCode()); + assertTrue(failureReason, !mapA.equals(mapB)); + assertTrue(failureReason, !mapB.equals(mapA)); + + mapA.put(keys[1], values[1]); + mapB.put(keys[0], values[0]); + + // entrySets are now same + failureReason = "Two SortedMapWritables with same entry sets formed in different order are now different"; + assertEquals(failureReason, mapA.hashCode(), mapB.hashCode()); + assertTrue(failureReason, mapA.equals(mapB)); + assertTrue(failureReason, mapB.equals(mapA)); + + // Let's check if entry sets of same keys but different values + mapA.put(keys[0], values[1]); + mapA.put(keys[1], values[0]); + + failureReason = "Two SortedMapWritables with different content are now equal"; + assertTrue(failureReason, mapA.hashCode() != mapB.hashCode()); + assertTrue(failureReason, !mapA.equals(mapB)); + assertTrue(failureReason, !mapB.equals(mapA)); + } } From bbdae834d2ec26b329b48b4c9343ebb182a63242 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 1 Feb 2013 19:42:06 +0000 Subject: [PATCH 12/69] HDFS-4456. Add concat to HttpFS and WebHDFS REST API docs. (plamenj2003 via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441603 13f79535-47bb-0310-9956-ffa450edef68 --- .../fs/http/client/HttpFSFileSystem.java | 28 +++++++++- .../hadoop/fs/http/server/FSOperations.java | 41 +++++++++++++++ .../http/server/HttpFSParametersProvider.java | 20 +++++++ .../hadoop/fs/http/server/HttpFSServer.java | 21 ++++++-- .../fs/http/client/BaseTestHttpFSWith.java | 30 ++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../web/resources/ConcatSourcesParam.java | 2 +- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 52 +++++++++++++++++++ 8 files changed, 189 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index c64cfd0879c..345b58a462b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.fs.http.client; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; @@ -86,6 +88,7 @@ public class HttpFSFileSystem extends FileSystem public static final String PERMISSION_PARAM = "permission"; public static final String DESTINATION_PARAM = "destination"; public static final String RECURSIVE_PARAM = "recursive"; + public static final String SOURCES_PARAM = "sources"; public static final String OWNER_PARAM = "owner"; public static final String GROUP_PARAM = "group"; public static final String MODIFICATION_TIME_PARAM = "modificationtime"; @@ -167,7 +170,7 @@ public class HttpFSFileSystem extends FileSystem GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), INSTRUMENTATION(HTTP_GET), - APPEND(HTTP_POST), + APPEND(HTTP_POST), CONCAT(HTTP_POST), CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), DELETE(HTTP_DELETE); @@ -528,6 +531,29 @@ public class HttpFSFileSystem extends FileSystem HttpURLConnection.HTTP_OK); } + /** + * Concat existing files together. + * @param f the path to the target destination. + * @param psrcs the paths to the sources to use for the concatenation. + * + * @throws IOException + */ + @Override + public void concat(Path f, Path[] psrcs) throws IOException { + List strPaths = new ArrayList(psrcs.length); + for(Path psrc : psrcs) { + strPaths.add(psrc.toUri().getPath()); + } + String srcs = StringUtils.join(",", strPaths); + + Map params = new HashMap(); + params.put(OP_PARAM, Operation.CONCAT.toString()); + params.put(SOURCES_PARAM, srcs); + HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(), + params, f, true); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + } + /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index f81e90e0642..8e41d04e443 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -198,6 +198,47 @@ public class FSOperations { } + /** + * Executor that performs an append FileSystemAccess files system operation. + */ + @InterfaceAudience.Private + public static class FSConcat implements FileSystemAccess.FileSystemExecutor { + private Path path; + private Path[] sources; + + /** + * Creates a Concat executor. + * + * @param path target path to concat to. + * @param sources comma seperated absolute paths to use as sources. + */ + public FSConcat(String path, String[] sources) { + this.sources = new Path[sources.length]; + + for(int i = 0; i < sources.length; i++) { + this.sources[i] = new Path(sources[i]); + } + + this.path = new Path(path); + } + + /** + * Executes the filesystem operation. + * + * @param fs filesystem instance to use. + * + * @return void. + * + * @throws IOException thrown if an IO error occured. + */ + @Override + public Void execute(FileSystem fs) throws IOException { + fs.concat(path, sources); + return null; + } + + } + /** * Executor that performs a content-summary FileSystemAccess files system operation. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index b2a28053da7..d217322b6ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -58,6 +58,7 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{DoAsParam.class}); PARAMS_DEF.put(Operation.APPEND, new Class[]{DoAsParam.class, DataParam.class}); + PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class}); PARAMS_DEF.put(Operation.CREATE, new Class[]{DoAsParam.class, PermissionParam.class, OverwriteParam.class, ReplicationParam.class, BlockSizeParam.class, DataParam.class}); @@ -388,6 +389,25 @@ public class HttpFSParametersProvider extends ParametersProvider { } } + /** + * Class for concat sources parameter. + */ + @InterfaceAudience.Private + public static class SourcesParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SOURCES_PARAM; + + /** + * Constructor. + */ + public SourcesParam() { + super(NAME, null); + } + } + /** * Class for to-path parameter. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 0c3418f5c40..ca7edcc7acc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -22,22 +22,23 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AccessTimeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DoAsParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.GroupParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ModifiedTimeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OffsetParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OverwriteParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OwnerParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.PermissionParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ReplicationParam; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.SourcesParam; import org.apache.hadoop.lib.service.FileSystemAccess; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.service.Groups; @@ -403,9 +404,9 @@ public class HttpFSServer { Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case APPEND: { + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); Boolean hasData = params.get(DataParam.NAME, DataParam.class); if (!hasData) { response = Response.temporaryRedirect( @@ -420,6 +421,18 @@ public class HttpFSServer { } break; } + case CONCAT: { + System.out.println("HTTPFS SERVER CONCAT"); + String sources = params.get(SourcesParam.NAME, SourcesParam.class); + + FSOperations.FSConcat command = + new FSOperations.FSConcat(path, sources.split(",")); + fsExecute(user, null, command); + AUDIT_LOG.info("[{}]", path); + System.out.println("SENT RESPONSE"); + response = Response.ok().build(); + break; + } default: { throw new IOException( MessageFormat.format("Invalid HTTP POST operation [{0}]", diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index d44bcc44943..3d96fd83264 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.HFSTestCase; import org.apache.hadoop.test.HadoopUsersConfTestHelper; @@ -206,6 +208,30 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { } } + private void testConcat() throws Exception { + Configuration config = getProxiedFSConf(); + config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + if (!isLocalFS()) { + FileSystem fs = FileSystem.get(config); + fs.mkdirs(getProxiedFSTestDir()); + Path path1 = new Path("/test/foo.txt"); + Path path2 = new Path("/test/bar.txt"); + Path path3 = new Path("/test/derp.txt"); + DFSTestUtil.createFile(fs, path1, 1024, (short) 3, 0); + DFSTestUtil.createFile(fs, path2, 1024, (short) 3, 0); + DFSTestUtil.createFile(fs, path3, 1024, (short) 3, 0); + fs.close(); + fs = getHttpFSFileSystem(); + fs.concat(path1, new Path[]{path2, path3}); + fs.close(); + fs = FileSystem.get(config); + Assert.assertTrue(fs.exists(path1)); + Assert.assertFalse(fs.exists(path2)); + Assert.assertFalse(fs.exists(path3)); + fs.close(); + } + } + private void testRename() throws Exception { FileSystem fs = FileSystem.get(getProxiedFSConf()); Path path = new Path(getProxiedFSTestDir(), "foo"); @@ -450,7 +476,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { } protected enum Operation { - GET, OPEN, CREATE, APPEND, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, + GET, OPEN, CREATE, APPEND, CONCAT, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY } @@ -468,6 +494,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { case APPEND: testAppend(); break; + case CONCAT: + testConcat(); case RENAME: testRename(); break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7008d74ce22..726a52333c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -505,6 +505,8 @@ Release 2.0.3-alpha - Unreleased HDFS-3598. WebHDFS support for file concat. (Plamen Jeliazkov via shv) + HDFS-4456. Add concat to HttpFS and WebHDFS REST API docs. (plamenj2003 via tucu) + OPTIMIZATIONS HDFS-3429. DataNode reads checksums even if client does not need them (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java index c29f2329c48..e6afbe3e4ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hdfs.web.resources; /** The concat source paths parameter. */ public class ConcatSourcesParam extends StringParam { /** Parameter name. */ - public static final String NAME = "srcs"; + public static final String NAME = "sources"; public static final String DEFAULT = NULL; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm index 38b8dc8ab09..90f8dabce71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm @@ -109,6 +109,9 @@ WebHDFS REST API * {{{Append to a File}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append) + * {{{Concat File(s)}<<>>}} + (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat) + * HTTP DELETE * {{{Delete a File/Directory}<<>>}} @@ -299,6 +302,32 @@ Content-Length: 0 {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append +** {Concat File(s)} + + * Submit a HTTP POST request. + ++--------------------------------- +curl -i -X POST "http://:/webhdfs/v1/?op=CONCAT&sources=" ++--------------------------------- + + The client receives a response with zero content length: + ++--------------------------------- +HTTP/1.1 200 OK +Content-Length: 0 ++--------------------------------- + + [] + + This REST API call is available as of Hadoop version 2.0.3. + Please note that is a comma seperated list of absolute paths. + (Example: sources=/test/file1,/test/file2,/test/file3) + + See also: + {{{Sources}<<>>}}, + {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat + + ** {Open and Read a File} * Submit a HTTP GET request with automatically following redirects. @@ -1727,6 +1756,29 @@ var tokenProperties = {{{Set Replication Factor}<<>>}} +** {Sources} + +*----------------+-------------------------------------------------------------------+ +|| Name | <<>> | +*----------------+-------------------------------------------------------------------+ +|| Description | The comma seperated absolute paths used for concatenation. | +*----------------+-------------------------------------------------------------------+ +|| Type | String | +*----------------+-------------------------------------------------------------------+ +|| Default Value | \ | +*----------------+-------------------------------------------------------------------+ +|| Valid Values | A list of comma seperated absolute FileSystem paths without scheme and authority. | +*----------------+-------------------------------------------------------------------+ +|| Syntax | See the note in {{Delegation}}. | +*----------------+-------------------------------------------------------------------+ + + <> that sources are absolute FileSystem paths. + + + See also: + {{{Concat File(s)}<<>>}} + + ** {Token} *----------------+-------------------------------------------------------------------+ From 5c791948de6e768032d6fe1278c915661d0eb14b Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 1 Feb 2013 21:09:05 +0000 Subject: [PATCH 13/69] HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure (Ivan A. Veselovsky via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441628 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/TestLocalFileSystem.java | 86 ++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b1cc950a747..930ec0b7ac9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1297,6 +1297,9 @@ Release 0.23.7 - UNRELEASED HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx permissions (Ivan A. Veselovsky via bobby) + HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure + (Ivan A. Veselovsky via bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index cf1d2df340c..b843e9cf13e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -28,6 +28,7 @@ import java.io.*; import static org.junit.Assert.*; import static org.junit.Assume.assumeTrue; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -38,8 +39,9 @@ public class TestLocalFileSystem { private static final String TEST_ROOT_DIR = System.getProperty("test.build.data","build/test/data") + "/work-dir/localfs"; + private final File base = new File(TEST_ROOT_DIR); private Configuration conf; - private FileSystem fileSys; + private LocalFileSystem fileSys; private void cleanupFile(FileSystem fs, Path name) throws IOException { assertTrue(fs.exists(name)); @@ -53,6 +55,13 @@ public class TestLocalFileSystem { fileSys = FileSystem.getLocal(conf); fileSys.delete(new Path(TEST_ROOT_DIR), true); } + + @After + public void after() throws IOException { + base.setWritable(true); + FileUtil.fullyDelete(base); + assertTrue(!base.exists()); + } /** * Test the capability of setting the working directory. @@ -269,10 +278,83 @@ public class TestLocalFileSystem { LocalFileSystem fs = FileSystem.getLocal(conf); File colonFile = new File(TEST_ROOT_DIR, "foo:bar"); colonFile.mkdirs(); - colonFile.createNewFile(); FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR)); assertEquals("Unexpected number of stats", 1, stats.length); assertEquals("Bad path from stat", colonFile.getAbsolutePath(), stats[0].getPath().toUri().getPath()); } + + @Test + public void testReportChecksumFailure() throws IOException { + base.mkdirs(); + assertTrue(base.exists() && base.isDirectory()); + + final File dir1 = new File(base, "dir1"); + final File dir2 = new File(dir1, "dir2"); + dir2.mkdirs(); + assertTrue(dir2.exists() && dir2.canWrite()); + + final String dataFileName = "corruptedData"; + final Path dataPath = new Path(new File(dir2, dataFileName).toURI()); + final Path checksumPath = fileSys.getChecksumFile(dataPath); + final FSDataOutputStream fsdos = fileSys.create(dataPath); + try { + fsdos.writeUTF("foo"); + } finally { + fsdos.close(); + } + assertTrue(fileSys.pathToFile(dataPath).exists()); + final long dataFileLength = fileSys.getFileStatus(dataPath).getLen(); + assertTrue(dataFileLength > 0); + + // check the the checksum file is created and not empty: + assertTrue(fileSys.pathToFile(checksumPath).exists()); + final long checksumFileLength = fileSys.getFileStatus(checksumPath).getLen(); + assertTrue(checksumFileLength > 0); + + // this is a hack to force the #reportChecksumFailure() method to stop + // climbing up at the 'base' directory and use 'dir1/bad_files' as the + // corrupted files storage: + base.setWritable(false); + + FSDataInputStream dataFsdis = fileSys.open(dataPath); + FSDataInputStream checksumFsdis = fileSys.open(checksumPath); + + boolean retryIsNecessary = fileSys.reportChecksumFailure(dataPath, dataFsdis, 0, checksumFsdis, 0); + assertTrue(!retryIsNecessary); + + // the data file should be moved: + assertTrue(!fileSys.pathToFile(dataPath).exists()); + // the checksum file should be moved: + assertTrue(!fileSys.pathToFile(checksumPath).exists()); + + // check that the files exist in the new location where they were moved: + File[] dir1files = dir1.listFiles(new FileFilter() { + @Override + public boolean accept(File pathname) { + return pathname != null && !pathname.getName().equals("dir2"); + } + }); + assertTrue(dir1files != null); + assertTrue(dir1files.length == 1); + File badFilesDir = dir1files[0]; + + File[] badFiles = badFilesDir.listFiles(); + assertTrue(badFiles != null); + assertTrue(badFiles.length == 2); + boolean dataFileFound = false; + boolean checksumFileFound = false; + for (File badFile: badFiles) { + if (badFile.getName().startsWith(dataFileName)) { + assertTrue(dataFileLength == badFile.length()); + dataFileFound = true; + } else if (badFile.getName().contains(dataFileName + ".crc")) { + assertTrue(checksumFileLength == badFile.length()); + checksumFileFound = true; + } + } + assertTrue(dataFileFound); + assertTrue(checksumFileFound); + } + } From b1baf11fb21e14d78fdf73a1411a3c9437173e24 Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Sat, 2 Feb 2013 00:43:54 +0000 Subject: [PATCH 14/69] YARN-372. Move InlineDispatcher from hadoop-yarn-server-resourcemanager to hadoop-yarn-common. Contributed by Siddharth Seth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441668 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/yarn/event}/InlineDispatcher.java | 5 +++-- .../yarn/server/resourcemanager/TestRMNodeTransitions.java | 2 +- .../server/resourcemanager/resourcetracker/TestNMExpiry.java | 1 + .../resourcetracker/TestRMNMRPCResponseId.java | 1 + .../rmapp/attempt/TestRMAppAttemptTransitions.java | 2 +- .../resourcemanager/scheduler/fifo/TestFifoScheduler.java | 2 +- 8 files changed, 12 insertions(+), 6 deletions(-) rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker => hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event}/InlineDispatcher.java (92%) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java index 656e49e6e70..c0cfc072284 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java @@ -60,7 +60,7 @@ import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Before; diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0813c172794..286a7ed2752 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -213,6 +213,9 @@ Release 2.0.3-alpha - Unreleased does not have permission causes the client to wait forever. (shenhong via tomwhite) + YARN-372. Move InlineDispatcher from hadoop-yarn-server-resourcemanager to + hadoop-yarn-common (sseth via hitesh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/InlineDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java similarity index 92% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/InlineDispatcher.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java index d771a61d864..eb1aa9dc990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/InlineDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.resourcetracker; +package org.apache.hadoop.yarn.event; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +@SuppressWarnings({"unchecked", "rawtypes"}) public class InlineDispatcher extends AsyncDispatcher { private static final Log LOG = LogFactory.getLog(InlineDispatcher.class); @@ -48,7 +49,7 @@ public class InlineDispatcher extends AsyncDispatcher { return new TestEventHandler(); } - static class EmptyEventHandler implements EventHandler { + public static class EmptyEventHandler implements EventHandler { @Override public void handle(Event event) { //do nothing diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 9ae8bf02902..6c14008626a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -35,8 +35,8 @@ import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; -import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 1c4b6f9e0cc..6ec3f5403b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index a7a52c29ab6..7d7f99d054e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 25a4b968fd1..c907df389b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -54,7 +55,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 97310093cbb..c4dbe876c78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -33,13 +33,13 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.Application; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; From a04bee4567cfa0611023eff33f3dcad22fa265e5 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Sat, 2 Feb 2013 00:47:58 +0000 Subject: [PATCH 15/69] MAPREDUCE-4971. Minor extensibility enhancements to Counters & FileOutputFormat. Contributed by Arun C Murthy git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441670 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/mapred/Counters.java | 4 ++++ .../org/apache/hadoop/mapred/FileOutputCommitter.java | 2 +- .../org/apache/hadoop/mapred/FileOutputFormat.java | 5 +++-- .../main/java/org/apache/hadoop/mapred/JobConf.java | 4 +++- .../mapreduce/counters/FileSystemCounterGroup.java | 11 +++++++++++ .../mapreduce/counters/FrameworkCounterGroup.java | 11 +++++++++++ 7 files changed, 36 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e405fde57dc..e09203afb72 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -215,6 +215,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4838. Add additional fields like Locality, Avataar to the JobHistory logs. (Zhijie Shen via sseth) + MAPREDUCE-4971. Minor extensibility enhancements to Counters & + FileOutputFormat. (Arun C Murthy via sseth) + OPTIMIZATIONS MAPREDUCE-4893. Fixed MR ApplicationMaster to do optimal assignment of diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java index da25b2aa137..7d648a49126 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java @@ -230,6 +230,10 @@ public class Counters public static class Group implements CounterGroupBase { private CounterGroupBase realGroup; + protected Group() { + realGroup = null; + } + Group(GenericGroup group) { this.realGroup = group; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java index a6190d2060d..496280a7371 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java @@ -92,7 +92,7 @@ public class FileOutputCommitter extends OutputCommitter { } @Private - Path getTaskAttemptPath(TaskAttemptContext context) throws IOException { + public Path getTaskAttemptPath(TaskAttemptContext context) throws IOException { Path out = getOutputPath(context); return out == null ? null : getTaskAttemptPath(context, out); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java index 892e6906b0c..9082de8ce50 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputFormat.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -152,8 +153,8 @@ public abstract class FileOutputFormat implements OutputFormat { * @param outputDir the {@link Path} of the output directory * for the map-reduce job. */ - - static void setWorkOutputPath(JobConf conf, Path outputDir) { + @Private + public static void setWorkOutputPath(JobConf conf, Path outputDir) { outputDir = new Path(conf.getWorkingDirectory(), outputDir); conf.set(JobContext.TASK_OUTPUT_DIR, outputDir.toString()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 1304755e3c3..e76f62856da 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -28,6 +28,7 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -419,7 +420,8 @@ public class JobConf extends Configuration { return credentials; } - void setCredentials(Credentials credentials) { + @Private + public void setCredentials(Credentials credentials) { this.credentials = credentials; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java index 3f3729fb056..b1b67969cfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java @@ -35,6 +35,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.FileSystemCounter; @@ -72,6 +73,16 @@ public abstract class FileSystemCounterGroup this.scheme = scheme; key = ref; } + + @Private + public String getScheme() { + return scheme; + } + + @Private + public FileSystemCounter getFileSystemCounter() { + return key; + } @Override public String getName() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java index 64e4cc81631..5ebed5a7319 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java @@ -29,6 +29,7 @@ import java.util.Iterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.util.ResourceBundles; @@ -66,7 +67,17 @@ public abstract class FrameworkCounterGroup, key = ref; this.groupName = groupName; } + + @Private + public T getKey() { + return key; + } + @Private + public String getGroupName() { + return groupName; + } + @Override public String getName() { return key.name(); From 61a262757ca70f30956b467ea8e40e73bf7dc634 Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Sat, 2 Feb 2013 02:07:26 +0000 Subject: [PATCH 16/69] HDFS-4452. getAdditionalBlock() can create multiple blocks if the client times out and retries. Contributed by Konstantin Shvachko. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441681 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSNamesystem.java | 270 +++++++++++------- .../server/namenode/TestAddBlockRetry.java | 140 +++++++++ 3 files changed, 305 insertions(+), 108 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 726a52333c6..166fb1b7418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -744,6 +744,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4428. FsDatasetImpl should disclose what the error is when a rename fails. (Colin Patrick McCabe via atm) + HDFS-4452. getAdditionalBlock() can create multiple blocks if the client + times out and retries. (shv) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 54dcd5e7ed2..cd1750f2ea0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2200,12 +2200,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throws LeaseExpiredException, NotReplicatedYetException, QuotaExceededException, SafeModeException, UnresolvedLinkException, IOException { - checkBlock(previous); - Block previousBlock = ExtendedBlock.getLocalBlock(previous); - long fileLength, blockSize; + long blockSize; int replication; DatanodeDescriptor clientNode = null; - Block newBlock = null; if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( @@ -2213,119 +2210,61 @@ public class FSNamesystem implements Namesystem, FSClusterStats, +src+" for "+clientName); } - writeLock(); + // Part I. Analyze the state of the file with respect to the input data. + readLock(); try { - checkOperation(OperationCategory.WRITE); + LocatedBlock[] onRetryBlock = new LocatedBlock[1]; + final INode[] inodes = analyzeFileState( + src, clientName, previous, onRetryBlock).getINodes(); + final INodeFileUnderConstruction pendingFile = + (INodeFileUnderConstruction) inodes[inodes.length - 1]; - if (isInSafeMode()) { - throw new SafeModeException("Cannot add block to " + src, safeMode); + if(onRetryBlock[0] != null) { + // This is a retry. Just return the last block. + return onRetryBlock[0]; } - // have we exceeded the configured limit of fs objects. - checkFsObjectLimit(); - - INodeFileUnderConstruction pendingFile = checkLease(src, clientName); - BlockInfo lastBlockInFile = pendingFile.getLastBlock(); - if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) { - // The block that the client claims is the current last block - // doesn't match up with what we think is the last block. There are - // three possibilities: - // 1) This is the first block allocation of an append() pipeline - // which started appending exactly at a block boundary. - // In this case, the client isn't passed the previous block, - // so it makes the allocateBlock() call with previous=null. - // We can distinguish this since the last block of the file - // will be exactly a full block. - // 2) This is a retry from a client that missed the response of a - // prior getAdditionalBlock() call, perhaps because of a network - // timeout, or because of an HA failover. In that case, we know - // by the fact that the client is re-issuing the RPC that it - // never began to write to the old block. Hence it is safe to - // abandon it and allocate a new one. - // 3) This is an entirely bogus request/bug -- we should error out - // rather than potentially appending a new block with an empty - // one in the middle, etc - - BlockInfo penultimateBlock = pendingFile.getPenultimateBlock(); - if (previous == null && - lastBlockInFile != null && - lastBlockInFile.getNumBytes() == pendingFile.getPreferredBlockSize() && - lastBlockInFile.isComplete()) { - // Case 1 - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.allocateBlock: handling block allocation" + - " writing to a file with a complete previous block: src=" + - src + " lastBlock=" + lastBlockInFile); - } - } else if (Block.matchingIdAndGenStamp(penultimateBlock, previousBlock)) { - // Case 2 - if (lastBlockInFile.getNumBytes() != 0) { - throw new IOException( - "Request looked like a retry to allocate block " + - lastBlockInFile + " but it already contains " + - lastBlockInFile.getNumBytes() + " bytes"); - } - - // The retry case ("b" above) -- abandon the old block. - NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + - "caught retry for allocation of a new block in " + - src + ". Abandoning old block " + lastBlockInFile); - dir.removeBlock(src, pendingFile, lastBlockInFile); - dir.persistBlocks(src, pendingFile); - } else { - - throw new IOException("Cannot allocate block in " + src + ": " + - "passed 'previous' block " + previous + " does not match actual " + - "last block in file " + lastBlockInFile); - } - } - - // commit the last block and complete it if it has minimum replicas - commitOrCompleteLastBlock(pendingFile, previousBlock); - - // - // If we fail this, bad things happen! - // - if (!checkFileProgress(pendingFile, false)) { - throw new NotReplicatedYetException("Not replicated yet:" + src); - } - fileLength = pendingFile.computeContentSummary().getLength(); blockSize = pendingFile.getPreferredBlockSize(); clientNode = pendingFile.getClientNode(); replication = pendingFile.getBlockReplication(); } finally { - writeUnlock(); + readUnlock(); } // choose targets for the new block to be allocated. - final DatanodeDescriptor targets[] = blockManager.chooseTarget( + final DatanodeDescriptor targets[] = getBlockManager().chooseTarget( src, replication, clientNode, excludedNodes, blockSize); - // Allocate a new block and record it in the INode. + // Part II. + // Allocate a new block, add it to the INode and the BlocksMap. + Block newBlock = null; + long offset; writeLock(); try { - checkOperation(OperationCategory.WRITE); - if (isInSafeMode()) { - throw new SafeModeException("Cannot add block to " + src, safeMode); + // Run the full analysis again, since things could have changed + // while chooseTarget() was executing. + LocatedBlock[] onRetryBlock = new LocatedBlock[1]; + INodesInPath inodesInPath = + analyzeFileState(src, clientName, previous, onRetryBlock); + INode[] inodes = inodesInPath.getINodes(); + final INodeFileUnderConstruction pendingFile = + (INodeFileUnderConstruction) inodes[inodes.length - 1]; + + if(onRetryBlock[0] != null) { + // This is a retry. Just return the last block. + return onRetryBlock[0]; } - final INodesInPath inodesInPath = dir.rootDir.getExistingPathINodes(src, true); - final INode[] inodes = inodesInPath.getINodes(); - final INodeFileUnderConstruction pendingFile - = checkLease(src, clientName, inodes[inodes.length - 1]); - - if (!checkFileProgress(pendingFile, false)) { - throw new NotReplicatedYetException("Not replicated yet:" + src); - } + // commit the last block and complete it if it has minimum replicas + commitOrCompleteLastBlock(pendingFile, + ExtendedBlock.getLocalBlock(previous)); + + // allocate new block, record block locations in INode. + newBlock = createNewBlock(); + saveAllocatedBlock(src, inodesInPath, newBlock, targets); - // allocate new block record block locations in INode. - newBlock = allocateBlock(src, inodesInPath, targets); - - for (DatanodeDescriptor dn : targets) { - dn.incBlocksScheduled(); - } dir.persistBlocks(src, pendingFile); + offset = pendingFile.computeFileSize(true); } finally { writeUnlock(); } @@ -2333,10 +2272,114 @@ public class FSNamesystem implements Namesystem, FSClusterStats, getEditLog().logSync(); } - // Create next block - LocatedBlock b = new LocatedBlock(getExtendedBlock(newBlock), targets, fileLength); - blockManager.setBlockToken(b, BlockTokenSecretManager.AccessMode.WRITE); - return b; + // Return located block + return makeLocatedBlock(newBlock, targets, offset); + } + + INodesInPath analyzeFileState(String src, + String clientName, + ExtendedBlock previous, + LocatedBlock[] onRetryBlock) + throws IOException { + assert hasReadOrWriteLock(); + + checkBlock(previous); + onRetryBlock[0] = null; + checkOperation(OperationCategory.WRITE); + if (isInSafeMode()) { + throw new SafeModeException("Cannot add block to " + src, safeMode); + } + + // have we exceeded the configured limit of fs objects. + checkFsObjectLimit(); + + Block previousBlock = ExtendedBlock.getLocalBlock(previous); + final INodesInPath inodesInPath = + dir.rootDir.getExistingPathINodes(src, true); + final INode[] inodes = inodesInPath.getINodes(); + final INodeFileUnderConstruction pendingFile + = checkLease(src, clientName, inodes[inodes.length - 1]); + BlockInfo lastBlockInFile = pendingFile.getLastBlock(); + if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) { + // The block that the client claims is the current last block + // doesn't match up with what we think is the last block. There are + // four possibilities: + // 1) This is the first block allocation of an append() pipeline + // which started appending exactly at a block boundary. + // In this case, the client isn't passed the previous block, + // so it makes the allocateBlock() call with previous=null. + // We can distinguish this since the last block of the file + // will be exactly a full block. + // 2) This is a retry from a client that missed the response of a + // prior getAdditionalBlock() call, perhaps because of a network + // timeout, or because of an HA failover. In that case, we know + // by the fact that the client is re-issuing the RPC that it + // never began to write to the old block. Hence it is safe to + // to return the existing block. + // 3) This is an entirely bogus request/bug -- we should error out + // rather than potentially appending a new block with an empty + // one in the middle, etc + // 4) This is a retry from a client that timed out while + // the prior getAdditionalBlock() is still being processed, + // currently working on chooseTarget(). + // There are no means to distinguish between the first and + // the second attempts in Part I, because the first one hasn't + // changed the namesystem state yet. + // We run this analysis again in Part II where case 4 is impossible. + + BlockInfo penultimateBlock = pendingFile.getPenultimateBlock(); + if (previous == null && + lastBlockInFile != null && + lastBlockInFile.getNumBytes() == pendingFile.getPreferredBlockSize() && + lastBlockInFile.isComplete()) { + // Case 1 + if (NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug( + "BLOCK* NameSystem.allocateBlock: handling block allocation" + + " writing to a file with a complete previous block: src=" + + src + " lastBlock=" + lastBlockInFile); + } + } else if (Block.matchingIdAndGenStamp(penultimateBlock, previousBlock)) { + if (lastBlockInFile.getNumBytes() != 0) { + throw new IOException( + "Request looked like a retry to allocate block " + + lastBlockInFile + " but it already contains " + + lastBlockInFile.getNumBytes() + " bytes"); + } + + // Case 2 + // Return the last block. + NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + + "caught retry for allocation of a new block in " + + src + ". Returning previously allocated block " + lastBlockInFile); + long offset = pendingFile.computeFileSize(true); + onRetryBlock[0] = makeLocatedBlock(lastBlockInFile, + ((BlockInfoUnderConstruction)lastBlockInFile).getExpectedLocations(), + offset); + return inodesInPath; + } else { + // Case 3 + throw new IOException("Cannot allocate block in " + src + ": " + + "passed 'previous' block " + previous + " does not match actual " + + "last block in file " + lastBlockInFile); + } + } + + // Check if the penultimate block is minimally replicated + if (!checkFileProgress(pendingFile, false)) { + throw new NotReplicatedYetException("Not replicated yet: " + src); + } + return inodesInPath; + } + + LocatedBlock makeLocatedBlock(Block blk, + DatanodeInfo[] locs, + long offset) throws IOException { + LocatedBlock lBlk = new LocatedBlock( + getExtendedBlock(blk), locs, offset); + getBlockManager().setBlockToken( + lBlk, BlockTokenSecretManager.AccessMode.WRITE); + return lBlk; } /** @see NameNode#getAdditionalDatanode(String, ExtendedBlock, DatanodeInfo[], DatanodeInfo[], int, String) */ @@ -2528,22 +2571,33 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } /** - * Allocate a block at the given pending filename + * Save allocated block at the given pending filename * * @param src path to the file * @param inodesInPath representing each of the components of src. * The last INode is the INode for the file. * @throws QuotaExceededException If addition of block exceeds space quota */ - private Block allocateBlock(String src, INodesInPath inodesInPath, - DatanodeDescriptor targets[]) throws IOException { + BlockInfo saveAllocatedBlock(String src, INodesInPath inodesInPath, + Block newBlock, DatanodeDescriptor targets[]) throws IOException { + assert hasWriteLock(); + BlockInfo b = dir.addBlock(src, inodesInPath, newBlock, targets); + NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " + + getBlockPoolId() + " " + b); + for (DatanodeDescriptor dn : targets) { + dn.incBlocksScheduled(); + } + return b; + } + + /** + * Create new block with a unique block id and a new generation stamp. + */ + Block createNewBlock() throws IOException { assert hasWriteLock(); Block b = new Block(getFSImage().getUniqueBlockId(), 0, 0); // Increment the generation stamp for every new block. b.setGenerationStamp(nextGenerationStamp()); - b = dir.addBlock(src, inodesInPath, b, targets); - NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " - + blockPoolId + " " + b); return b; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java new file mode 100644 index 00000000000..6bfdcc9b3c1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +import java.lang.reflect.Field; +import java.util.EnumSet; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.EnumSetWritable; +import org.apache.hadoop.net.Node; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * Race between two threads simultaneously calling + * FSNamesystem.getAdditionalBlock(). + */ +public class TestAddBlockRetry { + public static final Log LOG = LogFactory.getLog(TestAddBlockRetry.class); + + private static final short REPLICATION = 3; + + private Configuration conf; + private MiniDFSCluster cluster; + + private int count = 0; + private LocatedBlock lb1; + private LocatedBlock lb2; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(REPLICATION) + .build(); + cluster.waitActive(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Retry addBlock() while another thread is in chooseTarget(). + * See HDFS-4452. + */ + @Test + public void testRetryAddBlockWhileInChooseTarget() throws Exception { + final String src = "/testRetryAddBlockWhileInChooseTarget"; + + FSNamesystem ns = cluster.getNamesystem(); + BlockManager spyBM = spy(ns.getBlockManager()); + final NamenodeProtocols nn = cluster.getNameNodeRpc(); + + // substitute mocked BlockManager into FSNamesystem + Class nsClass = ns.getClass(); + Field bmField = nsClass.getDeclaredField("blockManager"); + bmField.setAccessible(true); + bmField.set(ns, spyBM); + + doAnswer(new Answer() { + @Override + public DatanodeDescriptor[] answer(InvocationOnMock invocation) + throws Throwable { + LOG.info("chooseTarget for " + src); + DatanodeDescriptor[] ret = + (DatanodeDescriptor[]) invocation.callRealMethod(); + count++; + if(count == 1) { // run second addBlock() + LOG.info("Starting second addBlock for " + src); + nn.addBlock(src, "clientName", null, null); + LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE); + assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size()); + lb2 = lbs.get(0); + assertEquals("Wrong replication", + REPLICATION, lb2.getLocations().length); + } + return ret; + } + }).when(spyBM).chooseTarget(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(), Mockito.>any(), + Mockito.anyLong()); + + // create file + nn.create(src, FsPermission.getFileDefault(), + "clientName", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), + true, (short)3, 1024); + + // start first addBlock() + LOG.info("Starting first addBlock for " + src); + nn.addBlock(src, "clientName", null, null); + + // check locations + LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE); + assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size()); + lb1 = lbs.get(0); + assertEquals("Wrong replication", REPLICATION, lb1.getLocations().length); + assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock()); + } +} From 8590564dc56195cb2caa245e3ee1c06eca3938d3 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sat, 2 Feb 2013 22:18:50 +0000 Subject: [PATCH 17/69] HDFS-4350. Make enabling of stale marking on read and write paths independent. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441819 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 12 +-- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 4 +- .../blockmanagement/DatanodeManager.java | 94 +++++++++---------- .../blockmanagement/HeartbeatManager.java | 70 ++++---------- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../src/main/resources/hdfs-default.xml | 23 ++--- .../org/apache/hadoop/hdfs/TestGetBlocks.java | 2 +- .../TestReplicationPolicy.java | 44 ++++----- .../namenode/metrics/TestNameNodeMetrics.java | 2 +- 10 files changed, 109 insertions(+), 147 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 166fb1b7418..3e1e85901b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -317,6 +317,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4451. hdfs balancer command returns exit code 1 on success instead of 0. (Joshua Blatt via suresh) + HDFS-4350. Make enabling of stale marking on read and write paths + independent. (Andrew Wang via suresh) + NEW FEATURES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index c62e9f7fc26..24cdba7e1bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -181,10 +181,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive"; public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000; - // Whether to enable datanode's stale state detection and usage - public static final String DFS_NAMENODE_CHECK_STALE_DATANODE_KEY = "dfs.namenode.check.stale.datanode"; - public static final boolean DFS_NAMENODE_CHECK_STALE_DATANODE_DEFAULT = false; - // Whether to enable datanode's stale state detection and usage + // Whether to enable datanode's stale state detection and usage for reads + public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode"; + public static final boolean DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false; + // Whether to enable datanode's stale state detection and usage for writes public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY = "dfs.namenode.avoid.write.stale.datanode"; public static final boolean DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT = false; // The default value of the time interval for marking datanodes as stale @@ -195,8 +195,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY = "dfs.namenode.stale.datanode.minimum.interval"; public static final int DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_DEFAULT = 3; // i.e. min_interval is 3 * heartbeat_interval = 9s - // When the number stale datanodes marked as stale reached this certian ratio, - // stop avoiding writing to stale nodes so as to prevent causing hotspots. + // When the percentage of stale datanodes reaches this ratio, + // allow writing to stale nodes to prevent hotspots. public static final String DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY = "dfs.namenode.write.stale.datanode.ratio"; public static final float DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT = 0.5f; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 69ef0095db6..7eaff61611e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -134,7 +134,7 @@ public class DFSUtil { /** * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states. * Decommissioned/stale nodes are moved to the end of the array on sorting - * with this compartor. + * with this comparator. */ @InterfaceAudience.Private public static class DecomStaleComparator implements Comparator { @@ -144,7 +144,7 @@ public class DFSUtil { * Constructor of DecomStaleComparator * * @param interval - * The time invertal for marking datanodes as stale is passed from + * The time interval for marking datanodes as stale is passed from * outside, since the interval may be changed dynamically */ public DecomStaleComparator(long interval) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e0a81e0d407..178c25b733d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -126,15 +126,26 @@ public class DatanodeManager { private final long heartbeatExpireInterval; /** Ask Datanode only up to this many blocks to delete. */ final int blockInvalidateLimit; - - /** Whether or not to check stale DataNodes for read/write */ - private final boolean checkForStaleDataNodes; /** The interval for judging stale DataNodes for read/write */ private final long staleInterval; - /** Whether or not to avoid using stale DataNodes for writing */ - private volatile boolean avoidStaleDataNodesForWrite; + /** Whether or not to avoid using stale DataNodes for reading */ + private final boolean avoidStaleDataNodesForRead; + + /** + * Whether or not to avoid using stale DataNodes for writing. + * Note that, even if this is configured, the policy may be + * temporarily disabled when a high percentage of the nodes + * are marked as stale. + */ + private final boolean avoidStaleDataNodesForWrite; + + /** + * When the ratio of stale datanodes reaches this number, stop avoiding + * writing to stale datanodes, i.e., continue using stale nodes for writing. + */ + private final float ratioUseStaleDataNodesForWrite; /** The number of stale DataNodes */ private volatile int numStaleNodes; @@ -183,14 +194,23 @@ public class DatanodeManager { DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, blockInvalidateLimit); LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY + "=" + this.blockInvalidateLimit); - - checkForStaleDataNodes = conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, - DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_DEFAULT); - staleInterval = getStaleIntervalFromConf(conf, heartbeatExpireInterval); - avoidStaleDataNodesForWrite = getAvoidStaleForWriteFromConf(conf, - checkForStaleDataNodes); + this.avoidStaleDataNodesForRead = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT); + this.avoidStaleDataNodesForWrite = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT); + this.staleInterval = getStaleIntervalFromConf(conf, heartbeatExpireInterval); + this.ratioUseStaleDataNodesForWrite = conf.getFloat( + DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY, + DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT); + Preconditions.checkArgument( + (ratioUseStaleDataNodesForWrite > 0 && + ratioUseStaleDataNodesForWrite <= 1.0f), + DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY + + " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " + + "It should be a positive non-zero float value, not greater than 1.0f."); } private static long getStaleIntervalFromConf(Configuration conf, @@ -230,22 +250,6 @@ public class DatanodeManager { return staleInterval; } - static boolean getAvoidStaleForWriteFromConf(Configuration conf, - boolean checkForStale) { - boolean avoid = conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, - DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT); - boolean avoidStaleDataNodesForWrite = checkForStale && avoid; - if (!checkForStale && avoid) { - LOG.warn("Cannot set " - + DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY - + " as false while setting " - + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY - + " as true."); - } - return avoidStaleDataNodesForWrite; - } - void activate(final Configuration conf) { final DecommissionManager dm = new DecommissionManager(namesystem, blockManager); this.decommissionthread = new Daemon(dm.new Monitor( @@ -299,7 +303,7 @@ public class DatanodeManager { client = new NodeBase(rName + NodeBase.PATH_SEPARATOR_STR + targethost); } - Comparator comparator = checkForStaleDataNodes ? + Comparator comparator = avoidStaleDataNodesForRead ? new DFSUtil.DecomStaleComparator(staleInterval) : DFSUtil.DECOM_COMPARATOR; @@ -825,32 +829,20 @@ public class DatanodeManager { } /* Getter and Setter for stale DataNodes related attributes */ - - /** - * @return whether or not to avoid writing to stale datanodes - */ - public boolean isAvoidingStaleDataNodesForWrite() { - return avoidStaleDataNodesForWrite; - } /** - * Set the value of {@link DatanodeManager#avoidStaleDataNodesForWrite}. - * The HeartbeatManager disable avoidStaleDataNodesForWrite when more than - * half of the DataNodes are marked as stale. + * Whether stale datanodes should be avoided as targets on the write path. + * The result of this function may change if the number of stale datanodes + * eclipses a configurable threshold. * - * @param avoidStaleDataNodesForWrite - * The value to set to - * {@link DatanodeManager#avoidStaleDataNodesForWrite} + * @return whether stale datanodes should be avoided on the write path */ - void setAvoidStaleDataNodesForWrite(boolean avoidStaleDataNodesForWrite) { - this.avoidStaleDataNodesForWrite = avoidStaleDataNodesForWrite; - } - - /** - * @return Whether or not to check stale DataNodes for R/W - */ - boolean isCheckingForStaleDataNodes() { - return checkForStaleDataNodes; + public boolean shouldAvoidStaleDataNodesForWrite() { + // If # stale exceeds maximum staleness ratio, disable stale + // datanode avoidance on the write path + return avoidStaleDataNodesForWrite && + (numStaleNodes <= heartbeatManager.getLiveDatanodeCount() + * ratioUseStaleDataNodesForWrite); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 6ee65d38c79..a033da36fbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -30,8 +30,6 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; -import com.google.common.base.Preconditions; - /** * Manage the heartbeats received from datanodes. * The datanode list and statistics are synchronized @@ -56,16 +54,7 @@ class HeartbeatManager implements DatanodeStatistics { private final long heartbeatRecheckInterval; /** Heartbeat monitor thread */ private final Daemon heartbeatThread = new Daemon(new Monitor()); - /** - * The initial setting of end user which indicates whether or not to avoid - * writing to stale datanodes. - */ - private final boolean initialAvoidWriteStaleNodes; - /** - * When the ratio of stale datanodes reaches this number, stop avoiding - * writing to stale datanodes, i.e., continue using stale nodes for writing. - */ - private final float ratioUseStaleDataNodesForWrite; + final Namesystem namesystem; final BlockManager blockManager; @@ -74,30 +63,25 @@ class HeartbeatManager implements DatanodeStatistics { final BlockManager blockManager, final Configuration conf) { this.namesystem = namesystem; this.blockManager = blockManager; - boolean checkStaleNodes = conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, - DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_DEFAULT); + boolean avoidStaleDataNodesForWrite = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT); long recheckInterval = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 min long staleInterval = conf.getLong( - DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, + DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);// 30s - this.initialAvoidWriteStaleNodes = DatanodeManager - .getAvoidStaleForWriteFromConf(conf, checkStaleNodes); - this.ratioUseStaleDataNodesForWrite = conf.getFloat( - DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY, - DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT); - Preconditions.checkArgument( - (ratioUseStaleDataNodesForWrite > 0 && - ratioUseStaleDataNodesForWrite <= 1.0f), - DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY + - " = '" + ratioUseStaleDataNodesForWrite + "' is invalid. " + - "It should be a positive non-zero float value, not greater than 1.0f."); - - this.heartbeatRecheckInterval = (checkStaleNodes - && initialAvoidWriteStaleNodes - && staleInterval < recheckInterval) ? staleInterval : recheckInterval; + + if (avoidStaleDataNodesForWrite && staleInterval < recheckInterval) { + this.heartbeatRecheckInterval = staleInterval; + LOG.info("Setting heartbeat recheck interval to " + staleInterval + + " since " + DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY + + " is less than " + + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY); + } else { + this.heartbeatRecheckInterval = recheckInterval; + } } void activate(Configuration conf) { @@ -242,7 +226,6 @@ class HeartbeatManager implements DatanodeStatistics { if (namesystem.isInSafeMode()) { return; } - boolean checkStaleNodes = dm.isCheckingForStaleDataNodes(); boolean allAlive = false; while (!allAlive) { // locate the first dead node. @@ -254,29 +237,14 @@ class HeartbeatManager implements DatanodeStatistics { if (dead == null && dm.isDatanodeDead(d)) { stats.incrExpiredHeartbeats(); dead = d; - if (!checkStaleNodes) { - break; - } } - if (checkStaleNodes && - d.isStale(dm.getStaleInterval())) { + if (d.isStale(dm.getStaleInterval())) { numOfStaleNodes++; } } - // Change whether to avoid using stale datanodes for writing - // based on proportion of stale datanodes - if (checkStaleNodes) { - dm.setNumStaleNodes(numOfStaleNodes); - if (numOfStaleNodes > - datanodes.size() * ratioUseStaleDataNodesForWrite) { - dm.setAvoidStaleDataNodesForWrite(false); - } else { - if (this.initialAvoidWriteStaleNodes) { - dm.setAvoidStaleDataNodesForWrite(true); - } - } - } + // Set the number of stale nodes in the DatanodeManager + dm.setNumStaleNodes(numOfStaleNodes); } allAlive = dead == null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index cd1750f2ea0..06c4beb345c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5636,7 +5636,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, @Override public boolean isAvoidingStaleDataNodesForWrite() { return this.blockManager.getDatanodeManager() - .isAvoidingStaleDataNodesForWrite(); + .shouldAvoidStaleDataNodesForWrite(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 34cd8465fd7..fa103e7f29a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -999,17 +999,14 @@ - dfs.namenode.check.stale.datanode + dfs.namenode.avoid.read.stale.datanode false - Indicate whether or not to check "stale" datanodes whose + Indicate whether or not to avoid reading from "stale" datanodes whose heartbeat messages have not been received by the namenode - for more than a specified time interval. If this configuration - parameter is set as true, the system will keep track - of the number of stale datanodes. The stale datanodes will be + for more than a specified time interval. Stale datanodes will be moved to the end of the node list returned for reading. See - dfs.namenode.avoid.write.stale.datanode for details on how this - affects writes. + dfs.namenode.avoid.write.stale.datanode for a similar setting for writes. @@ -1017,13 +1014,13 @@ dfs.namenode.avoid.write.stale.datanode false - Indicate whether or not to avoid writing to "stale" datanodes whose + Indicate whether or not to avoid writing to "stale" datanodes whose heartbeat messages have not been received by the namenode - for more than a specified time interval. If this configuration - parameter and dfs.namenode.check.stale.datanode are both set as true, - the writing will avoid using stale datanodes unless a high number - of datanodes are marked as stale. See - dfs.namenode.write.stale.datanode.ratio for details. + for more than a specified time interval. Writes will avoid using + stale datanodes unless more than a configured ratio + (dfs.namenode.write.stale.datanode.ratio) of datanodes are marked as + stale. See dfs.namenode.avoid.read.stale.datanode for a similar setting + for reads. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java index 55d1aa7ee51..6eaa55fbe92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java @@ -88,7 +88,7 @@ public class TestGetBlocks { @Test public void testReadSelectNonStaleDatanode() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true); long staleInterval = 30 * 1000 * 60; conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, staleInterval); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 78ac007e35a..38a5f0df3bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -88,9 +88,11 @@ public class TestReplicationPolicy { "test.build.data", "build/test/data"), "dfs/"); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath()); - // Enable the checking for stale datanodes in the beginning - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); DFSTestUtil.formatNameNode(conf); namenode = new NameNode(conf); @@ -100,6 +102,8 @@ public class TestReplicationPolicy { // construct network topology for (int i=0; i < NUM_OF_DATANODES; i++) { cluster.add(dataNodes[i]); + bm.getDatanodeManager().getHeartbeatManager().addDatanode( + dataNodes[i]); } for (int i=0; i < NUM_OF_DATANODES; i++) { dataNodes[i].updateHeartbeat( @@ -393,11 +397,11 @@ public class TestReplicationPolicy { throws Exception { try { namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(true); + .setNumStaleNodes(NUM_OF_DATANODES); testChooseTargetWithMoreThanAvailableNodes(); } finally { namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(false); + .setNumStaleNodes(0); } } @@ -479,12 +483,12 @@ public class TestReplicationPolicy { @Test public void testChooseTargetWithStaleNodes() throws Exception { - // Enable avoidng writing to stale datanodes - namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(true); // Set dataNodes[0] as stale dataNodes[0].setLastUpdate(Time.now() - staleInterval - 1); - + namenode.getNamesystem().getBlockManager() + .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); + assertTrue(namenode.getNamesystem().getBlockManager() + .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); DatanodeDescriptor[] targets; // We set the datanode[0] as stale, thus should choose datanode[1] since // datanode[1] is on the same rack with datanode[0] (writer) @@ -503,9 +507,9 @@ public class TestReplicationPolicy { assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); // reset - namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(false); dataNodes[0].setLastUpdate(Time.now()); + namenode.getNamesystem().getBlockManager() + .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); } /** @@ -518,20 +522,20 @@ public class TestReplicationPolicy { */ @Test public void testChooseTargetWithHalfStaleNodes() throws Exception { - // Enable stale datanodes checking - namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(true); // Set dataNodes[0], dataNodes[1], and dataNodes[2] as stale for (int i = 0; i < 3; i++) { dataNodes[i].setLastUpdate(Time.now() - staleInterval - 1); } + namenode.getNamesystem().getBlockManager() + .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], new ArrayList(), BLOCK_SIZE); assertEquals(targets.length, 0); - // We set the datanode[0] as stale, thus should choose datanode[1] + // Since we have 6 datanodes total, stale nodes should + // not be returned until we ask for more than 3 targets targets = replicator.chooseTarget(filename, 1, dataNodes[0], new ArrayList(), BLOCK_SIZE); assertEquals(targets.length, 1); @@ -557,18 +561,16 @@ public class TestReplicationPolicy { assertTrue(containsWithinRange(dataNodes[4], targets, 0, 3)); assertTrue(containsWithinRange(dataNodes[5], targets, 0, 3)); - // reset - namenode.getNamesystem().getBlockManager().getDatanodeManager() - .setAvoidStaleDataNodesForWrite(false); for (int i = 0; i < dataNodes.length; i++) { dataNodes[i].setLastUpdate(Time.now()); } + namenode.getNamesystem().getBlockManager() + .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); } @Test public void testChooseTargetWithMoreThanHalfStaleNodes() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true); conf.setBoolean( DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); String[] hosts = new String[]{"host1", "host2", "host3", @@ -598,7 +600,7 @@ public class TestReplicationPolicy { .getBlockManager().getDatanodeManager().getNumStaleNodes(); assertEquals(numStaleNodes, 2); assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager() - .getDatanodeManager().isAvoidingStaleDataNodesForWrite()); + .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); // Call chooseTarget DatanodeDescriptor staleNodeInfo = miniCluster.getNameNode() .getNamesystem().getBlockManager().getDatanodeManager() @@ -627,7 +629,7 @@ public class TestReplicationPolicy { // According to our strategy, stale datanodes will be included for writing // to avoid hotspots assertFalse(miniCluster.getNameNode().getNamesystem().getBlockManager() - .getDatanodeManager().isAvoidingStaleDataNodesForWrite()); + .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); // Call chooseTarget targets = replicator.chooseTarget(filename, 3, staleNodeInfo, new ArrayList(), BLOCK_SIZE); @@ -650,7 +652,7 @@ public class TestReplicationPolicy { .getBlockManager().getDatanodeManager().getNumStaleNodes(); assertEquals(numStaleNodes, 2); assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager() - .getDatanodeManager().isAvoidingStaleDataNodesForWrite()); + .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); // Call chooseTarget targets = replicator.chooseTarget(filename, 3, staleNodeInfo, new ArrayList(), BLOCK_SIZE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index d31f12ddfa3..56bf4a8ecae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -82,7 +82,7 @@ public class TestNameNodeMetrics { CONF.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + PERCENTILES_INTERVAL); // Enable stale DataNodes checking - CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true); + CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true); ((Log4JLogger)LogFactory.getLog(MetricsAsserts.class)) .getLogger().setLevel(Level.DEBUG); } From 52e6f5a27643be6ab4cc9536f3bec93f5d77d4f9 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Sun, 3 Feb 2013 17:33:55 +0000 Subject: [PATCH 18/69] HDFS-4445. All BKJM ledgers are not checked while tailing, So failover will fail. Contributed Vinay. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441935 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../bkjournal/BookKeeperJournalManager.java | 18 +++++++- .../TestBookKeeperAsHASharedDir.java | 41 +++++++++++++++++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3e1e85901b9..00c0fb83c84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -750,6 +750,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4452. getAdditionalBlock() can create multiple blocks if the client times out and retries. (shv) + HDFS-4445. All BKJM ledgers are not checked while tailing, So failover will fail. + (Vinay via umamahesh) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java index 5d1814233f8..2baf4dc0747 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java @@ -503,7 +503,8 @@ public class BookKeeperJournalManager implements JournalManager { @Override public void selectInputStreams(Collection streams, long fromTxId, boolean inProgressOk) throws IOException { - List currentLedgerList = getLedgerList(inProgressOk); + List currentLedgerList = getLedgerList(fromTxId, + inProgressOk); try { BookKeeperEditLogInputStream elis = null; for (EditLogLedgerMetadata l : currentLedgerList) { @@ -511,6 +512,8 @@ public class BookKeeperJournalManager implements JournalManager { if (l.isInProgress()) { lastTxId = recoverLastTxId(l, false); } + // Check once again, required in case of InProgress and is case of any + // gap. if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) { LedgerHandle h; if (l.isInProgress()) { // we don't want to fence the current journal @@ -523,6 +526,8 @@ public class BookKeeperJournalManager implements JournalManager { elis = new BookKeeperEditLogInputStream(h, l); elis.skipTo(fromTxId); } else { + // If mismatches then there might be some gap, so we should not check + // further. return; } streams.add(elis); @@ -732,6 +737,11 @@ public class BookKeeperJournalManager implements JournalManager { */ List getLedgerList(boolean inProgressOk) throws IOException { + return getLedgerList(-1, inProgressOk); + } + + private List getLedgerList(long fromTxId, + boolean inProgressOk) throws IOException { List ledgers = new ArrayList(); try { @@ -744,6 +754,12 @@ public class BookKeeperJournalManager implements JournalManager { try { EditLogLedgerMetadata editLogLedgerMetadata = EditLogLedgerMetadata .read(zkc, legderMetadataPath); + if (editLogLedgerMetadata.getLastTxId() != HdfsConstants.INVALID_TXID + && editLogLedgerMetadata.getLastTxId() < fromTxId) { + // exclude already read closed edits, but include inprogress edits + // as this will be handled in caller + continue; + } ledgers.add(editLogLedgerMetadata); } catch (KeeperException.NoNodeException e) { LOG.warn("ZNode: " + legderMetadataPath diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java index ebbf80aa375..0a14e785758 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java @@ -21,7 +21,6 @@ import static org.junit.Assert.*; import org.junit.Test; import org.junit.Before; -import org.junit.After; import org.junit.BeforeClass; import org.junit.AfterClass; @@ -34,11 +33,9 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.ipc.RemoteException; @@ -352,4 +349,42 @@ public class TestBookKeeperAsHASharedDir { } } } + + /** + * NameNode should load the edits correctly if the applicable edits are + * present in the BKJM. + */ + @Test + public void testNameNodeMultipleSwitchesUsingBKJM() throws Exception { + MiniDFSCluster cluster = null; + try { + Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil + .createJournalURI("/correctEditLogSelection").toString()); + BKJMUtil.addJournalManagerDefinition(conf); + + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0) + .manageNameDfsSharedDirs(false).build(); + NameNode nn1 = cluster.getNameNode(0); + NameNode nn2 = cluster.getNameNode(1); + cluster.waitActive(); + cluster.transitionToActive(0); + nn1.getRpcServer().rollEditLog(); // Roll Edits from current Active. + // Transition to standby current active gracefully. + cluster.transitionToStandby(0); + // Make the other Active and Roll edits multiple times + cluster.transitionToActive(1); + nn2.getRpcServer().rollEditLog(); + nn2.getRpcServer().rollEditLog(); + // Now One more failover. So NN1 should be able to failover successfully. + cluster.transitionToStandby(1); + cluster.transitionToActive(0); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } From 114e23f7bd1ec6b6711a2eb4ed35f9c3a58908ae Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Mon, 4 Feb 2013 19:42:51 +0000 Subject: [PATCH 19/69] HADOOP-9276. Allow BoundedByteArrayOutputStream to be resettable. Contributed by Arun Murthy git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442312 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../io/BoundedByteArrayOutputStream.java | 47 +++++++++------ .../io/TestBoundedByteArrayOutputStream.java | 57 +++++++++++++++++++ 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 930ec0b7ac9..f08e20a2d94 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -467,6 +467,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9231. Parametrize staging URL for the uniformity of distributionManagement. (Konstantin Boudnik via suresh) + HADOOP-9276. Allow BoundedByteArrayOutputStream to be resettable. + (Arun Murthy via hitesh) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java index 60836ab26a3..c27449d3618 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java @@ -32,9 +32,10 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Unstable public class BoundedByteArrayOutputStream extends OutputStream { - private final byte[] buffer; + private byte[] buffer; + private int startOffset; private int limit; - private int count; + private int currentPointer; /** * Create a BoundedByteArrayOutputStream with the specified @@ -52,20 +53,30 @@ public class BoundedByteArrayOutputStream extends OutputStream { * @param limit The maximum limit upto which data can be written */ public BoundedByteArrayOutputStream(int capacity, int limit) { + this(new byte[capacity], 0, limit); + } + + protected BoundedByteArrayOutputStream(byte[] buf, int offset, int limit) { + resetBuffer(buf, offset, limit); + } + + protected void resetBuffer(byte[] buf, int offset, int limit) { + int capacity = buf.length - offset; if ((capacity < limit) || (capacity | limit) < 0) { throw new IllegalArgumentException("Invalid capacity/limit"); } - this.buffer = new byte[capacity]; - this.limit = limit; - this.count = 0; + this.buffer = buf; + this.startOffset = offset; + this.currentPointer = offset; + this.limit = offset + limit; } - + @Override public void write(int b) throws IOException { - if (count >= limit) { + if (currentPointer >= limit) { throw new EOFException("Reaching the limit of the buffer."); } - buffer[count++] = (byte) b; + buffer[currentPointer++] = (byte) b; } @Override @@ -77,12 +88,12 @@ public class BoundedByteArrayOutputStream extends OutputStream { return; } - if (count + len > limit) { + if (currentPointer + len > limit) { throw new EOFException("Reach the limit of the buffer"); } - System.arraycopy(b, off, buffer, count, len); - count += len; + System.arraycopy(b, off, buffer, currentPointer, len); + currentPointer += len; } /** @@ -90,17 +101,17 @@ public class BoundedByteArrayOutputStream extends OutputStream { * @param newlim New Limit */ public void reset(int newlim) { - if (newlim > buffer.length) { + if (newlim > (buffer.length - startOffset)) { throw new IndexOutOfBoundsException("Limit exceeds buffer size"); } this.limit = newlim; - this.count = 0; + this.currentPointer = startOffset; } /** Reset the buffer */ public void reset() { - this.limit = buffer.length; - this.count = 0; + this.limit = buffer.length - startOffset; + this.currentPointer = startOffset; } /** Return the current limit */ @@ -119,6 +130,10 @@ public class BoundedByteArrayOutputStream extends OutputStream { * currently in the buffer. */ public int size() { - return count; + return currentPointer - startOffset; + } + + public int available() { + return limit - currentPointer; } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBoundedByteArrayOutputStream.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBoundedByteArrayOutputStream.java index a00d38bfa63..44215278ca6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBoundedByteArrayOutputStream.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBoundedByteArrayOutputStream.java @@ -88,4 +88,61 @@ public class TestBoundedByteArrayOutputStream extends TestCase { assertTrue("Writing beyond limit did not throw an exception", caughtException); } + + + static class ResettableBoundedByteArrayOutputStream + extends BoundedByteArrayOutputStream { + + public ResettableBoundedByteArrayOutputStream(int capacity) { + super(capacity); + } + + public void resetBuffer(byte[] buf, int offset, int length) { + super.resetBuffer(buf, offset, length); + } + + } + + public void testResetBuffer() throws IOException { + + ResettableBoundedByteArrayOutputStream stream = + new ResettableBoundedByteArrayOutputStream(SIZE); + + // Write to the stream, get the data back and check for contents + stream.write(INPUT, 0, SIZE); + assertTrue("Array Contents Mismatch", + Arrays.equals(INPUT, stream.getBuffer())); + + // Try writing beyond end of buffer. Should throw an exception + boolean caughtException = false; + + try { + stream.write(INPUT[0]); + } catch (Exception e) { + caughtException = true; + } + + assertTrue("Writing beyond limit did not throw an exception", + caughtException); + + //Reset the stream and try, should succeed + byte[] newBuf = new byte[SIZE]; + stream.resetBuffer(newBuf, 0, newBuf.length); + assertTrue("Limit did not get reset correctly", + (stream.getLimit() == SIZE)); + stream.write(INPUT, 0, SIZE); + assertTrue("Array Contents Mismatch", + Arrays.equals(INPUT, stream.getBuffer())); + + // Try writing one more byte, should fail + caughtException = false; + try { + stream.write(INPUT[0]); + } catch (Exception e) { + caughtException = true; + } + assertTrue("Writing beyond limit did not throw an exception", + caughtException); + } + } From a8e39feed2642dbfedb8fd22648904ad4bb6af97 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Mon, 4 Feb 2013 21:29:39 +0000 Subject: [PATCH 20/69] HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a pre-federation version of HDFS. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442375 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/common/Storage.java | 2 +- .../hdfs/server/common/StorageInfo.java | 6 +++++ .../server/namenode/CheckpointSignature.java | 4 ++++ .../hdfs/server/namenode/NNStorage.java | 4 ++-- .../server/namenode/SecondaryNameNode.java | 12 +++++++--- .../hdfs/server/namenode/FSImageTestUtil.java | 6 ++++- .../TestSecondaryNameNodeUpgrade.java | 23 +++++++++++++++---- 8 files changed, 48 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 00c0fb83c84..86c31dee49c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -753,6 +753,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4445. All BKJM ledgers are not checked while tailing, So failover will fail. (Vinay via umamahesh) + HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a + pre-federation version of HDFS. (atm) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index fc69978fb22..acb4ce79588 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -905,7 +905,7 @@ public abstract class Storage extends StorageInfo { props.setProperty("storageType", storageType.toString()); props.setProperty("namespaceID", String.valueOf(namespaceID)); // Set clusterID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { props.setProperty("clusterID", clusterID); } props.setProperty("cTime", String.valueOf(cTime)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java index c3dd5d6209b..1dc83405303 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.common; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import com.google.common.base.Joiner; @@ -77,6 +79,10 @@ public class StorageInfo { namespaceID = from.namespaceID; cTime = from.cTime; } + + public boolean versionSupportsFederation() { + return LayoutVersion.supports(Feature.FEDERATION, layoutVersion); + } @Override public String toString() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java index 8c4d79cc9bf..13d8f85cbfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java @@ -123,6 +123,10 @@ public class CheckpointSignature extends StorageInfo blockpoolID.equals(si.getBlockPoolID()); } + boolean namespaceIdMatches(FSImage si) { + return namespaceID == si.getStorage().namespaceID; + } + void validateStorageInfo(FSImage si) throws IOException { if (!isSameCluster(si) || !storageVersionMatches(si.getStorage())) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index abc871fa9f1..a5a4167e294 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -587,7 +587,7 @@ public class NNStorage extends Storage implements Closeable, } // Set Block pool ID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); } @@ -634,7 +634,7 @@ public class NNStorage extends Storage implements Closeable, ) throws IOException { super.setPropertiesFromFields(props, sd); // Set blockpoolID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { props.setProperty("blockpoolID", blockpoolID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index c22d941010a..763c7089abd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -475,14 +475,20 @@ public class SecondaryNameNode implements Runnable { // Returns a token that would be used to upload the merged image. CheckpointSignature sig = namenode.rollEditLog(); - if ((checkpointImage.getNamespaceID() == 0) || - (sig.isSameCluster(checkpointImage) && + boolean loadImage = false; + boolean isFreshCheckpointer = (checkpointImage.getNamespaceID() == 0); + boolean isSameCluster = + (dstStorage.versionSupportsFederation() && sig.isSameCluster(checkpointImage)) || + (!dstStorage.versionSupportsFederation() && sig.namespaceIdMatches(checkpointImage)); + if (isFreshCheckpointer || + (isSameCluster && !sig.storageVersionMatches(checkpointImage.getStorage()))) { // if we're a fresh 2NN, or if we're on the same cluster and our storage // needs an upgrade, just take the storage info from the server. dstStorage.setStorageInfo(sig); dstStorage.setClusterID(sig.getClusterID()); dstStorage.setBlockPoolID(sig.getBlockpoolID()); + loadImage = true; } sig.validateStorageInfo(checkpointImage); @@ -492,7 +498,7 @@ public class SecondaryNameNode implements Runnable { RemoteEditLogManifest manifest = namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1); - boolean loadImage = downloadCheckpointFiles( + loadImage |= downloadCheckpointFiles( fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits doMerge(sig, manifest, loadImage, checkpointImage, namesystem); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index d75c17bf964..f4dc62bf45e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -506,7 +506,11 @@ public abstract class FSImageTestUtil { props.load(fis); IOUtils.closeStream(fis); - props.setProperty(key, value); + if (value == null || value.isEmpty()) { + props.remove(key); + } else { + props.setProperty(key, value); + } out = new FileOutputStream(versionFile); props.store(out, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java index f3925c963cd..7f5110739ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import com.google.common.collect.ImmutableMap; + import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import org.junit.Test; import org.junit.Before; @@ -51,7 +54,7 @@ public class TestSecondaryNameNodeUpgrade { } } - private void doIt(String param, String val) throws IOException { + private void doIt(Map paramsToCorrupt) throws IOException { MiniDFSCluster cluster = null; FileSystem fs = null; SecondaryNameNode snn = null; @@ -76,8 +79,12 @@ public class TestSecondaryNameNodeUpgrade { snn.shutdown(); for (File versionFile : versionFiles) { - System.out.println("Changing '" + param + "' to '" + val + "' in " + versionFile); - FSImageTestUtil.corruptVersionFile(versionFile, param, val); + for (Map.Entry paramToCorrupt : paramsToCorrupt.entrySet()) { + String param = paramToCorrupt.getKey(); + String val = paramToCorrupt.getValue(); + System.out.println("Changing '" + param + "' to '" + val + "' in " + versionFile); + FSImageTestUtil.corruptVersionFile(versionFile, param, val); + } } snn = new SecondaryNameNode(conf); @@ -94,13 +101,19 @@ public class TestSecondaryNameNodeUpgrade { @Test public void testUpgradeLayoutVersionSucceeds() throws IOException { - doIt("layoutVersion", "-39"); + doIt(ImmutableMap.of("layoutVersion", "-39")); + } + + @Test + public void testUpgradePreFedSucceeds() throws IOException { + doIt(ImmutableMap.of("layoutVersion", "-19", "clusterID", "", + "blockpoolID", "")); } @Test public void testChangeNsIDFails() throws IOException { try { - doIt("namespaceID", "2"); + doIt(ImmutableMap.of("namespaceID", "2")); Assert.fail("Should throw InconsistentFSStateException"); } catch(IOException e) { GenericTestUtils.assertExceptionContains("Inconsistent checkpoint fields", e); From ef2ff99d36752d2e95236c4ab8e5290a2e41e5bf Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 4 Feb 2013 21:40:59 +0000 Subject: [PATCH 21/69] HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and the synchronization of limitDecimalTo2(double) can be avoided. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442386 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/shell/FsUsage.java | 2 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 2 +- .../org/apache/hadoop/util/StringUtils.java | 159 ++++++++++-------- .../apache/hadoop/util/TestStringUtils.java | 131 +++++++++++---- 5 files changed, 186 insertions(+), 111 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f08e20a2d94..125e9f83b77 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -599,6 +599,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9124. SortedMapWritable violates contract of Map interface for equals() and hashCode(). (Surenkumar Nihalani via tomwhite) + HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and + the synchronization of limitDecimalTo2(double) can be avoided. (szetszwo) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java index 0d73f76e666..a99494573fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java @@ -48,7 +48,7 @@ class FsUsage extends FsCommand { protected String formatSize(long size) { return humanReadable - ? StringUtils.humanReadableInt(size) + ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) : String.valueOf(size); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index 289adea8d34..76192e73b4a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -67,7 +67,7 @@ class Ls extends FsCommand { protected boolean humanReadable = false; protected String formatSize(long size) { return humanReadable - ? StringUtils.humanReadableInt(size) + ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) : String.valueOf(size); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index cf1d58b4e23..898901e5053 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -23,8 +23,6 @@ import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.text.DateFormat; -import java.text.DecimalFormat; -import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -34,12 +32,13 @@ import java.util.List; import java.util.Locale; import java.util.StringTokenizer; -import com.google.common.net.InetAddresses; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; import org.apache.hadoop.net.NetUtils; +import com.google.common.net.InetAddresses; + /** * General string utils */ @@ -52,13 +51,6 @@ public class StringUtils { */ public static final int SHUTDOWN_HOOK_PRIORITY = 0; - private static final DecimalFormat decimalFormat; - static { - NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH); - decimalFormat = (DecimalFormat) numberFormat; - decimalFormat.applyPattern("#.##"); - } - /** * Make a string representation of the exception. * @param e The exception to stringify @@ -87,50 +79,33 @@ public class StringUtils { } return fullHostname; } - - private static DecimalFormat oneDecimal = new DecimalFormat("0.0"); /** * Given an integer, return a string that is in an approximate, but human * readable format. - * It uses the bases 'k', 'm', and 'g' for 1024, 1024**2, and 1024**3. * @param number the number to format * @return a human readable form of the integer + * + * @deprecated use {@link TraditionalBinaryPrefix#long2String(long, String, int)}. */ + @Deprecated public static String humanReadableInt(long number) { - long absNumber = Math.abs(number); - double result = number; - String suffix = ""; - if (absNumber < 1024) { - // since no division has occurred, don't format with a decimal point - return String.valueOf(number); - } else if (absNumber < 1024 * 1024) { - result = number / 1024.0; - suffix = "k"; - } else if (absNumber < 1024 * 1024 * 1024) { - result = number / (1024.0 * 1024); - suffix = "m"; - } else { - result = number / (1024.0 * 1024 * 1024); - suffix = "g"; - } - return oneDecimal.format(result) + suffix; + return TraditionalBinaryPrefix.long2String(number, "", 1); } - + + /** The same as String.format(Locale.ENGLISH, format, objects). */ + public static String format(final String format, final Object... objects) { + return String.format(Locale.ENGLISH, format, objects); + } + /** * Format a percentage for presentation to the user. - * @param done the percentage to format (0.0 to 1.0) - * @param digits the number of digits past the decimal point + * @param fraction the percentage as a fraction, e.g. 0.1 = 10% + * @param decimalPlaces the number of decimal places * @return a string representation of the percentage */ - public static String formatPercent(double done, int digits) { - DecimalFormat percentFormat = new DecimalFormat("0.00%"); - double scale = Math.pow(10.0, digits+2); - double rounded = Math.floor(done * scale); - percentFormat.setDecimalSeparatorAlwaysShown(false); - percentFormat.setMinimumFractionDigits(digits); - percentFormat.setMaximumFractionDigits(digits); - return percentFormat.format(rounded / scale); + public static String formatPercent(double fraction, int decimalPlaces) { + return format("%." + decimalPlaces + "f%%", fraction*100); } /** @@ -165,7 +140,7 @@ public class StringUtils { } StringBuilder s = new StringBuilder(); for(int i = start; i < end; i++) { - s.append(String.format("%02x", bytes[i])); + s.append(format("%02x", bytes[i])); } return s.toString(); } @@ -630,18 +605,22 @@ public class StringUtils { * TraditionalBinaryPrefix symbol are case insensitive. */ public static enum TraditionalBinaryPrefix { - KILO(1024), - MEGA(KILO.value << 10), - GIGA(MEGA.value << 10), - TERA(GIGA.value << 10), - PETA(TERA.value << 10), - EXA(PETA.value << 10); + KILO(10), + MEGA(KILO.bitShift + 10), + GIGA(MEGA.bitShift + 10), + TERA(GIGA.bitShift + 10), + PETA(TERA.bitShift + 10), + EXA (PETA.bitShift + 10); public final long value; public final char symbol; + public final int bitShift; + public final long bitMask; - TraditionalBinaryPrefix(long value) { - this.value = value; + private TraditionalBinaryPrefix(int bitShift) { + this.bitShift = bitShift; + this.value = 1L << bitShift; + this.bitMask = this.value - 1L; this.symbol = toString().charAt(0); } @@ -692,8 +671,58 @@ public class StringUtils { return num * prefix; } } + + /** + * Convert a long integer to a string with traditional binary prefix. + * + * @param n the value to be converted + * @param unit The unit, e.g. "B" for bytes. + * @param decimalPlaces The number of decimal places. + * @return a string with traditional binary prefix. + */ + public static String long2String(long n, String unit, int decimalPlaces) { + if (unit == null) { + unit = ""; + } + //take care a special case + if (n == Long.MIN_VALUE) { + return "-8 " + EXA.symbol + unit; + } + + final StringBuilder b = new StringBuilder(); + //take care negative numbers + if (n < 0) { + b.append('-'); + n = -n; + } + if (n < KILO.value) { + //no prefix + b.append(n); + return (unit.isEmpty()? b: b.append(" ").append(unit)).toString(); + } else { + //find traditional binary prefix + int i = 0; + for(; i < values().length && n >= values()[i].value; i++); + TraditionalBinaryPrefix prefix = values()[i - 1]; + + if ((n & prefix.bitMask) == 0) { + //exact division + b.append(n >> prefix.bitShift); + } else { + final String format = "%." + decimalPlaces + "f"; + String s = format(format, n/(double)prefix.value); + //check a special rounding up case + if (s.startsWith("1024")) { + prefix = values()[i]; + s = format(format, n/(double)prefix.value); + } + b.append(s); + } + return b.append(' ').append(prefix.symbol).append(unit).toString(); + } + } } - + /** * Escapes HTML Special characters present in the string. * @param string @@ -731,32 +760,16 @@ public class StringUtils { } /** - * Return an abbreviated English-language desc of the byte length + * @return a byte description of the given long interger value. */ public static String byteDesc(long len) { - double val = 0.0; - String ending = ""; - if (len < 1024 * 1024) { - val = (1.0 * len) / 1024; - ending = " KB"; - } else if (len < 1024 * 1024 * 1024) { - val = (1.0 * len) / (1024 * 1024); - ending = " MB"; - } else if (len < 1024L * 1024 * 1024 * 1024) { - val = (1.0 * len) / (1024 * 1024 * 1024); - ending = " GB"; - } else if (len < 1024L * 1024 * 1024 * 1024 * 1024) { - val = (1.0 * len) / (1024L * 1024 * 1024 * 1024); - ending = " TB"; - } else { - val = (1.0 * len) / (1024L * 1024 * 1024 * 1024 * 1024); - ending = " PB"; - } - return limitDecimalTo2(val) + ending; + return TraditionalBinaryPrefix.long2String(len, "B", 2); } - public static synchronized String limitDecimalTo2(double d) { - return decimalFormat.format(d); + /** @deprecated use StringUtils.format("%.2f", d). */ + @Deprecated + public static String limitDecimalTo2(double d) { + return format("%.2f", d); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 8d2fa155e1f..3dcf8dd3979 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -18,6 +18,8 @@ package org.apache.hadoop.util; +import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String; +import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.string2long; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -26,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.test.UnitTestcaseTimeLimit; +import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; import org.junit.Test; public class TestStringUtils extends UnitTestcaseTimeLimit { @@ -134,45 +137,34 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { @Test public void testTraditionalBinaryPrefix() throws Exception { + //test string2long(..) String[] symbol = {"k", "m", "g", "t", "p", "e"}; long m = 1024; for(String s : symbol) { - assertEquals(0, StringUtils.TraditionalBinaryPrefix.string2long(0 + s)); - assertEquals(m, StringUtils.TraditionalBinaryPrefix.string2long(1 + s)); + assertEquals(0, string2long(0 + s)); + assertEquals(m, string2long(1 + s)); m *= 1024; } - assertEquals(0L, StringUtils.TraditionalBinaryPrefix.string2long("0")); - assertEquals(1024L, StringUtils.TraditionalBinaryPrefix.string2long("1k")); - assertEquals(-1024L, StringUtils.TraditionalBinaryPrefix.string2long("-1k")); - assertEquals(1259520L, - StringUtils.TraditionalBinaryPrefix.string2long("1230K")); - assertEquals(-1259520L, - StringUtils.TraditionalBinaryPrefix.string2long("-1230K")); - assertEquals(104857600L, - StringUtils.TraditionalBinaryPrefix.string2long("100m")); - assertEquals(-104857600L, - StringUtils.TraditionalBinaryPrefix.string2long("-100M")); - assertEquals(956703965184L, - StringUtils.TraditionalBinaryPrefix.string2long("891g")); - assertEquals(-956703965184L, - StringUtils.TraditionalBinaryPrefix.string2long("-891G")); - assertEquals(501377302265856L, - StringUtils.TraditionalBinaryPrefix.string2long("456t")); - assertEquals(-501377302265856L, - StringUtils.TraditionalBinaryPrefix.string2long("-456T")); - assertEquals(11258999068426240L, - StringUtils.TraditionalBinaryPrefix.string2long("10p")); - assertEquals(-11258999068426240L, - StringUtils.TraditionalBinaryPrefix.string2long("-10P")); - assertEquals(1152921504606846976L, - StringUtils.TraditionalBinaryPrefix.string2long("1e")); - assertEquals(-1152921504606846976L, - StringUtils.TraditionalBinaryPrefix.string2long("-1E")); + assertEquals(0L, string2long("0")); + assertEquals(1024L, string2long("1k")); + assertEquals(-1024L, string2long("-1k")); + assertEquals(1259520L, string2long("1230K")); + assertEquals(-1259520L, string2long("-1230K")); + assertEquals(104857600L, string2long("100m")); + assertEquals(-104857600L, string2long("-100M")); + assertEquals(956703965184L, string2long("891g")); + assertEquals(-956703965184L, string2long("-891G")); + assertEquals(501377302265856L, string2long("456t")); + assertEquals(-501377302265856L, string2long("-456T")); + assertEquals(11258999068426240L, string2long("10p")); + assertEquals(-11258999068426240L, string2long("-10P")); + assertEquals(1152921504606846976L, string2long("1e")); + assertEquals(-1152921504606846976L, string2long("-1E")); String tooLargeNumStr = "10e"; try { - StringUtils.TraditionalBinaryPrefix.string2long(tooLargeNumStr); + string2long(tooLargeNumStr); fail("Test passed for a number " + tooLargeNumStr + " too large"); } catch (IllegalArgumentException e) { assertEquals(tooLargeNumStr + " does not fit in a Long", e.getMessage()); @@ -180,7 +172,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { String tooSmallNumStr = "-10e"; try { - StringUtils.TraditionalBinaryPrefix.string2long(tooSmallNumStr); + string2long(tooSmallNumStr); fail("Test passed for a number " + tooSmallNumStr + " too small"); } catch (IllegalArgumentException e) { assertEquals(tooSmallNumStr + " does not fit in a Long", e.getMessage()); @@ -189,7 +181,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { String invalidFormatNumStr = "10kb"; char invalidPrefix = 'b'; try { - StringUtils.TraditionalBinaryPrefix.string2long(invalidFormatNumStr); + string2long(invalidFormatNumStr); fail("Test passed for a number " + invalidFormatNumStr + " has invalid format"); } catch (IllegalArgumentException e) { @@ -199,6 +191,74 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { e.getMessage()); } + //test long2string(..) + assertEquals("0", long2String(0, null, 2)); + for(int decimalPlace = 0; decimalPlace < 2; decimalPlace++) { + for(int n = 1; n < TraditionalBinaryPrefix.KILO.value; n++) { + assertEquals(n + "", long2String(n, null, decimalPlace)); + assertEquals(-n + "", long2String(-n, null, decimalPlace)); + } + assertEquals("1 K", long2String(1L << 10, null, decimalPlace)); + assertEquals("-1 K", long2String(-1L << 10, null, decimalPlace)); + } + + assertEquals("8.00 E", long2String(Long.MAX_VALUE, null, 2)); + assertEquals("8.00 E", long2String(Long.MAX_VALUE - 1, null, 2)); + assertEquals("-8 E", long2String(Long.MIN_VALUE, null, 2)); + assertEquals("-8.00 E", long2String(Long.MIN_VALUE + 1, null, 2)); + + final String[] zeros = {" ", ".0 ", ".00 "}; + for(int decimalPlace = 0; decimalPlace < zeros.length; decimalPlace++) { + final String trailingZeros = zeros[decimalPlace]; + + for(int e = 11; e < Long.SIZE - 1; e++) { + final TraditionalBinaryPrefix p + = TraditionalBinaryPrefix.values()[e/10 - 1]; + + { // n = 2^e + final long n = 1L << e; + final String expected = (n/p.value) + " " + p.symbol; + assertEquals("n=" + n, expected, long2String(n, null, 2)); + } + + { // n = 2^e + 1 + final long n = (1L << e) + 1; + final String expected = (n/p.value) + trailingZeros + p.symbol; + assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); + } + + { // n = 2^e - 1 + final long n = (1L << e) - 1; + final String expected = ((n+1)/p.value) + trailingZeros + p.symbol; + assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); + } + } + } + + assertEquals("1.50 K", long2String(3L << 9, null, 2)); + assertEquals("1.5 K", long2String(3L << 9, null, 1)); + assertEquals("1.50 M", long2String(3L << 19, null, 2)); + assertEquals("2 M", long2String(3L << 19, null, 0)); + assertEquals("3 G", long2String(3L << 30, null, 2)); + + // test byteDesc(..) + assertEquals("0 B", StringUtils.byteDesc(0)); + assertEquals("-100 B", StringUtils.byteDesc(-100)); + assertEquals("1 KB", StringUtils.byteDesc(1024)); + assertEquals("1.50 KB", StringUtils.byteDesc(3L << 9)); + assertEquals("1.50 MB", StringUtils.byteDesc(3L << 19)); + assertEquals("3 GB", StringUtils.byteDesc(3L << 30)); + + // test formatPercent(..) + assertEquals("10%", StringUtils.formatPercent(0.1, 0)); + assertEquals("10.0%", StringUtils.formatPercent(0.1, 1)); + assertEquals("10.00%", StringUtils.formatPercent(0.1, 2)); + + assertEquals("1%", StringUtils.formatPercent(0.00543, 0)); + assertEquals("0.5%", StringUtils.formatPercent(0.00543, 1)); + assertEquals("0.54%", StringUtils.formatPercent(0.00543, 2)); + assertEquals("0.543%", StringUtils.formatPercent(0.00543, 3)); + assertEquals("0.5430%", StringUtils.formatPercent(0.00543, 4)); } @Test @@ -314,10 +374,9 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { } long et = System.nanoTime(); if (outer > 3) { - System.out.println( - (useOurs ? "StringUtils impl" : "Java impl") + - " #" + outer + ":" + - (et - st)/1000000 + "ms"); + System.out.println( (useOurs ? "StringUtils impl" : "Java impl") + + " #" + outer + ":" + (et - st)/1000000 + "ms, components=" + + components ); } } } From 5d679c4f43494eb4508edebc6cc5aa4ddc5a63d2 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 5 Feb 2013 00:26:16 +0000 Subject: [PATCH 22/69] YARN-360. Allow apps to concurrently register tokens for renewal. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442441 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../dev-support/findbugs-exclude.xml | 5 ++ .../security/DelegationTokenRenewer.java | 2 +- .../security/TestDelegationTokenRenewer.java | 58 +++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 286a7ed2752..a1c15797c3f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -107,6 +107,9 @@ Release 2.0.3-alpha - Unreleased YARN-277. Use AMRMClient in DistributedShell to exemplify the approach. (Bikas Saha via hitesh) + YARN-360. Allow apps to concurrently register tokens for renewal. + (Daryn Sharp via sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index e00bc4dde7b..bee8ce24e32 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -167,6 +167,11 @@ + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 9232190ba3b..066a0a5b969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -261,7 +261,7 @@ public class DelegationTokenRenewer extends AbstractService { * done else false. * @throws IOException */ - public synchronized void addApplication( + public void addApplication( ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd) throws IOException { if (ts == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index ad127a9264d..c59625361ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -21,11 +21,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,6 +56,8 @@ import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; /** * unit test - @@ -541,4 +549,54 @@ public class TestDelegationTokenRenewer { fail("Renewal of cancelled token should have failed"); } catch (InvalidToken ite) {} } + + @Test(timeout=2000) + public void testConncurrentAddApplication() + throws IOException, InterruptedException, BrokenBarrierException { + final CyclicBarrier startBarrier = new CyclicBarrier(2); + final CyclicBarrier endBarrier = new CyclicBarrier(2); + + // this token uses barriers to block during renew + final Credentials creds1 = new Credentials(); + final Token token1 = mock(Token.class); + creds1.addToken(new Text("token"), token1); + doReturn(true).when(token1).isManaged(); + doAnswer(new Answer() { + public Long answer(InvocationOnMock invocation) + throws InterruptedException, BrokenBarrierException { + startBarrier.await(); + endBarrier.await(); + return Long.MAX_VALUE; + }}).when(token1).renew(any(Configuration.class)); + + // this dummy token fakes renewing + final Credentials creds2 = new Credentials(); + final Token token2 = mock(Token.class); + creds2.addToken(new Text("token"), token2); + doReturn(true).when(token2).isManaged(); + doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class)); + + // fire up the renewer + final DelegationTokenRenewer dtr = new DelegationTokenRenewer(); + dtr.init(conf); + dtr.start(); + + // submit a job that blocks during renewal + Thread submitThread = new Thread() { + @Override + public void run() { + try { + dtr.addApplication(mock(ApplicationId.class), creds1, false); + } catch (IOException e) {} + } + }; + submitThread.start(); + + // wait till 1st submit blocks, then submit another + startBarrier.await(); + dtr.addApplication(mock(ApplicationId.class), creds2, false); + // signal 1st to complete + endBarrier.await(); + submitThread.join(); + } } From 5a0b74663951475f668c7c68b9902c2bb54dc861 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 5 Feb 2013 03:18:14 +0000 Subject: [PATCH 23/69] HDFS-4404. Create file failure when the machine of first attempted NameNode is down. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442461 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/io/retry/RetryPolicies.java | 2 + .../java/org/apache/hadoop/ipc/Client.java | 9 +-- .../hadoop/net/ConnectTimeoutException.java | 37 ++++++++++ .../java/org/apache/hadoop/net/NetUtils.java | 50 +++++++++----- .../java/org/apache/hadoop/ipc/TestIPC.java | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/TestDFSClientFailover.java | 69 +++++++++++++++++++ 7 files changed, 151 insertions(+), 22 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ConnectTimeoutException.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index 5c29a33312d..9d99c47b32f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -35,6 +35,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; +import org.apache.hadoop.net.ConnectTimeoutException; /** *

@@ -543,6 +544,7 @@ public class RetryPolicies { e instanceof NoRouteToHostException || e instanceof UnknownHostException || e instanceof StandbyException || + e instanceof ConnectTimeoutException || isWrappedStandbyException(e)) { return new RetryAction( RetryAction.RetryDecision.FAILOVER_AND_RETRY, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 36ea776b777..24a84c35661 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -67,6 +67,7 @@ import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto; import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto; import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto; import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto; +import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SaslRpcClient; @@ -511,14 +512,14 @@ public class Client { } this.socket.setSoTimeout(pingInterval); return; - } catch (SocketTimeoutException toe) { + } catch (ConnectTimeoutException toe) { /* Check for an address change and update the local reference. * Reset the failure counter if the address was changed */ if (updateAddress()) { timeoutFailures = ioFailures = 0; } - handleConnectionFailure(timeoutFailures++, + handleConnectionTimeout(timeoutFailures++, maxRetriesOnSocketTimeouts, toe); } catch (IOException ie) { if (updateAddress()) { @@ -680,7 +681,7 @@ public class Client { socket = null; } - /* Handle connection failures + /* Handle connection failures due to timeout on connect * * If the current number of retries is equal to the max number of retries, * stop retrying and throw the exception; Otherwise backoff 1 second and @@ -694,7 +695,7 @@ public class Client { * @param ioe failure reason * @throws IOException if max number of retries is reached */ - private void handleConnectionFailure( + private void handleConnectionTimeout( int curRetries, int maxRetries, IOException ioe) throws IOException { closeConnection(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ConnectTimeoutException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ConnectTimeoutException.java new file mode 100644 index 00000000000..12d7d175906 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ConnectTimeoutException.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import java.net.SocketTimeoutException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Thrown by {@link NetUtils#connect(java.net.Socket, java.net.SocketAddress, int)} + * if it times out while connecting to the remote host. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class ConnectTimeoutException extends SocketTimeoutException { + private static final long serialVersionUID = 1L; + + public ConnectTimeoutException(String msg) { + super(msg); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index 29d8af9fd7f..7ff9030f94a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.net; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -517,11 +518,15 @@ public class NetUtils { socket.bind(localAddr); } - if (ch == null) { - // let the default implementation handle it. - socket.connect(endpoint, timeout); - } else { - SocketIOWithTimeout.connect(ch, endpoint, timeout); + try { + if (ch == null) { + // let the default implementation handle it. + socket.connect(endpoint, timeout); + } else { + SocketIOWithTimeout.connect(ch, endpoint, timeout); + } + } catch (SocketTimeoutException ste) { + throw new ConnectTimeoutException(ste.getMessage()); } // There is a very rare case allowed by the TCP specification, such that @@ -719,7 +724,7 @@ public class NetUtils { + see("BindException")); } else if (exception instanceof ConnectException) { // connection refused; include the host:port in the error - return (ConnectException) new ConnectException( + return wrapWithMessage(exception, "Call From " + localHost + " to " @@ -729,32 +734,28 @@ public class NetUtils { + " failed on connection exception: " + exception + ";" - + see("ConnectionRefused")) - .initCause(exception); + + see("ConnectionRefused")); } else if (exception instanceof UnknownHostException) { - return (UnknownHostException) new UnknownHostException( + return wrapWithMessage(exception, "Invalid host name: " + getHostDetailsAsString(destHost, destPort, localHost) + exception + ";" - + see("UnknownHost")) - .initCause(exception); + + see("UnknownHost")); } else if (exception instanceof SocketTimeoutException) { - return (SocketTimeoutException) new SocketTimeoutException( + return wrapWithMessage(exception, "Call From " + localHost + " to " + destHost + ":" + destPort + " failed on socket timeout exception: " + exception + ";" - + see("SocketTimeout")) - .initCause(exception); + + see("SocketTimeout")); } else if (exception instanceof NoRouteToHostException) { - return (NoRouteToHostException) new NoRouteToHostException( + return wrapWithMessage(exception, "No Route to Host from " + localHost + " to " + destHost + ":" + destPort + " failed on socket timeout exception: " + exception + ";" - + see("NoRouteToHost")) - .initCause(exception); + + see("NoRouteToHost")); } else { return (IOException) new IOException("Failed on local exception: " @@ -769,6 +770,21 @@ public class NetUtils { private static String see(final String entry) { return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry; } + + @SuppressWarnings("unchecked") + private static T wrapWithMessage( + T exception, String msg) { + Class clazz = exception.getClass(); + try { + Constructor ctor = clazz.getConstructor(String.class); + Throwable t = ctor.newInstance(msg); + return (T)(t.initCause(exception)); + } catch (Throwable e) { + LOG.warn("Unable to wrap exception of type " + + clazz + ": it has no (String) constructor", e); + return exception; + } + } /** * Get the host details as a string diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 5762b56b9a0..3847bfd0814 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -26,6 +26,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.net.NetUtils; import java.util.Random; @@ -586,7 +587,7 @@ public class TestIPC { private void assertRetriesOnSocketTimeouts(Configuration conf, int maxTimeoutRetries) throws IOException, InterruptedException { SocketFactory mockFactory = Mockito.mock(SocketFactory.class); - doThrow(new SocketTimeoutException()).when(mockFactory).createSocket(); + doThrow(new ConnectTimeoutException("fake")).when(mockFactory).createSocket(); Client client = new Client(IntWritable.class, conf, mockFactory); InetSocketAddress address = new InetSocketAddress("127.0.0.1", 9090); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 86c31dee49c..a241e5e0a48 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -756,6 +756,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a pre-federation version of HDFS. (atm) + HDFS-4404. Create file failure when the machine of first attempted NameNode + is down. (Todd Lipcon via atm) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index a88e8a74edf..86bcae3a0a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -23,22 +23,34 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; import java.net.URI; import java.net.URISyntaxException; +import javax.net.SocketFactory; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.ConnectTimeoutException; +import org.apache.hadoop.net.StandardSocketFactory; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; public class TestDFSClientFailover { @@ -91,6 +103,63 @@ public class TestDFSClientFailover { fs.close(); } + /** + * Test that even a non-idempotent method will properly fail-over if the + * first IPC attempt times out trying to connect. Regression test for + * HDFS-4404. + */ + @Test + public void testFailoverOnConnectTimeout() throws Exception { + conf.setClass(CommonConfigurationKeysPublic.HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, + InjectingSocketFactory.class, SocketFactory.class); + // Set up the InjectingSocketFactory to throw a ConnectTimeoutException + // when connecting to the first NN. + InjectingSocketFactory.portToInjectOn = cluster.getNameNodePort(0); + + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + + // Make the second NN the active one. + cluster.shutdownNameNode(0); + cluster.transitionToActive(1); + + // Call a non-idempotent method, and ensure the failover of the call proceeds + // successfully. + IOUtils.closeStream(fs.create(TEST_FILE)); + } + + private static class InjectingSocketFactory extends StandardSocketFactory { + + static SocketFactory defaultFactory = SocketFactory.getDefault(); + + static int portToInjectOn; + + @Override + public Socket createSocket() throws IOException { + Socket spy = Mockito.spy(defaultFactory.createSocket()); + // Simplify our spying job by not having to also spy on the channel + Mockito.doReturn(null).when(spy).getChannel(); + // Throw a ConnectTimeoutException when connecting to our target "bad" + // host. + Mockito.doThrow(new ConnectTimeoutException("injected")) + .when(spy).connect( + Mockito.argThat(new MatchesPort()), + Mockito.anyInt()); + return spy; + } + + private class MatchesPort extends BaseMatcher { + @Override + public boolean matches(Object arg0) { + return ((InetSocketAddress)arg0).getPort() == portToInjectOn; + } + + @Override + public void describeTo(Description desc) { + desc.appendText("matches port " + portToInjectOn); + } + } + } + /** * Regression test for HDFS-2683. */ From ef8dd606aba790a097f499dcd3bd129d385a961f Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 5 Feb 2013 04:07:31 +0000 Subject: [PATCH 24/69] HDFS-4344. dfshealth.jsp throws NumberFormatException when dfs.hosts/dfs.hosts.exclude includes port number. Contributed by Andy Isaacson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442465 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../blockmanagement/DatanodeManager.java | 2 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 + .../hdfs/server/namenode/TestHostsFiles.java | 134 ++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a241e5e0a48..d86404af6c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -759,6 +759,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4404. Create file failure when the machine of first attempted NameNode is down. (Todd Lipcon via atm) + HDFS-4344. dfshealth.jsp throws NumberFormatException when + dfs.hosts/dfs.hosts.exclude includes port number. (Andy Isaacson via atm) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 178c25b733d..e4b9ffb8dd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -959,7 +959,7 @@ public class DatanodeManager { port = DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT; } else { hostStr = hostLine.substring(0, idx); - port = Integer.valueOf(hostLine.substring(idx)); + port = Integer.valueOf(hostLine.substring(idx+1)); } if (InetAddresses.isInetAddress(hostStr)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 3e1451ce651..a17bffad366 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -33,6 +33,7 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Socket; import java.net.URL; @@ -620,6 +621,9 @@ public class DFSTestUtil { */ public static byte[] urlGetBytes(URL url) throws IOException { URLConnection conn = url.openConnection(); + HttpURLConnection hc = (HttpURLConnection)conn; + + assertEquals(HttpURLConnection.HTTP_OK, hc.getResponseCode()); ByteArrayOutputStream out = new ByteArrayOutputStream(); IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); return out.toByteArray(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java new file mode 100644 index 00000000000..2a0ab2003b8 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.junit.Assert.assertTrue; + +import java.net.InetSocketAddress; +import java.net.URL; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.junit.Test; + +/** + * DFS_HOSTS and DFS_HOSTS_EXCLUDE tests + * + */ +public class TestHostsFiles { + private static final Log LOG = + LogFactory.getLog(TestHostsFiles.class.getName()); + + /* + * Return a configuration object with low timeouts for testing and + * a topology script set (which enables rack awareness). + */ + private Configuration getConf() { + Configuration conf = new HdfsConfiguration(); + + // Lower the heart beat interval so the NN quickly learns of dead + // or decommissioned DNs and the NN issues replication and invalidation + // commands quickly (as replies to heartbeats) + conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); + + // Have the NN ReplicationMonitor compute the replication and + // invalidation commands to send DNs every second. + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + + // Have the NN check for pending replications every second so it + // quickly schedules additional replicas as they are identified. + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 1); + + // The DNs report blocks every second. + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); + + // Indicates we have multiple racks + conf.set(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, "xyz"); + return conf; + } + + @Test + public void testHostsExcludeDfshealthJsp() throws Exception { + Configuration conf = getConf(); + short REPLICATION_FACTOR = 2; + final Path filePath = new Path("/testFile"); + + // Configure an excludes file + FileSystem localFileSys = FileSystem.getLocal(conf); + Path workingDir = localFileSys.getWorkingDirectory(); + Path dir = new Path(workingDir, "build/test/data/temp/decommission"); + Path excludeFile = new Path(dir, "exclude"); + Path includeFile = new Path(dir, "include"); + assertTrue(localFileSys.mkdirs(dir)); + DFSTestUtil.writeFile(localFileSys, excludeFile, ""); + DFSTestUtil.writeFile(localFileSys, includeFile, ""); + conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); + conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); + + // Two blocks and four racks + String racks[] = {"/rack1", "/rack1", "/rack2", "/rack2"}; + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(racks.length).racks(racks).build(); + final FSNamesystem ns = cluster.getNameNode().getNamesystem(); + + try { + // Create a file with one block + final FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); + ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath); + DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); + + // Decommission one of the hosts with the block, this should cause + // the block to get replicated to another host on the same rack, + // otherwise the rack policy is violated. + BlockLocation locs[] = fs.getFileBlockLocations( + fs.getFileStatus(filePath), 0, Long.MAX_VALUE); + String name = locs[0].getNames()[0]; + String names = name + "\n" + "localhost:42\n"; + LOG.info("adding '" + names + "' to exclude file " + excludeFile.toUri().getPath()); + DFSTestUtil.writeFile(localFileSys, excludeFile, name); + ns.getBlockManager().getDatanodeManager().refreshNodes(conf); + DFSTestUtil.waitForDecommission(fs, name); + + // Check the block still has sufficient # replicas across racks + DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); + + InetSocketAddress nnHttpAddress = cluster.getNameNode().getHttpAddress(); + LOG.info("nnaddr = '" + nnHttpAddress + "'"); + URL nnjsp = new URL("http://" + nnHttpAddress.getHostName() + ":" + nnHttpAddress.getPort() + "/dfshealth.jsp"); + LOG.info("fetching " + nnjsp); + String dfshealthPage = StringEscapeUtils.unescapeHtml(DFSTestUtil.urlGet(nnjsp)); + LOG.info("got " + dfshealthPage); + assertTrue("dfshealth should contain localhost, got:" + dfshealthPage, + dfshealthPage.contains("localhost")); + + } finally { + cluster.shutdown(); + } + } +} From bdb5342b2dee3ee86d40dc093844725f2afb2abb Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 5 Feb 2013 04:53:32 +0000 Subject: [PATCH 25/69] MAPREDUCE-4953. HadoopPipes misuses fprintf. Contributed by Andy Isaacson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442471 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e09203afb72..ec76f222633 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -287,6 +287,8 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7. (Arpit Agarwal via suresh) + MAPREDUCE-4953. HadoopPipes misuses fprintf. (Andy Isaacson via atm) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc index 1d2fc953a46..64c845456b4 100644 --- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc +++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc @@ -127,7 +127,7 @@ namespace HadoopPipes { static const char lineSeparator = '\n'; void writeBuffer(const string& buffer) { - fprintf(stream, quoteString(buffer, "\t\n").c_str()); + fputs(quoteString(buffer, "\t\n").c_str(), stream); } public: From 96398870b4a08ed260d9a14ec346cf1c7d36b211 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Tue, 5 Feb 2013 16:02:52 +0000 Subject: [PATCH 26/69] HADOOP-9260. Hadoop version may be not correct when starting name node or data node. Contributed by Chris Nauroth git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442639 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/main/resources/assemblies/hadoop-dist.xml | 10 ++++++++-- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml index 8db090062eb..4d93b11a043 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml @@ -91,13 +91,19 @@ ${project.artifactId}-${project.version}.jar ${project.artifactId}-${project.version}-tests.jar - ${project.artifactId}-${project.version}-sources.jar - ${project.artifactId}-${project.version}-test-sources.jar hadoop-tools-dist-*.jar + + ${project.build.directory} + /share/hadoop/${hadoop.component}/sources + + ${project.artifactId}-${project.version}-sources.jar + ${project.artifactId}-${project.version}-test-sources.jar + + ${basedir}/dev-support/jdiff /share/hadoop/${hadoop.component}/jdiff diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 125e9f83b77..aa9e00b7a20 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -602,6 +602,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and the synchronization of limitDecimalTo2(double) can be avoided. (szetszwo) + HADOOP-9260. Hadoop version may be not correct when starting name node or + data node. (Chris Nauroth via jlowe) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES From 0b565a967d7abb1bdab9827f1209118bf17e4471 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 5 Feb 2013 21:23:29 +0000 Subject: [PATCH 27/69] HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in HarFileSystem. Contributed by Chris Nauroth git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442755 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/HarFileSystem.java | 119 +++++++++--------- .../org/apache/hadoop/util/LineReader.java | 3 +- .../hadoop/fs/TestHarFileSystemBasics.java | 15 ++- 4 files changed, 81 insertions(+), 59 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aa9e00b7a20..f01eea5b71e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -605,6 +605,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9260. Hadoop version may be not correct when starting name node or data node. (Chris Nauroth via jlowe) + HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in + HarFileSystem. (Chris Nauroth via szetszwo) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 1f5a2a12612..a82fba7068f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -30,8 +30,11 @@ import java.util.TreeMap; import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.util.LineReader; import org.apache.hadoop.util.Progressable; @@ -50,6 +53,9 @@ import org.apache.hadoop.util.Progressable; */ public class HarFileSystem extends FilterFileSystem { + + private static final Log LOG = LogFactory.getLog(HarFileSystem.class); + public static final int VERSION = 3; private static final Map harMetaCache = @@ -1025,68 +1031,69 @@ public class HarFileSystem extends FilterFileSystem { } private void parseMetaData() throws IOException { - FSDataInputStream in = fs.open(masterIndexPath); - FileStatus masterStat = fs.getFileStatus(masterIndexPath); - masterIndexTimestamp = masterStat.getModificationTime(); - LineReader lin = new LineReader(in, getConf()); - Text line = new Text(); - long read = lin.readLine(line); + Text line; + long read; + FSDataInputStream in = null; + LineReader lin = null; - // the first line contains the version of the index file - String versionLine = line.toString(); - String[] arr = versionLine.split(" "); - version = Integer.parseInt(arr[0]); - // make it always backwards-compatible - if (this.version > HarFileSystem.VERSION) { - throw new IOException("Invalid version " + - this.version + " expected " + HarFileSystem.VERSION); - } - - // each line contains a hashcode range and the index file name - String[] readStr = null; - while(read < masterStat.getLen()) { - int b = lin.readLine(line); - read += b; - readStr = line.toString().split(" "); - int startHash = Integer.parseInt(readStr[0]); - int endHash = Integer.parseInt(readStr[1]); - stores.add(new Store(Long.parseLong(readStr[2]), - Long.parseLong(readStr[3]), startHash, - endHash)); - line.clear(); - } try { - // close the master index - lin.close(); - } catch(IOException io){ - // do nothing just a read. + in = fs.open(masterIndexPath); + FileStatus masterStat = fs.getFileStatus(masterIndexPath); + masterIndexTimestamp = masterStat.getModificationTime(); + lin = new LineReader(in, getConf()); + line = new Text(); + read = lin.readLine(line); + + // the first line contains the version of the index file + String versionLine = line.toString(); + String[] arr = versionLine.split(" "); + version = Integer.parseInt(arr[0]); + // make it always backwards-compatible + if (this.version > HarFileSystem.VERSION) { + throw new IOException("Invalid version " + + this.version + " expected " + HarFileSystem.VERSION); + } + + // each line contains a hashcode range and the index file name + String[] readStr = null; + while(read < masterStat.getLen()) { + int b = lin.readLine(line); + read += b; + readStr = line.toString().split(" "); + int startHash = Integer.parseInt(readStr[0]); + int endHash = Integer.parseInt(readStr[1]); + stores.add(new Store(Long.parseLong(readStr[2]), + Long.parseLong(readStr[3]), startHash, + endHash)); + line.clear(); + } + } finally { + IOUtils.cleanup(LOG, lin, in); } FSDataInputStream aIn = fs.open(archiveIndexPath); - FileStatus archiveStat = fs.getFileStatus(archiveIndexPath); - archiveIndexTimestamp = archiveStat.getModificationTime(); - LineReader aLin; - - // now start reading the real index file - for (Store s: stores) { - read = 0; - aIn.seek(s.begin); - aLin = new LineReader(aIn, getConf()); - while (read + s.begin < s.end) { - int tmp = aLin.readLine(line); - read += tmp; - String lineFeed = line.toString(); - String[] parsed = lineFeed.split(" "); - parsed[0] = decodeFileName(parsed[0]); - archive.put(new Path(parsed[0]), new HarStatus(lineFeed)); - line.clear(); - } - } try { - // close the archive index - aIn.close(); - } catch(IOException io) { - // do nothing just a read. + FileStatus archiveStat = fs.getFileStatus(archiveIndexPath); + archiveIndexTimestamp = archiveStat.getModificationTime(); + LineReader aLin; + + // now start reading the real index file + for (Store s: stores) { + read = 0; + aIn.seek(s.begin); + aLin = new LineReader(aIn, getConf()); + while (read + s.begin < s.end) { + int tmp = aLin.readLine(line); + read += tmp; + String lineFeed = line.toString(); + String[] parsed = lineFeed.split(" "); + parsed[0] = decodeFileName(parsed[0]); + archive.put(new Path(parsed[0]), new HarStatus(lineFeed)); + line.clear(); + } + } + } finally { + IOUtils.cleanup(LOG, aIn); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java index 1681d6dfe70..682322dc640 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.util; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -39,7 +40,7 @@ import org.apache.hadoop.io.Text; */ @InterfaceAudience.LimitedPrivate({"MapReduce"}) @InterfaceStability.Unstable -public class LineReader { +public class LineReader implements Closeable { private static final int DEFAULT_BUFFER_SIZE = 64 * 1024; private int bufferSize = DEFAULT_BUFFER_SIZE; private InputStream in; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java index c11afbadd27..de7c3fd1745 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java @@ -28,6 +28,7 @@ import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -46,8 +47,18 @@ public class TestHarFileSystemBasics { private static final String ROOT_PATH = System.getProperty("test.build.data", "build/test/data"); - private static final Path rootPath = new Path( - new File(ROOT_PATH).getAbsolutePath() + "/localfs"); + private static final Path rootPath; + static { + String root = new Path(new File(ROOT_PATH).getAbsolutePath(), "localfs") + .toUri().getPath(); + // Strip drive specifier on Windows, which would make the HAR URI invalid and + // cause tests to fail. + if (Shell.WINDOWS) { + root = root.substring(root.indexOf(':') + 1); + } + rootPath = new Path(root); + } + // NB: .har suffix is necessary private static final Path harPath = new Path(rootPath, "path1/path2/my.har"); From e28edbffe15e9d176d14ea2af8d9460d807b3fc4 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 6 Feb 2013 01:13:16 +0000 Subject: [PATCH 28/69] HDFS-4468. Use the new StringUtils methods added by HADOOP-9252 and fix TestHDFSCLI and TestQuota. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442824 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 5 ++ .../protocol/DSQuotaExceededException.java | 8 ++-- .../hadoop/hdfs/protocol/DatanodeInfo.java | 9 ++-- .../server/namenode/ClusterJspHelper.java | 6 +-- .../server/namenode/NamenodeJspHelper.java | 46 +++++++++++-------- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 3 +- .../org/apache/hadoop/hdfs/TestQuota.java | 4 +- .../src/test/resources/testHDFSConf.xml | 4 +- 9 files changed, 52 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d86404af6c1..fa6a8fd951a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -762,6 +762,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4344. dfshealth.jsp throws NumberFormatException when dfs.hosts/dfs.hosts.exclude includes port number. (Andy Isaacson via atm) + HDFS-4468. Use the new StringUtils methods added by HADOOP-9252 and fix + TestHDFSCLI and TestQuota. (szetszwo) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 7eaff61611e..39f839246fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -925,6 +925,11 @@ public class DFSUtil { return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; } + /** Convert percentage to a string. */ + public static String percent2String(double percentage) { + return StringUtils.format("%.2f%%", percentage); + } + /** * Round bytes to GiB (gibibyte) * @param bytes number of bytes diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java index c7b22f7ac24..481c1305e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String; @InterfaceAudience.Private @InterfaceStability.Evolving @@ -41,9 +41,9 @@ public class DSQuotaExceededException extends QuotaExceededException { public String getMessage() { String msg = super.getMessage(); if (msg == null) { - return "The DiskSpace quota" + (pathName==null?"":(" of " + pathName)) + - " is exceeded: quota=" + StringUtils.humanReadableInt(quota) + - " diskspace consumed=" + StringUtils.humanReadableInt(count); + return "The DiskSpace quota" + (pathName==null?"": " of " + pathName) + + " is exceeded: quota = " + quota + " B = " + long2String(quota, "B", 2) + + " but diskspace consumed = " + count + " B = " + long2String(count, "B", 2); } else { return msg; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index cf7438a7ed5..df46cd6a0c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hdfs.protocol; +import static org.apache.hadoop.hdfs.DFSUtil.percent2String; + import java.util.Date; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; @@ -244,8 +247,8 @@ public class DatanodeInfo extends DatanodeID implements Node { buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n"); buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n"); buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n"); - buffer.append("DFS Used%: "+StringUtils.limitDecimalTo2(usedPercent)+"%\n"); - buffer.append("DFS Remaining%: "+StringUtils.limitDecimalTo2(remainingPercent)+"%\n"); + buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n"); + buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n"); buffer.append("Last contact: "+new Date(lastUpdate)+"\n"); return buffer.toString(); } @@ -269,7 +272,7 @@ public class DatanodeInfo extends DatanodeID implements Node { } buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")"); buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")"); - buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%"); + buffer.append(" " + percent2String(u/(double)c)); buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")"); buffer.append(" " + new Date(lastUpdate)); return buffer.toString(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java index 1b3db818d15..0f0a989f8c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java @@ -569,12 +569,10 @@ class ClusterJspHelper { toXmlItemBlock(doc, "DFS Remaining", StringUtils.byteDesc(free)); // dfsUsedPercent - toXmlItemBlock(doc, "DFS Used%", - StringUtils.limitDecimalTo2(dfsUsedPercent)+ "%"); + toXmlItemBlock(doc, "DFS Used%", DFSUtil.percent2String(dfsUsedPercent)); // dfsRemainingPercent - toXmlItemBlock(doc, "DFS Remaining%", - StringUtils.limitDecimalTo2(dfsRemainingPercent) + "%"); + toXmlItemBlock(doc, "DFS Remaining%", DFSUtil.percent2String(dfsRemainingPercent)); doc.endTag(); // storage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index c4ae5d7756e..005ba6a51ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.DFSUtil.percent2String; + import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -64,6 +66,14 @@ import org.znerd.xmlenc.XMLOutputter; import com.google.common.base.Preconditions; class NamenodeJspHelper { + static String fraction2String(double value) { + return StringUtils.format("%.2f", value); + } + + static String fraction2String(long numerator, long denominator) { + return fraction2String(numerator/(double)denominator); + } + static String getSafeModeText(FSNamesystem fsn) { if (!fsn.isInSafeMode()) return ""; @@ -361,20 +371,20 @@ class NamenodeJspHelper { + "DFS Remaining" + colTxt() + ":" + colTxt() + StringUtils.byteDesc(remaining) + rowTxt() + colTxt() + "DFS Used%" + colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentUsed) + " %" + rowTxt() + + percent2String(percentUsed) + rowTxt() + colTxt() + "DFS Remaining%" + colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentRemaining) + " %" + + percent2String(percentRemaining) + rowTxt() + colTxt() + "Block Pool Used" + colTxt() + ":" + colTxt() + StringUtils.byteDesc(bpUsed) + rowTxt() + colTxt() + "Block Pool Used%"+ colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentBpUsed) + " %" + + percent2String(percentBpUsed) + rowTxt() + colTxt() + "DataNodes usages" + colTxt() + ":" + colTxt() + "Min %" + colTxt() + "Median %" + colTxt() + "Max %" + colTxt() + "stdev %" + rowTxt() + colTxt() + colTxt() + colTxt() - + StringUtils.limitDecimalTo2(min) + " %" - + colTxt() + StringUtils.limitDecimalTo2(median) + " %" - + colTxt() + StringUtils.limitDecimalTo2(max) + " %" - + colTxt() + StringUtils.limitDecimalTo2(dev) + " %" + + percent2String(min) + + colTxt() + percent2String(median) + + colTxt() + percent2String(max) + + colTxt() + percent2String(dev) + rowTxt() + colTxt() + "Live Nodes " + colTxt() + ":" + colTxt() + live.size() @@ -562,9 +572,9 @@ class NamenodeJspHelper { long u = d.getDfsUsed(); long nu = d.getNonDfsUsed(); long r = d.getRemaining(); - String percentUsed = StringUtils.limitDecimalTo2(d.getDfsUsedPercent()); - String percentRemaining = StringUtils.limitDecimalTo2(d - .getRemainingPercent()); + final double percentUsedValue = d.getDfsUsedPercent(); + String percentUsed = fraction2String(percentUsedValue); + String percentRemaining = fraction2String(d.getRemainingPercent()); String adminState = d.getAdminState().toString(); @@ -572,32 +582,30 @@ class NamenodeJspHelper { long currentTime = Time.now(); long bpUsed = d.getBlockPoolUsed(); - String percentBpUsed = StringUtils.limitDecimalTo2(d - .getBlockPoolUsedPercent()); + String percentBpUsed = fraction2String(d.getBlockPoolUsedPercent()); out.print(" " + ((currentTime - timestamp) / 1000) + "" + adminState + "" - + StringUtils.limitDecimalTo2(c * 1.0 / diskBytes) + + fraction2String(c, diskBytes) + "" - + StringUtils.limitDecimalTo2(u * 1.0 / diskBytes) + + fraction2String(u, diskBytes) + "" - + StringUtils.limitDecimalTo2(nu * 1.0 / diskBytes) + + fraction2String(nu, diskBytes) + "" - + StringUtils.limitDecimalTo2(r * 1.0 / diskBytes) + + fraction2String(r, diskBytes) + "" + percentUsed + "" - + ServletUtil.percentageGraph((int) Double.parseDouble(percentUsed), - 100) + + ServletUtil.percentageGraph((int)percentUsedValue, 100) + "" + percentRemaining + "" + d.numBlocks()+"\n" + "" - + StringUtils.limitDecimalTo2(bpUsed * 1.0 / diskBytes) + + fraction2String(bpUsed, diskBytes) + "" + percentBpUsed + "" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index b3c9309054d..7db2b5614ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -316,8 +316,7 @@ public class DFSAdmin extends FsShell { System.out.println("DFS Used: " + used + " (" + StringUtils.byteDesc(used) + ")"); System.out.println("DFS Used%: " - + StringUtils.limitDecimalTo2(((1.0 * used) / presentCapacity) * 100) - + "%"); + + StringUtils.formatPercent(used/(double)presentCapacity, 2)); /* These counts are not always upto date. They are updated after * iteration of an internal list. Should be updated in a few seconds to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java index 600829b1186..0f6d7ada666 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java @@ -70,8 +70,8 @@ public class TestQuota { throw new DSQuotaExceededException(bytes, bytes); } catch(DSQuotaExceededException e) { - assertEquals("The DiskSpace quota is exceeded: quota=1.0k " + - "diskspace consumed=1.0k", e.getMessage()); + assertEquals("The DiskSpace quota is exceeded: quota = 1024 B = 1 KB" + + " but diskspace consumed = 1024 B = 1 KB", e.getMessage()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 81f955a9be8..2fb10837fcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -1182,7 +1182,7 @@ RegexpComparator - ^1\.0k\s+hdfs:///dir0/data1k + ^1\.0 K\s+hdfs:///dir0/data1k @@ -15590,7 +15590,7 @@ RegexpComparator - put: The DiskSpace quota of /dir1 is exceeded: quota=1.0k diskspace consumed=[0-9.]+[kmg]* + put: The DiskSpace quota of /dir1 is exceeded: quota = 1024 B = 1 KB but diskspace consumed = [0-9]+ B = [0-9.]+ [KMG]B* From 38bb4ad73827edd1990fe99e259cd54b6cee42bd Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 6 Feb 2013 04:24:56 +0000 Subject: [PATCH 29/69] YARN-370. Fix SchedulerUtils to correctly round up the resource for containers. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442840 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/SchedulerUtils.java | 3 ++- .../scheduler/TestSchedulerUtils.java | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a1c15797c3f..c7ec79d87a9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -219,6 +219,9 @@ Release 2.0.3-alpha - Unreleased YARN-372. Move InlineDispatcher from hadoop-yarn-server-resourcemanager to hadoop-yarn-common (sseth via hitesh) + YARN-370. Fix SchedulerUtils to correctly round up the resource for + containers. (Zhijie Shen via acmurthy) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index e1bb437763e..f651566d657 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -108,7 +108,8 @@ public class SchedulerUtils { Resource normalized = Resources.normalize( resourceCalculator, ask.getCapability(), minimumResource); - ask.setCapability(normalized); + ask.getCapability().setMemory(normalized.getMemory()); + ask.getCapability().setVirtualCores(normalized.getVirtualCores()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 681e8d01698..bc806f60a2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -42,23 +43,35 @@ public class TestSchedulerUtils { // case negative memory ask.setCapability(Resources.createResource(-1024)); + Resource before = ask.getCapability(); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); + Resource after = ask.getCapability(); assertEquals(minMemory, ask.getCapability().getMemory()); + assertTrue(before == after); // case zero memory ask.setCapability(Resources.createResource(0)); + before = ask.getCapability(); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); + after = ask.getCapability(); assertEquals(minMemory, ask.getCapability().getMemory()); + assertTrue(before == after); // case memory is a multiple of minMemory ask.setCapability(Resources.createResource(2 * minMemory)); + before = ask.getCapability(); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); + after = ask.getCapability(); assertEquals(2 * minMemory, ask.getCapability().getMemory()); + assertTrue(before == after); // case memory is not a multiple of minMemory ask.setCapability(Resources.createResource(minMemory + 10)); + before = ask.getCapability(); SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource); + after = ask.getCapability(); assertEquals(2 * minMemory, ask.getCapability().getMemory()); + assertTrue(before == after); } @@ -73,24 +86,33 @@ public class TestSchedulerUtils { // case negative memory/vcores ask.setCapability(Resources.createResource(-1024, -1)); + Resource before = ask.getCapability(); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource); + Resource after = ask.getCapability(); assertEquals(minResource, ask.getCapability()); + assertTrue(before == after); // case zero memory/vcores ask.setCapability(Resources.createResource(0, 0)); + before = ask.getCapability(); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource); + after = ask.getCapability(); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemory()); + assertTrue(before == after); // case non-zero memory & zero cores ask.setCapability(Resources.createResource(1536, 0)); + before = ask.getCapability(); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource); + after = ask.getCapability(); assertEquals(Resources.createResource(2048, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(2048, ask.getCapability().getMemory()); + assertTrue(before == after); } } From 7cc2050041eb96dbbeed496ffc6ffb24ea88b3ef Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 6 Feb 2013 13:31:52 +0000 Subject: [PATCH 30/69] Preparing for hadoop-2.0.3-alpha release. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1442959 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 14 +++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 14 +++++++++++++- hadoop-mapreduce-project/CHANGES.txt | 14 +++++++++++++- hadoop-yarn-project/CHANGES.txt | 14 +++++++++++++- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f01eea5b71e..ee2d1cf0fd3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -339,7 +339,19 @@ Trunk (Unreleased) HADOOP-9190. packaging docs is broken. (Andy Isaacson via atm) -Release 2.0.3-alpha - Unreleased +Release 2.0.4-beta - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fa6a8fd951a..89f0ceb30d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -296,7 +296,19 @@ Trunk (Unreleased) HDFS-4382. Fix typo MAX_NOT_CHANGED_INTERATIONS. (Ted Yu via suresh) -Release 2.0.3-alpha - Unreleased +Release 2.0.4-beta - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ec76f222633..4853cf26b80 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -152,7 +152,19 @@ Trunk (Unreleased) MAPREDUCE-4884. Streaming tests fail to start MiniMRCluster due to missing queue configuration. (Chris Nauroth via suresh) -Release 2.0.3-alpha - Unreleased +Release 2.0.4-beta - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c7ec79d87a9..47093852697 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -17,7 +17,19 @@ Trunk - Unreleased BUG FIXES -Release 2.0.3-alpha - Unreleased +Release 2.0.4-beta - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES From c52a27768791166540f887aa22c27338c6224502 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 6 Feb 2013 15:19:24 +0000 Subject: [PATCH 31/69] YARN-3. Merged to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443011 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 47093852697..5baa0a60466 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -6,9 +6,6 @@ Trunk - Unreleased NEW FEATURES - YARN-3. Add support for CPU isolation/monitoring of containers. - (adferguson via tucu) - IMPROVEMENTS YARN-84. Use Builder to build RPC server. (Brandon Li via suresh) @@ -37,6 +34,9 @@ Release 2.0.3-alpha - 2013-02-06 YARN-145. Add a Web UI to the fair share scheduler. (Sandy Ryza via tomwhite) + YARN-3. Add support for CPU isolation/monitoring of containers. + (adferguson via tucu) + YARN-230. RM Restart phase 1 - includes support for saving/restarting all applications on an RM bounce. (Bikas Saha via acmurthy) From d27291842605555f6727faa4454211f55da28cca Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Wed, 6 Feb 2013 15:26:30 +0000 Subject: [PATCH 32/69] YARN-357. App submission should not be synchronized (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443016 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../server/resourcemanager/RMAppManager.java | 2 +- .../resourcemanager/TestClientRMService.java | 97 ++++++++++++++++++- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5baa0a60466..1fd179dc1a6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -300,6 +300,8 @@ Release 0.23.7 - UNRELEASED OPTIMIZATIONS + YARN-357. App submission should not be synchronized (daryn) + BUG FIXES YARN-343. Capacity Scheduler maximum-capacity value -1 is invalid (Xuan diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 52b4d2892a4..70fd2576ab0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -228,7 +228,7 @@ public class RMAppManager implements EventHandler, } @SuppressWarnings("unchecked") - protected synchronized void submitApplication( + protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime) { ApplicationId applicationId = submissionContext.getApplicationId(); RMApp application = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 8479c2c87a3..871755c8f5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -27,7 +27,9 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.List; +import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; import junit.framework.Assert; @@ -37,6 +39,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; @@ -44,28 +47,36 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; -import org.junit.Test; import org.junit.AfterClass; import org.junit.BeforeClass; - +import org.junit.Test; public class TestClientRMService { @@ -235,6 +246,88 @@ public class TestClientRMService { rmService.renewDelegationToken(request); } + @Test(timeout=4000) + public void testConcurrentAppSubmit() + throws IOException, InterruptedException, BrokenBarrierException { + YarnScheduler yarnScheduler = mock(YarnScheduler.class); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + RMStateStore stateStore = mock(RMStateStore.class); + when(rmContext.getStateStore()).thenReturn(stateStore); + RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, + null, mock(ApplicationACLsManager.class), new Configuration()); + + final ApplicationId appId1 = getApplicationId(100); + final ApplicationId appId2 = getApplicationId(101); + final SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1); + final SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2); + + final CyclicBarrier startBarrier = new CyclicBarrier(2); + final CyclicBarrier endBarrier = new CyclicBarrier(2); + + @SuppressWarnings("rawtypes") + EventHandler eventHandler = new EventHandler() { + @Override + public void handle(Event rawEvent) { + if (rawEvent instanceof RMAppEvent) { + RMAppEvent event = (RMAppEvent) rawEvent; + if (event.getApplicationId().equals(appId1)) { + try { + startBarrier.await(); + endBarrier.await(); + } catch (BrokenBarrierException e) { + LOG.warn("Broken Barrier", e); + } catch (InterruptedException e) { + LOG.warn("Interrupted while awaiting barriers", e); + } + } + } + } + }; + + when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler); + + final ClientRMService rmService = + new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + + // submit an app and wait for it to block while in app submission + Thread t = new Thread() { + @Override + public void run() { + try { + rmService.submitApplication(submitRequest1); + } catch (YarnRemoteException e) {} + } + }; + t.start(); + + // submit another app, so go through while the first app is blocked + startBarrier.await(); + rmService.submitApplication(submitRequest2); + endBarrier.await(); + t.join(); + } + + private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId) { + String user = MockApps.newUserName(); + String queue = MockApps.newQueue(); + + ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); + Resource resource = mock(Resource.class); + when(amContainerSpec.getResource()).thenReturn(resource); + + ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); + when(submissionContext.getUser()).thenReturn(user); + when(submissionContext.getQueue()).thenReturn(queue); + when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); + when(submissionContext.getApplicationId()).thenReturn(appId); + + SubmitApplicationRequest submitRequest = + recordFactory.newRecordInstance(SubmitApplicationRequest.class); + submitRequest.setApplicationSubmissionContext(submissionContext); + return submitRequest; + } + private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) throws IOException { Dispatcher dispatcher = mock(Dispatcher.class); From 79b12f6f4ccd06a67c65ce9ce0cf8155ecdc6652 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 6 Feb 2013 15:50:23 +0000 Subject: [PATCH 33/69] MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes (Aleksey Gorshkov via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443027 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../apache/hadoop/mapred/pipes/Submitter.java | 3 +- .../hadoop/mapred/pipes/CommonStub.java | 153 ++++ .../pipes/PipeApplicationRunnableStub.java | 87 ++ .../mapred/pipes/PipeApplicationStub.java | 101 +++ .../hadoop/mapred/pipes/PipeReducerStub.java | 80 ++ .../mapred/pipes/TestPipeApplication.java | 747 ++++++++++++++++++ .../pipes/TestPipesNonJavaInputFormat.java | 89 +++ 8 files changed, 1262 insertions(+), 1 deletion(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/CommonStub.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationRunnableStub.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationStub.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeReducerStub.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4853cf26b80..c302e8f63e2 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -685,6 +685,9 @@ Release 0.23.7 - UNRELEASED IMPROVEMENTS + MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes + (Aleksey Gorshkov via bobby) + OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java index 57370872e30..8f4259e3ec1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java @@ -58,6 +58,7 @@ import org.apache.hadoop.mapred.lib.LazyOutputFormat; import org.apache.hadoop.mapred.lib.NullOutputFormat; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; @@ -515,7 +516,7 @@ public class Submitter extends Configured implements Tool { */ public static void main(String[] args) throws Exception { int exitCode = new Submitter().run(args); - System.exit(exitCode); + ExitUtil.terminate(exitCode); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/CommonStub.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/CommonStub.java new file mode 100644 index 00000000000..f1f11a17be4 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/CommonStub.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.pipes; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; + +import javax.crypto.SecretKey; + +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.mapreduce.security.SecureShuffleUtils; +import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; + +public class CommonStub { + + protected Socket socket = null; + protected DataInputStream dataInput; + protected DataOutputStream dataOut; + + protected String createDigest(byte[] password, String data) throws IOException { + SecretKey key = JobTokenSecretManager.createSecretKey(password); + + return SecureShuffleUtils.hashFromString(data, key); + + } + + protected void readObject(Writable obj, DataInputStream inStream) throws IOException { + int numBytes = WritableUtils.readVInt(inStream); + byte[] buffer; + // For BytesWritable and Text, use the specified length to set the length + // this causes the "obvious" translations to work. So that if you emit + // a string "abc" from C++, it shows up as "abc". + if (obj instanceof BytesWritable) { + buffer = new byte[numBytes]; + inStream.readFully(buffer); + ((BytesWritable) obj).set(buffer, 0, numBytes); + } else if (obj instanceof Text) { + buffer = new byte[numBytes]; + inStream.readFully(buffer); + ((Text) obj).set(buffer); + } else { + obj.readFields(inStream); + } + } + + + protected void writeObject(Writable obj, DataOutputStream stream) + throws IOException { + // For Text and BytesWritable, encode them directly, so that they end up + // in C++ as the natural translations. + DataOutputBuffer buffer = new DataOutputBuffer(); + if (obj instanceof Text) { + Text t = (Text) obj; + int len = t.getLength(); + WritableUtils.writeVLong(stream, len); + stream.flush(); + + stream.write(t.getBytes(), 0, len); + stream.flush(); + + } else if (obj instanceof BytesWritable) { + BytesWritable b = (BytesWritable) obj; + int len = b.getLength(); + WritableUtils.writeVLong(stream, len); + stream.write(b.getBytes(), 0, len); + } else { + buffer.reset(); + obj.write(buffer); + int length = buffer.getLength(); + + WritableUtils.writeVInt(stream, length); + stream.write(buffer.getData(), 0, length); + } + stream.flush(); + + } + + protected void initSoket() throws Exception { + int port = Integer.parseInt(System.getenv("mapreduce.pipes.command.port")); + + java.net.InetAddress address = java.net.InetAddress.getLocalHost(); + + socket = new Socket(address.getHostName(), port); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + + // try to read + dataInput = new DataInputStream(input); + + WritableUtils.readVInt(dataInput); + + String str = Text.readString(dataInput); + + Text.readString(dataInput); + + dataOut = new DataOutputStream(output); + WritableUtils.writeVInt(dataOut, 57); + String s = createDigest("password".getBytes(), str); + + Text.writeString(dataOut, s); + + // start + WritableUtils.readVInt(dataInput); + int cuttentAnswer = WritableUtils.readVInt(dataInput); + System.out.println("CURRENT_PROTOCOL_VERSION:" + cuttentAnswer); + + // get configuration + // should be MessageType.SET_JOB_CONF.code + WritableUtils.readVInt(dataInput); + + // array length + + int j = WritableUtils.readVInt(dataInput); + for (int i = 0; i < j; i++) { + Text.readString(dataInput); + i++; + Text.readString(dataInput); + } + } + + protected void closeSoket() { + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationRunnableStub.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationRunnableStub.java new file mode 100644 index 00000000000..c2cc794c56d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationRunnableStub.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.pipes; + + +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; + +/* + Stub for TestPipeApplication test. This stub produced test data for main test. Main test checks data + */ + +public class PipeApplicationRunnableStub extends CommonStub { + + public static void main(String[] args) { + PipeApplicationRunnableStub client = new PipeApplicationRunnableStub(); + client.binaryProtocolStub(); + } + + public void binaryProtocolStub() { + try { + + initSoket(); + System.out.println("start OK"); + + // RUN_MAP.code + // should be 3 + + int answer = WritableUtils.readVInt(dataInput); + System.out.println("RunMap:" + answer); + TestPipeApplication.FakeSplit split = new TestPipeApplication.FakeSplit(); + readObject(split, dataInput); + + WritableUtils.readVInt(dataInput); + WritableUtils.readVInt(dataInput); + // end runMap + // get InputTypes + WritableUtils.readVInt(dataInput); + String inText = Text.readString(dataInput); + System.out.println("Key class:" + inText); + inText = Text.readString(dataInput); + System.out.println("Value class:" + inText); + + @SuppressWarnings("unused") + int inCode = 0; + + // read all data from sender and write to output + while ((inCode = WritableUtils.readVInt(dataInput)) == 4) { + FloatWritable key = new FloatWritable(); + NullWritable value = NullWritable.get(); + readObject(key, dataInput); + System.out.println("value:" + key.get()); + readObject(value, dataInput); + } + + WritableUtils.writeVInt(dataOut, 54); + + dataOut.flush(); + dataOut.close(); + + } catch (Exception x) { + x.printStackTrace(); + } finally { + closeSoket(); + } + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationStub.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationStub.java new file mode 100644 index 00000000000..33e0b813eb3 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeApplicationStub.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.pipes; + + +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; +/* +Stub for TestPipeApplication test. This stub produced test data for main test. Main test checks data + */ + +public class PipeApplicationStub extends CommonStub { + + public static void main(String[] args) { + PipeApplicationStub client = new PipeApplicationStub(); + client.binaryProtocolStub(); + } + + public void binaryProtocolStub() { + try { + + initSoket(); + + // output code + WritableUtils.writeVInt(dataOut, 50); + IntWritable wt = new IntWritable(); + wt.set(123); + writeObject(wt, dataOut); + writeObject(new Text("value"), dataOut); + + // PARTITIONED_OUTPUT + WritableUtils.writeVInt(dataOut, 51); + WritableUtils.writeVInt(dataOut, 0); + writeObject(wt, dataOut); + writeObject(new Text("value"), dataOut); + + + // STATUS + WritableUtils.writeVInt(dataOut, 52); + Text.writeString(dataOut, "PROGRESS"); + dataOut.flush(); + + // progress + WritableUtils.writeVInt(dataOut, 53); + dataOut.writeFloat(0.55f); + // register counter + WritableUtils.writeVInt(dataOut, 55); + // id + WritableUtils.writeVInt(dataOut, 0); + Text.writeString(dataOut, "group"); + Text.writeString(dataOut, "name"); + // increment counter + WritableUtils.writeVInt(dataOut, 56); + WritableUtils.writeVInt(dataOut, 0); + + WritableUtils.writeVLong(dataOut, 2); + + // map item + int intValue = WritableUtils.readVInt(dataInput); + System.out.println("intValue:" + intValue); + IntWritable iw = new IntWritable(); + readObject(iw, dataInput); + System.out.println("key:" + iw.get()); + Text txt = new Text(); + readObject(txt, dataInput); + System.out.println("value:" + txt.toString()); + + // done + // end of session + WritableUtils.writeVInt(dataOut, 54); + + System.out.println("finish"); + dataOut.flush(); + dataOut.close(); + + } catch (Exception x) { + x.printStackTrace(); + } finally { + closeSoket(); + } + } + + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeReducerStub.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeReducerStub.java new file mode 100644 index 00000000000..a0afdbc9477 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/PipeReducerStub.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.pipes; + + +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableUtils; + +/* +Stub for TestPipeApplication test. This stub produced test data for main test. Main test checks data + */ + +public class PipeReducerStub extends CommonStub { + + public static void main(String[] args) { + PipeReducerStub client = new PipeReducerStub(); + client.binaryProtocolStub(); + } + + public void binaryProtocolStub() { + try { + + initSoket(); + + //should be 5 + //RUN_REDUCE boolean + WritableUtils.readVInt(dataInput); + WritableUtils.readVInt(dataInput); + int intValue = WritableUtils.readVInt(dataInput); + System.out.println("getIsJavaRecordWriter:" + intValue); + + // reduce key + WritableUtils.readVInt(dataInput); + // value of reduce key + BooleanWritable value = new BooleanWritable(); + readObject(value, dataInput); + System.out.println("reducer key :" + value); + // reduce value code: + + // reduce values + while ((intValue = WritableUtils.readVInt(dataInput)) == 7) { + Text txt = new Text(); + // value + readObject(txt, dataInput); + System.out.println("reduce value :" + txt); + } + + + // done + WritableUtils.writeVInt(dataOut, 54); + + dataOut.flush(); + dataOut.close(); + + } catch (Exception x) { + x.printStackTrace(); + } finally { + closeSoket(); + + } + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java new file mode 100644 index 00000000000..aa345c42c72 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -0,0 +1,747 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.pipes; + +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.IFile.Writer; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapred.Counters; +import org.apache.hadoop.mapred.Counters.Counter; +import org.apache.hadoop.mapred.Counters.Group; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapred.TaskLog; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestPipeApplication { + private static File workSpace = new File("target", + TestPipeApplication.class.getName() + "-workSpace"); + + private static String taskName = "attempt_001_02_r03_04_05"; + + /** + * test PipesMapRunner test the transfer data from reader + * + * @throws Exception + */ + @Test + public void testRunner() throws Exception { + + // clean old password files + File[] psw = cleanTokenPasswordFile(); + try { + RecordReader rReader = new ReaderPipesMapRunner(); + JobConf conf = new JobConf(); + conf.set(Submitter.IS_JAVA_RR, "true"); + // for stdour and stderror + + conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); + + CombineOutputCollector output = new CombineOutputCollector( + new Counters.Counter(), new Progress()); + FileSystem fs = new RawLocalFileSystem(); + fs.setConf(conf); + Writer wr = new Writer(conf, fs, + new Path(workSpace + File.separator + "outfile"), IntWritable.class, + Text.class, null, null); + output.setWriter(wr); + // stub for client + File fCommand = getFileCommand("org.apache.hadoop.mapred.pipes.PipeApplicationRunnableStub"); + + conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); + // token for authorization + Token token = new Token( + "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( + "service")); + conf.getCredentials().addToken(new Text("ShuffleAndJobToken"), token); + conf.setBoolean(MRJobConfig.SKIP_RECORDS, true); + TestTaskReporter reporter = new TestTaskReporter(); + PipesMapRunner runner = new PipesMapRunner(); + + initStdOut(conf); + + runner.configure(conf); + runner.run(rReader, output, reporter); + + String stdOut = readStdOut(conf); + + // test part of translated data. As common file for client and test - + // clients stdOut + // check version + assertTrue(stdOut.contains("CURRENT_PROTOCOL_VERSION:0")); + // check key and value classes + assertTrue(stdOut + .contains("Key class:org.apache.hadoop.io.FloatWritable")); + assertTrue(stdOut + .contains("Value class:org.apache.hadoop.io.NullWritable")); + // test have sent all data from reader + assertTrue(stdOut.contains("value:0.0")); + assertTrue(stdOut.contains("value:9.0")); + + } finally { + if (psw != null) { + // remove password files + for (File file : psw) { + file.deleteOnExit(); + } + } + + } + } + + /** + * test org.apache.hadoop.mapred.pipes.Application + * test a internal functions: MessageType.REGISTER_COUNTER, INCREMENT_COUNTER, STATUS, PROGRESS... + * + * @throws Throwable + */ + + @Test + public void testApplication() throws Throwable { + JobConf conf = new JobConf(); + + RecordReader rReader = new Reader(); + + // client for test + File fCommand = getFileCommand("org.apache.hadoop.mapred.pipes.PipeApplicationStub"); + + TestTaskReporter reporter = new TestTaskReporter(); + + File[] psw = cleanTokenPasswordFile(); + try { + + conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); + conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); + + // token for authorization + Token token = new Token( + "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( + "service")); + + conf.getCredentials().addToken(new Text("ShuffleAndJobToken"), token); + FakeCollector output = new FakeCollector(new Counters.Counter(), + new Progress()); + FileSystem fs = new RawLocalFileSystem(); + fs.setConf(conf); + Writer wr = new Writer(conf, fs, + new Path(workSpace.getAbsolutePath() + File.separator + "outfile"), + IntWritable.class, Text.class, null, null); + output.setWriter(wr); + conf.set(Submitter.PRESERVE_COMMANDFILE, "true"); + + Application, Writable, IntWritable, Text> application = new Application, Writable, IntWritable, Text>( + conf, rReader, output, reporter, IntWritable.class, Text.class); + application.getDownlink().flush(); + + application.getDownlink().mapItem(new IntWritable(3), new Text("txt")); + + application.getDownlink().flush(); + + application.waitForFinish(); + + wr.close(); + + // test getDownlink().mapItem(); + String stdOut = readStdOut(conf); + assertTrue(stdOut.contains("key:3")); + assertTrue(stdOut.contains("value:txt")); + + // reporter test counter, and status should be sended + // test MessageType.REGISTER_COUNTER and INCREMENT_COUNTER + assertEquals(1.0, reporter.getProgress(), 0.01); + assertNotNull(reporter.getCounter("group", "name")); + // test status MessageType.STATUS + assertEquals(reporter.getStatus(), "PROGRESS"); + stdOut = readFile(new File(workSpace.getAbsolutePath() + File.separator + + "outfile")); + // check MessageType.PROGRESS + assertEquals(0.55f, rReader.getProgress(), 0.001); + application.getDownlink().close(); + // test MessageType.OUTPUT + Entry entry = output.getCollect().entrySet() + .iterator().next(); + assertEquals(123, entry.getKey().get()); + assertEquals("value", entry.getValue().toString()); + try { + // try to abort + application.abort(new Throwable()); + fail(); + } catch (IOException e) { + // abort works ? + assertEquals("pipe child exception", e.getMessage()); + } + } finally { + if (psw != null) { + // remove password files + for (File file : psw) { + file.deleteOnExit(); + } + } + } + } + + /** + * test org.apache.hadoop.mapred.pipes.Submitter + * + * @throws Exception + */ + @Test + public void testSubmitter() throws Exception { + + JobConf conf = new JobConf(); + + File[] psw = cleanTokenPasswordFile(); + + System.setProperty("test.build.data", + "target/tmp/build/TEST_SUBMITTER_MAPPER/data"); + conf.set("hadoop.log.dir", "target/tmp"); + + // prepare configuration + Submitter.setIsJavaMapper(conf, false); + Submitter.setIsJavaReducer(conf, false); + Submitter.setKeepCommandFile(conf, false); + Submitter.setIsJavaRecordReader(conf, false); + Submitter.setIsJavaRecordWriter(conf, false); + PipesPartitioner partitioner = new PipesPartitioner(); + partitioner.configure(conf); + + Submitter.setJavaPartitioner(conf, partitioner.getClass()); + + assertEquals(PipesPartitioner.class, (Submitter.getJavaPartitioner(conf))); + // test going to call main method with System.exit(). Change Security + SecurityManager securityManager = System.getSecurityManager(); + // store System.out + PrintStream oldps = System.out; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ExitUtil.disableSystemExit(); + // test without parameters + try { + System.setOut(new PrintStream(out)); + Submitter.main(new String[0]); + fail(); + } catch (ExitUtil.ExitException e) { + // System.exit prohibited! output message test + assertTrue(out.toString().contains("")); + assertTrue(out.toString().contains("bin/hadoop pipes")); + assertTrue(out.toString().contains("[-input ] // Input directory")); + assertTrue(out.toString() + .contains("[-output ] // Output directory")); + assertTrue(out.toString().contains("[-jar // jar filename")); + assertTrue(out.toString().contains( + "[-inputformat ] // InputFormat class")); + assertTrue(out.toString().contains("[-map ] // Java Map class")); + assertTrue(out.toString().contains( + "[-partitioner ] // Java Partitioner")); + assertTrue(out.toString().contains( + "[-reduce ] // Java Reduce class")); + assertTrue(out.toString().contains( + "[-writer ] // Java RecordWriter")); + assertTrue(out.toString().contains( + "[-program ] // executable URI")); + assertTrue(out.toString().contains( + "[-reduces ] // number of reduces")); + assertTrue(out.toString().contains( + "[-lazyOutput ] // createOutputLazily")); + + assertTrue(out + .toString() + .contains( + "-conf specify an application configuration file")); + assertTrue(out.toString().contains( + "-D use value for given property")); + assertTrue(out.toString().contains( + "-fs specify a namenode")); + assertTrue(out.toString().contains( + "-jt specify a job tracker")); + assertTrue(out + .toString() + .contains( + "-files specify comma separated files to be copied to the map reduce cluster")); + assertTrue(out + .toString() + .contains( + "-libjars specify comma separated jar files to include in the classpath.")); + assertTrue(out + .toString() + .contains( + "-archives specify comma separated archives to be unarchived on the compute machines.")); + } finally { + System.setOut(oldps); + // restore + System.setSecurityManager(securityManager); + if (psw != null) { + // remove password files + for (File file : psw) { + file.deleteOnExit(); + } + } + } + // test call Submitter form command line + try { + File fCommand = getFileCommand(null); + String[] args = new String[22]; + File input = new File(workSpace + File.separator + "input"); + if (!input.exists()) { + Assert.assertTrue(input.createNewFile()); + } + File outPut = new File(workSpace + File.separator + "output"); + FileUtil.fullyDelete(outPut); + + args[0] = "-input"; + args[1] = input.getAbsolutePath();// "input"; + args[2] = "-output"; + args[3] = outPut.getAbsolutePath();// "output"; + args[4] = "-inputformat"; + args[5] = "org.apache.hadoop.mapred.TextInputFormat"; + args[6] = "-map"; + args[7] = "org.apache.hadoop.mapred.lib.IdentityMapper"; + args[8] = "-partitioner"; + args[9] = "org.apache.hadoop.mapred.pipes.PipesPartitioner"; + args[10] = "-reduce"; + args[11] = "org.apache.hadoop.mapred.lib.IdentityReducer"; + args[12] = "-writer"; + args[13] = "org.apache.hadoop.mapred.TextOutputFormat"; + args[14] = "-program"; + args[15] = fCommand.getAbsolutePath();// "program"; + args[16] = "-reduces"; + args[17] = "2"; + args[18] = "-lazyOutput"; + args[19] = "lazyOutput"; + args[20] = "-jobconf"; + args[21] = "mapreduce.pipes.isjavarecordwriter=false,mapreduce.pipes.isjavarecordreader=false"; + + Submitter.main(args); + fail(); + } catch (ExitUtil.ExitException e) { + // status should be 0 + assertEquals(e.status, 0); + + } finally { + System.setOut(oldps); + System.setSecurityManager(securityManager); + } + + } + + /** + * test org.apache.hadoop.mapred.pipes.PipesReducer + * test the transfer of data: key and value + * + * @throws Exception + */ + @Test + public void testPipesReduser() throws Exception { + + File[] psw = cleanTokenPasswordFile(); + JobConf conf = new JobConf(); + try { + Token token = new Token( + "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( + "service")); + conf.getCredentials().addToken(new Text("ShuffleAndJobToken"), token); + + File fCommand = getFileCommand("org.apache.hadoop.mapred.pipes.PipeReducerStub"); + conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); + + PipesReducer reducer = new PipesReducer(); + reducer.configure(conf); + BooleanWritable bw = new BooleanWritable(true); + + conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName); + initStdOut(conf); + conf.setBoolean(MRJobConfig.SKIP_RECORDS, true); + CombineOutputCollector output = new CombineOutputCollector( + new Counters.Counter(), new Progress()); + Reporter reporter = new TestTaskReporter(); + List texts = new ArrayList(); + texts.add(new Text("first")); + texts.add(new Text("second")); + texts.add(new Text("third")); + + reducer.reduce(bw, texts.iterator(), output, reporter); + reducer.close(); + String stdOut = readStdOut(conf); + // test data: key + assertTrue(stdOut.contains("reducer key :true")); + // and values + assertTrue(stdOut.contains("reduce value :first")); + assertTrue(stdOut.contains("reduce value :second")); + assertTrue(stdOut.contains("reduce value :third")); + + } finally { + if (psw != null) { + // remove password files + for (File file : psw) { + file.deleteOnExit(); + } + } + } + + } + + /** + * test PipesPartitioner + * test set and get data from PipesPartitioner + */ + @Test + public void testPipesPartitioner() { + + PipesPartitioner partitioner = new PipesPartitioner(); + JobConf configuration = new JobConf(); + Submitter.getJavaPartitioner(configuration); + partitioner.configure(new JobConf()); + IntWritable iw = new IntWritable(4); + // the cache empty + assertEquals(0, partitioner.getPartition(iw, new Text("test"), 2)); + // set data into cache + PipesPartitioner.setNextPartition(3); + // get data from cache + assertEquals(3, partitioner.getPartition(iw, new Text("test"), 2)); + } + + /** + * clean previous std error and outs + */ + + private void initStdOut(JobConf configuration) { + TaskAttemptID taskId = TaskAttemptID.forName(configuration + .get(MRJobConfig.TASK_ATTEMPT_ID)); + File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT); + File stdErr = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDERR); + // prepare folder + if (!stdOut.getParentFile().exists()) { + stdOut.getParentFile().mkdirs(); + } else { // clean logs + stdOut.deleteOnExit(); + stdErr.deleteOnExit(); + } + } + + private String readStdOut(JobConf conf) throws Exception { + TaskAttemptID taskId = TaskAttemptID.forName(conf + .get(MRJobConfig.TASK_ATTEMPT_ID)); + File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT); + + return readFile(stdOut); + + } + + private String readFile(File file) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + InputStream is = new FileInputStream(file); + byte[] buffer = new byte[1024]; + int counter = 0; + while ((counter = is.read(buffer)) >= 0) { + out.write(buffer, 0, counter); + } + + is.close(); + + return out.toString(); + + } + + private class Progress implements Progressable { + + @Override + public void progress() { + + } + + } + + private File[] cleanTokenPasswordFile() throws Exception { + File[] result = new File[2]; + result[0] = new File("./jobTokenPassword"); + if (result[0].exists()) { + FileUtil.chmod(result[0].getAbsolutePath(), "700"); + assertTrue(result[0].delete()); + } + result[1] = new File("./.jobTokenPassword.crc"); + if (result[1].exists()) { + FileUtil.chmod(result[1].getAbsolutePath(), "700"); + result[1].delete(); + } + return result; + } + + private File getFileCommand(String clazz) throws Exception { + String classpath = System.getProperty("java.class.path"); + File fCommand = new File(workSpace + File.separator + "cache.sh"); + fCommand.deleteOnExit(); + if (!fCommand.getParentFile().exists()) { + fCommand.getParentFile().mkdirs(); + } + fCommand.createNewFile(); + OutputStream os = new FileOutputStream(fCommand); + os.write("#!/bin/sh \n".getBytes()); + if (clazz == null) { + os.write(("ls ").getBytes()); + } else { + os.write(("java -cp " + classpath + " " + clazz).getBytes()); + } + os.flush(); + os.close(); + FileUtil.chmod(fCommand.getAbsolutePath(), "700"); + return fCommand; + } + + private class CombineOutputCollector implements + OutputCollector { + private Writer writer; + private Counters.Counter outCounter; + private Progressable progressable; + + public CombineOutputCollector(Counters.Counter outCounter, + Progressable progressable) { + this.outCounter = outCounter; + this.progressable = progressable; + } + + public synchronized void setWriter(Writer writer) { + this.writer = writer; + } + + public synchronized void collect(K key, V value) throws IOException { + outCounter.increment(1); + writer.append(key, value); + progressable.progress(); + } + } + + public static class FakeSplit implements InputSplit { + public void write(DataOutput out) throws IOException { + } + + public void readFields(DataInput in) throws IOException { + } + + public long getLength() { + return 0L; + } + + public String[] getLocations() { + return new String[0]; + } + } + + private class TestTaskReporter implements Reporter { + private int recordNum = 0; // number of records processed + private String status = null; + private Counters counters = new Counters(); + private InputSplit split = new FakeSplit(); + + @Override + public void progress() { + + recordNum++; + } + + @Override + public void setStatus(String status) { + this.status = status; + + } + + public String getStatus() { + return this.status; + + } + + public Counters.Counter getCounter(String group, String name) { + Counters.Counter counter = null; + if (counters != null) { + counter = counters.findCounter(group, name); + if (counter == null) { + Group grp = counters.addGroup(group, group); + counter = grp.addCounter(name, name, 10); + } + } + return counter; + } + + public Counters.Counter getCounter(Enum name) { + return counters == null ? null : counters.findCounter(name); + } + + public void incrCounter(Enum key, long amount) { + if (counters != null) { + counters.incrCounter(key, amount); + } + } + + public void incrCounter(String group, String counter, long amount) { + + if (counters != null) { + counters.incrCounter(group, counter, amount); + } + + } + + @Override + public InputSplit getInputSplit() throws UnsupportedOperationException { + return split; + } + + @Override + public float getProgress() { + return recordNum; + } + + } + + private class Reader implements RecordReader { + private int index = 0; + private FloatWritable progress; + + @Override + public boolean next(FloatWritable key, NullWritable value) + throws IOException { + progress = key; + index++; + return index <= 10; + } + + @Override + public float getProgress() throws IOException { + return progress.get(); + } + + @Override + public long getPos() throws IOException { + + return index; + } + + @Override + public NullWritable createValue() { + + return NullWritable.get(); + } + + @Override + public FloatWritable createKey() { + FloatWritable result = new FloatWritable(index); + return result; + } + + @Override + public void close() throws IOException { + + } + } + + + private class ReaderPipesMapRunner implements RecordReader { + private int index = 0; + + @Override + public boolean next(FloatWritable key, NullWritable value) + throws IOException { + key.set(index++); + return index <= 10; + } + + @Override + public float getProgress() throws IOException { + return index; + } + + @Override + public long getPos() throws IOException { + + return index; + } + + @Override + public NullWritable createValue() { + + return NullWritable.get(); + } + + @Override + public FloatWritable createKey() { + FloatWritable result = new FloatWritable(index); + return result; + } + + @Override + public void close() throws IOException { + + } + } + + private class FakeCollector extends + CombineOutputCollector { + + final private Map collect = new HashMap(); + + public FakeCollector(Counter outCounter, Progressable progressable) { + super(outCounter, progressable); + } + + @Override + public synchronized void collect(IntWritable key, Text value) + throws IOException { + collect.put(key, value); + super.collect(key, value); + } + + public Map getCollect() { + return collect; + } + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java new file mode 100644 index 00000000000..b3277854d18 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipesNonJavaInputFormat.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.pipes; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.pipes.TestPipeApplication.FakeSplit; +import org.junit.Assert; +import org.junit.Test; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + + +public class TestPipesNonJavaInputFormat { + private static File workSpace = new File("target", + TestPipesNonJavaInputFormat.class.getName() + "-workSpace"); + + /** + * test PipesNonJavaInputFormat + */ + + @Test + public void testFormat() throws IOException { + + PipesNonJavaInputFormat inputFormat = new PipesNonJavaInputFormat(); + JobConf conf = new JobConf(); + + Reporter reporter= mock(Reporter.class); + RecordReader reader = inputFormat + .getRecordReader(new FakeSplit(), conf, reporter); + assertEquals(0.0f, reader.getProgress(), 0.001); + + // input and output files + File input1 = new File(workSpace + File.separator + "input1"); + if (!input1.getParentFile().exists()) { + Assert.assertTrue(input1.getParentFile().mkdirs()); + } + + if (!input1.exists()) { + Assert.assertTrue(input1.createNewFile()); + } + + File input2 = new File(workSpace + File.separator + "input2"); + if (!input2.exists()) { + Assert.assertTrue(input2.createNewFile()); + } + // set data for splits + conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR, + input1.getAbsolutePath() + "," + input2.getAbsolutePath()); + InputSplit[] splits = inputFormat.getSplits(conf, 2); + assertEquals(2, splits.length); + + PipesNonJavaInputFormat.PipesDummyRecordReader dummyRecordReader = new PipesNonJavaInputFormat.PipesDummyRecordReader( + conf, splits[0]); + // empty dummyRecordReader + assertNull(dummyRecordReader.createKey()); + assertNull(dummyRecordReader.createValue()); + assertEquals(0, dummyRecordReader.getPos()); + assertEquals(0.0, dummyRecordReader.getProgress(), 0.001); + // test method next + assertTrue(dummyRecordReader.next(new FloatWritable(2.0f), NullWritable.get())); + assertEquals(2.0, dummyRecordReader.getProgress(), 0.001); + dummyRecordReader.close(); + } + +} From 8159dad8566641053045e8099939d858fa30f757 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 6 Feb 2013 18:20:54 +0000 Subject: [PATCH 34/69] HADOOP-9289. FsShell rm -f fails for non-matching globs. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443108 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/shell/Delete.java | 15 +++++++ .../hadoop/fs/TestFsShellReturnCode.java | 40 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ee2d1cf0fd3..d75aaac932b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -620,6 +620,9 @@ Release 2.0.3-alpha - 2013-02-06 HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in HarFileSystem. (Chris Nauroth via szetszwo) + HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via + suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java index 4dc550501c8..ed190d37461 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java @@ -21,6 +21,7 @@ package org.apache.hadoop.fs.shell; import java.io.FileNotFoundException; import java.io.IOException; import java.util.LinkedList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -28,6 +29,7 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.fs.Trash; /** @@ -71,6 +73,19 @@ class Delete { skipTrash = cf.getOpt("skipTrash"); } + @Override + protected List expandArgument(String arg) throws IOException { + try { + return super.expandArgument(arg); + } catch (PathNotFoundException e) { + if (!ignoreFNF) { + throw e; + } + // prevent -f on a non-existent glob from failing + return new LinkedList(); + } + } + @Override protected void processNonexistentPath(PathData item) throws IOException { if (!ignoreFNF) super.processNonexistentPath(item); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index c6812a19304..92980776637 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -303,6 +303,46 @@ public class TestFsShellReturnCode { } } + @Test + public void testRmWithNonexistentGlob() throws Exception { + Configuration conf = new Configuration(); + FsShell shell = new FsShell(); + shell.setConf(conf); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final PrintStream err = new PrintStream(bytes); + final PrintStream oldErr = System.err; + System.setErr(err); + final String results; + try { + int exit = shell.run(new String[]{"-rm", "nomatch*"}); + assertEquals(1, exit); + results = bytes.toString(); + assertTrue(results.contains("rm: `nomatch*': No such file or directory")); + } finally { + IOUtils.closeStream(err); + System.setErr(oldErr); + } + } + + @Test + public void testRmForceWithNonexistentGlob() throws Exception { + Configuration conf = new Configuration(); + FsShell shell = new FsShell(); + shell.setConf(conf); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final PrintStream err = new PrintStream(bytes); + final PrintStream oldErr = System.err; + System.setErr(err); + try { + int exit = shell.run(new String[]{"-rm", "-f", "nomatch*"}); + assertEquals(0, exit); + assertTrue(bytes.toString().isEmpty()); + } finally { + IOUtils.closeStream(err); + System.setErr(oldErr); + } + } + @Test public void testInvalidDefaultFS() throws Exception { // if default fs doesn't exist or is invalid, but the path provided in From ab16a375720f1b5fa8400498d6a727007275e1d8 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 6 Feb 2013 19:03:52 +0000 Subject: [PATCH 35/69] YARN-355. Fixes a bug where RM app submission could jam under load. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443131 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/client/YarnClientImpl.java | 28 ---- .../security/RMDelegationTokenRenewer.java | 83 ----------- ....apache.hadoop.security.token.TokenRenewer | 14 -- .../client/RMDelegationTokenIdentifier.java | 114 +++++++++++++++ ....apache.hadoop.security.token.TokenRenewer | 1 + .../resourcemanager/ClientRMService.java | 4 + .../resourcemanager/TestClientRMTokens.java | 134 +++++++++++++++++- 8 files changed, 252 insertions(+), 129 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/security/RMDelegationTokenRenewer.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1fd179dc1a6..620215692d4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -234,6 +234,9 @@ Release 2.0.3-alpha - 2013-02-06 YARN-370. Fix SchedulerUtils to correctly round up the resource for containers. (Zhijie Shen via acmurthy) + YARN-355. Fixes a bug where RM app submission could jam under load. + (Daryn Sharp via sseth) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java index 14994f97a7b..eb84b31c798 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java @@ -25,13 +25,11 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.api.ClientRMProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -47,8 +45,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; -import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -199,30 +195,6 @@ public class YarnClientImpl extends AbstractService implements YarnClient { } - // Not part of YarnClient itself. Placed in YarnClientImpl while renew/cancel - // are part of ClientRMProtocol. - @Private - public long renewRMDelegationToken(DelegationToken rmToken) - throws YarnRemoteException { - RenewDelegationTokenRequest request = Records - .newRecord(RenewDelegationTokenRequest.class); - request.setDelegationToken(rmToken); - RenewDelegationTokenResponse response = rmClient - .renewDelegationToken(request); - return response.getNextExpirationTime(); - } - - // Not part of YarnClient itself. Placed in YarnClientImpl while renew/cancel - // are part of ClietnRMProtocol - @Private - public void cancelRMDelegationToken(DelegationToken rmToken) - throws YarnRemoteException { - CancelDelegationTokenRequest request = Records - .newRecord(CancelDelegationTokenRequest.class); - request.setDelegationToken(rmToken); - rmClient.cancelDelegationToken(request); - } - private GetQueueInfoRequest getQueueInfoRequest(String queueName, boolean includeApplications, boolean includeChildQueues, boolean recursive) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/security/RMDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/security/RMDelegationTokenRenewer.java deleted file mode 100644 index 3f1caeec183..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/security/RMDelegationTokenRenewer.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.yarn.security; - -import java.io.IOException; -import java.net.InetSocketAddress; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenRenewer; -import org.apache.hadoop.yarn.api.records.DelegationToken; -import org.apache.hadoop.yarn.client.YarnClientImpl; -import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; -import org.apache.hadoop.yarn.util.BuilderUtils; - -public class RMDelegationTokenRenewer extends TokenRenewer { - - @Override - public boolean handleKind(Text kind) { - return RMDelegationTokenIdentifier.KIND_NAME.equals(kind); - } - - @Override - public boolean isManaged(Token token) throws IOException { - return true; - } - - @Override - public long renew(Token token, Configuration conf) throws IOException, - InterruptedException { - YarnClientImpl yarnClient = getYarnClient(conf, - SecurityUtil.getTokenServiceAddr(token)); - try { - DelegationToken dToken = BuilderUtils.newDelegationToken( - token.getIdentifier(), token.getKind().toString(), - token.getPassword(), token.getService().toString()); - return yarnClient.renewRMDelegationToken(dToken); - } finally { - yarnClient.stop(); - } - } - - @Override - public void cancel(Token token, Configuration conf) throws IOException, - InterruptedException { - YarnClientImpl yarnClient = getYarnClient(conf, - SecurityUtil.getTokenServiceAddr(token)); - try { - DelegationToken dToken = BuilderUtils.newDelegationToken( - token.getIdentifier(), token.getKind().toString(), - token.getPassword(), token.getService().toString()); - yarnClient.cancelRMDelegationToken(dToken); - return; - } finally { - yarnClient.stop(); - } - } - - private YarnClientImpl getYarnClient(Configuration conf, - InetSocketAddress rmAddress) { - YarnClientImpl yarnClient = new YarnClientImpl(rmAddress); - yarnClient.init(conf); - yarnClient.start(); - return yarnClient; - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer deleted file mode 100644 index 9e78b1187e2..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.hadoop.yarn.security.RMDelegationTokenRenewer; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java index fd3dbf06b68..73bdce4e99b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java @@ -19,10 +19,28 @@ package org.apache.hadoop.yarn.security.client; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenRenewer; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; /** * Delegation Token Identifier that identifies the delegation tokens from the @@ -51,4 +69,100 @@ public class RMDelegationTokenIdentifier extends AbstractDelegationTokenIdentifi public Text getKind() { return KIND_NAME; } + + public static class Renewer extends TokenRenewer { + + @Override + public boolean handleKind(Text kind) { + return KIND_NAME.equals(kind); + } + + @Override + public boolean isManaged(Token token) throws IOException { + return true; + } + + private static + AbstractDelegationTokenSecretManager localSecretManager; + private static InetSocketAddress localServiceAddress; + + @Private + public static void setSecretManager( + AbstractDelegationTokenSecretManager secretManager, + InetSocketAddress serviceAddress) { + localSecretManager = secretManager; + localServiceAddress = serviceAddress; + } + + @SuppressWarnings("unchecked") + @Override + public long renew(Token token, Configuration conf) throws IOException, + InterruptedException { + final ClientRMProtocol rmClient = getRmClient(token, conf); + if (rmClient != null) { + try { + RenewDelegationTokenRequest request = + Records.newRecord(RenewDelegationTokenRequest.class); + request.setDelegationToken(convertToProtoToken(token)); + return rmClient.renewDelegationToken(request).getNextExpirationTime(); + } finally { + RPC.stopProxy(rmClient); + } + } else { + return localSecretManager.renewToken( + (Token)token, getRenewer(token)); + } + } + + @SuppressWarnings("unchecked") + @Override + public void cancel(Token token, Configuration conf) throws IOException, + InterruptedException { + final ClientRMProtocol rmClient = getRmClient(token, conf); + if (rmClient != null) { + try { + CancelDelegationTokenRequest request = + Records.newRecord(CancelDelegationTokenRequest.class); + request.setDelegationToken(convertToProtoToken(token)); + rmClient.cancelDelegationToken(request); + } finally { + RPC.stopProxy(rmClient); + } + } else { + localSecretManager.cancelToken( + (Token)token, getRenewer(token)); + } + } + + private static ClientRMProtocol getRmClient(Token token, + Configuration conf) { + InetSocketAddress addr = SecurityUtil.getTokenServiceAddr(token); + if (localSecretManager != null) { + // return null if it's our token + if (localServiceAddress.getAddress().isAnyLocalAddress()) { + if (NetUtils.isLocalAddress(addr.getAddress()) && + addr.getPort() == localServiceAddress.getPort()) { + return null; + } + } else if (addr.equals(localServiceAddress)) { + return null; + } + } + final YarnRPC rpc = YarnRPC.create(conf); + return (ClientRMProtocol)rpc.getProxy(ClientRMProtocol.class, addr, conf); + } + + // get renewer so we can always renew our own tokens + @SuppressWarnings("unchecked") + private static String getRenewer(Token token) throws IOException { + return ((Token)token).decodeIdentifier() + .getRenewer().toString(); + } + + private static DelegationToken convertToProtoToken(Token token) { + return BuilderUtils.newDelegationToken( + token.getIdentifier(), token.getKind().toString(), + token.getPassword(), token.getService().toString()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index 3380cb8b019..0e87a7c2d17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -13,3 +13,4 @@ # org.apache.hadoop.yarn.security.ApplicationTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer +org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index a464b3ae000..e0522a33fdc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -157,6 +157,10 @@ public class ClientRMService extends AbstractService implements this.server.start(); clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_ADDRESS, server.getListenerAddress()); + // enable RM to short-circuit token operations directly to itself + RMDelegationTokenIdentifier.Renewer.setSecretManager( + rmDTSecretManager, clientBindAddress); + super.start(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 3f786964745..5ee851b8438 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -17,13 +17,12 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; @@ -34,9 +33,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; @@ -46,12 +51,14 @@ import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ProtoUtils; import org.apache.hadoop.yarn.util.Records; +import org.junit.Before; import org.junit.Test; @@ -59,6 +66,10 @@ public class TestClientRMTokens { private static final Log LOG = LogFactory.getLog(TestClientRMTokens.class); + @Before + public void resetSecretManager() { + RMDelegationTokenIdentifier.Renewer.setSecretManager(null, null); + } @Test public void testDelegationToken() throws IOException, InterruptedException { @@ -200,7 +211,122 @@ public class TestClientRMTokens { RPC.stopProxy(clientRMWithDT); } } + } + + @Test + public void testShortCircuitRenewCancel() + throws IOException, InterruptedException { + InetSocketAddress addr = + new InetSocketAddress(InetAddress.getLocalHost(), 123); + checkShortCircuitRenewCancel(addr, addr, true); + } + + @Test + public void testShortCircuitRenewCancelWildcardAddress() + throws IOException, InterruptedException { + InetSocketAddress rmAddr = new InetSocketAddress(123); + checkShortCircuitRenewCancel( + rmAddr, + new InetSocketAddress(InetAddress.getLocalHost(), rmAddr.getPort()), + true); + } + + @Test + public void testShortCircuitRenewCancelSameHostDifferentPort() + throws IOException, InterruptedException { + InetSocketAddress rmAddr = + new InetSocketAddress(InetAddress.getLocalHost(), 123); + checkShortCircuitRenewCancel( + rmAddr, + new InetSocketAddress(rmAddr.getAddress(), rmAddr.getPort()+1), + false); + } + + @Test + public void testShortCircuitRenewCancelDifferentHostSamePort() + throws IOException, InterruptedException { + InetSocketAddress rmAddr = + new InetSocketAddress(InetAddress.getLocalHost(), 123); + checkShortCircuitRenewCancel( + rmAddr, + new InetSocketAddress("1.1.1.1", rmAddr.getPort()), + false); + } + + @Test + public void testShortCircuitRenewCancelDifferentHostDifferentPort() + throws IOException, InterruptedException { + InetSocketAddress rmAddr = + new InetSocketAddress(InetAddress.getLocalHost(), 123); + checkShortCircuitRenewCancel( + rmAddr, + new InetSocketAddress("1.1.1.1", rmAddr.getPort()+1), + false); + } + + @SuppressWarnings("unchecked") + private void checkShortCircuitRenewCancel(InetSocketAddress rmAddr, + InetSocketAddress serviceAddr, + boolean shouldShortCircuit + ) throws IOException, InterruptedException { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.IPC_RPC_IMPL, + YarnBadRPC.class, YarnRPC.class); + RMDelegationTokenSecretManager secretManager = + mock(RMDelegationTokenSecretManager.class); + RMDelegationTokenIdentifier.Renewer.setSecretManager(secretManager, rmAddr); + + RMDelegationTokenIdentifier ident = new RMDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), null); + Token token = + new Token(ident, secretManager); + + SecurityUtil.setTokenService(token, serviceAddr); + if (shouldShortCircuit) { + token.renew(conf); + verify(secretManager).renewToken(eq(token), eq("renewer")); + reset(secretManager); + token.cancel(conf); + verify(secretManager).cancelToken(eq(token), eq("renewer")); + } else { + try { + token.renew(conf); + fail(); + } catch (RuntimeException e) { + assertEquals("getProxy", e.getMessage()); + } + verify(secretManager, never()).renewToken(any(Token.class), anyString()); + try { + token.cancel(conf); + fail(); + } catch (RuntimeException e) { + assertEquals("getProxy", e.getMessage()); + } + verify(secretManager, never()).cancelToken(any(Token.class), anyString()); + } + } + + @SuppressWarnings("rawtypes") + public static class YarnBadRPC extends YarnRPC { + @Override + public Object getProxy(Class protocol, InetSocketAddress addr, + Configuration conf) { + throw new RuntimeException("getProxy"); + } + + @Override + public void stopProxy(Object proxy, Configuration conf) { + throw new RuntimeException("stopProxy"); + } + + @Override + public Server getServer(Class protocol, Object instance, + InetSocketAddress addr, Configuration conf, + SecretManager secretManager, + int numHandlers, String portRangeConfig) { + throw new RuntimeException("getServer"); + } } // Get the delegation token directly as it is a little difficult to setup From 17e72be6d89ceb68dcdccbf15c85c004f3b31a9a Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 6 Feb 2013 19:52:25 +0000 Subject: [PATCH 36/69] MAPREDUCE-4977. Documentation for pluggable shuffle and pluggable sort. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443168 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../PluggableShuffleAndPluggableSort.apt.vm | 96 +++++++++++++++++++ hadoop-project/src/site/site.xml | 1 + 3 files changed, 100 insertions(+) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c302e8f63e2..37ccdda2c0e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -230,6 +230,9 @@ Release 2.0.3-alpha - 2013-02-06 MAPREDUCE-4971. Minor extensibility enhancements to Counters & FileOutputFormat. (Arun C Murthy via sseth) + MAPREDUCE-4977. Documentation for pluggable shuffle and pluggable sort. + (tucu) + OPTIMIZATIONS MAPREDUCE-4893. Fixed MR ApplicationMaster to do optimal assignment of diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm new file mode 100644 index 00000000000..8dd2f2eceff --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm @@ -0,0 +1,96 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + Hadoop Map Reduce Next Generation-${project.version} - Pluggable Shuffle and Pluggable Sort + --- + --- + ${maven.build.timestamp} + +Hadoop MapReduce Next Generation - Pluggable Shuffle and Pluggable Sort + + \[ {{{./index.html}Go Back}} \] + +* Introduction + + The pluggable shuffle and pluggable sort capabilities allow replacing the + built in shuffle and sort logic with alternate implementations. Example use + cases for this are: using a different application protocol other than HTTP + such as RDMA for shuffling data from the Map nodes to the Reducer nodes; or + replacing the sort logic with custom algorithms that enable Hash aggregation + and Limit-N query. + + <> The pluggable shuffle and pluggable sort capabilities are + experimental and unstable. This means the provided APIs may change and break + compatibility in future versions of Hadoop. + +* Implementing a Custom Shuffle and a Custom Sort + + A custom shuffle implementation requires a + <<>> + implementation class running in the NodeManagers and a + <<>> implementation class + running in the Reducer tasks. + + The default implementations provided by Hadoop can be used as references: + + * <<>> + + * <<>> + + A custom sort implementation requires a <<>> + implementation class running in the Mapper tasks and (optionally, depending + on the sort implementation) a <<>> + implementation class running in the Reducer tasks. + + The default implementations provided by Hadoop can be used as references: + + * <<>> + + * <<>> + +* Configuration + + Except for the auxiliary service running in the NodeManagers serving the + shuffle (by default the <<>>), all the pluggable components + run in the job tasks. This means, they can be configured on per job basis. + The auxiliary service servicing the Shuffle must be configured in the + NodeManagers configuration. + +** Job Configuration Properties (on per job basis): + +*--------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | The <<>> implementation to use | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | The <<>> implementation to use | +*--------------------------------------+---------------------+-----------------+ + + These properties can also be set in the <<>> to change the default values for all jobs. + +** NodeManager Configuration properties, <<>> in all nodes: + +*--------------------------------------+---------------------+-----------------+ +| <> | <> | <> | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<<...,mapreduce.shuffle>>> | The auxiliary service name | +*--------------------------------------+---------------------+-----------------+ +| <<>> | <<>> | The auxiliary service class to use | +*--------------------------------------+---------------------+-----------------+ + + <> If setting an auxiliary service in addition the default + <<>> service, then a new service key should be added to the + <<>> property, for example <<>>. + Then the property defining the corresponding class must be + <<>>. + \ No newline at end of file diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 2cfe2e8c456..a199ce35022 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -65,6 +65,7 @@

+ From 4525c4a25ba90163c9543116e2bd54239e0dd097 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 6 Feb 2013 19:52:34 +0000 Subject: [PATCH 37/69] HDFS-4340. Update addBlock() to inculde inode id as additional argument. Contributed Brandon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443169 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 1 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../apache/hadoop/hdfs/DFSOutputStream.java | 65 ++++++++++--------- .../hadoop/hdfs/protocol/ClientProtocol.java | 20 +++--- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 9 ++- .../hdfs/protocol/HdfsLocatedFileStatus.java | 20 +++--- ...amenodeProtocolServerSideTranslatorPB.java | 28 ++++---- .../ClientNamenodeProtocolTranslatorPB.java | 28 ++++---- .../hadoop/hdfs/protocolPB/PBHelper.java | 3 + .../hdfs/server/namenode/FSDirectory.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 59 +++++++++-------- .../hadoop/hdfs/server/namenode/INodeId.java | 26 ++++++-- .../server/namenode/NameNodeRpcServer.java | 42 ++++++------ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 4 +- .../main/proto/ClientNamenodeProtocol.proto | 4 +- .../hadoop-hdfs/src/main/proto/hdfs.proto | 3 + .../hadoop/hdfs/TestDFSClientRetries.java | 32 +++++++-- .../apache/hadoop/hdfs/TestFileCreation.java | 9 +-- .../org/apache/hadoop/hdfs/TestLease.java | 23 +++++++ .../namenode/NNThroughputBenchmark.java | 3 +- .../server/namenode/TestAddBlockRetry.java | 5 +- .../hdfs/server/namenode/TestINodeFile.java | 55 +++++++++++++++- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 8 ++- 23 files changed, 307 insertions(+), 147 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d75aaac932b..642a686593a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -623,6 +623,7 @@ Release 2.0.3-alpha - 2013-02-06 HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 89f0ceb30d7..6c865bfa687 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -296,6 +296,9 @@ Trunk (Unreleased) HDFS-4382. Fix typo MAX_NOT_CHANGED_INTERATIONS. (Ted Yu via suresh) + HDFS-4340. Update addBlock() to inculde inode id as additional argument. + (Brandon Li via suresh) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 3232b1234a6..1edd2fe746f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -115,6 +115,7 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { private volatile boolean closed = false; private String src; + private final long fileId; private final long blockSize; private final DataChecksum checksum; // both dataQueue and ackQueue are protected by dataQueue lock @@ -1148,7 +1149,8 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { long localstart = Time.now(); while (true) { try { - return dfsClient.namenode.addBlock(src, dfsClient.clientName, block, excludedNodes); + return dfsClient.namenode.addBlock(src, dfsClient.clientName, + block, excludedNodes, fileId); } catch (RemoteException e) { IOException ue = e.unwrapRemoteException(FileNotFoundException.class, @@ -1261,20 +1263,21 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { return value; } - private DFSOutputStream(DFSClient dfsClient, String src, long blockSize, Progressable progress, - DataChecksum checksum, short replication) throws IOException { + private DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, + HdfsFileStatus stat, DataChecksum checksum) throws IOException { super(checksum, checksum.getBytesPerChecksum(), checksum.getChecksumSize()); - int bytesPerChecksum = checksum.getBytesPerChecksum(); this.dfsClient = dfsClient; this.src = src; - this.blockSize = blockSize; - this.blockReplication = replication; + this.fileId = stat.getFileId(); + this.blockSize = stat.getBlockSize(); + this.blockReplication = stat.getReplication(); this.progress = progress; if ((progress != null) && DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug( "Set non-null progress callback on DFSOutputStream " + src); } + final int bytesPerChecksum = checksum.getBytesPerChecksum(); if ( bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) { throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum + ") and blockSize(" + blockSize + @@ -1286,19 +1289,27 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { } /** Construct a new output stream for creating a file. */ - private DFSOutputStream(DFSClient dfsClient, String src, FsPermission masked, - EnumSet flag, boolean createParent, short replication, - long blockSize, Progressable progress, int buffersize, + private DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat, + EnumSet flag, Progressable progress, DataChecksum checksum) throws IOException { - this(dfsClient, src, blockSize, progress, checksum, replication); + this(dfsClient, src, progress, stat, checksum); this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK); computePacketChunkSize(dfsClient.getConf().writePacketSize, checksum.getBytesPerChecksum()); + streamer = new DataStreamer(); + } + + static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, + FsPermission masked, EnumSet flag, boolean createParent, + short replication, long blockSize, Progressable progress, int buffersize, + DataChecksum checksum) throws IOException { + final HdfsFileStatus stat; try { - dfsClient.namenode.create( - src, masked, dfsClient.clientName, new EnumSetWritable(flag), createParent, replication, blockSize); + stat = dfsClient.namenode.create(src, masked, dfsClient.clientName, + new EnumSetWritable(flag), createParent, replication, + blockSize); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, @@ -1309,30 +1320,20 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { SafeModeException.class, UnresolvedPathException.class); } - streamer = new DataStreamer(); - } - - static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, - FsPermission masked, EnumSet flag, boolean createParent, - short replication, long blockSize, Progressable progress, int buffersize, - DataChecksum checksum) throws IOException { - final DFSOutputStream out = new DFSOutputStream(dfsClient, src, masked, - flag, createParent, replication, blockSize, progress, buffersize, - checksum); - out.streamer.start(); + final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat, + flag, progress, checksum); + out.start(); return out; } /** Construct a new output stream for append. */ - private DFSOutputStream(DFSClient dfsClient, String src, int buffersize, Progressable progress, - LocatedBlock lastBlock, HdfsFileStatus stat, + private DFSOutputStream(DFSClient dfsClient, String src, + Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException { - this(dfsClient, src, stat.getBlockSize(), progress, checksum, stat.getReplication()); + this(dfsClient, src, progress, stat, checksum); initialFileSize = stat.getLen(); // length of file when opened - // // The last partial block of the file has to be filled. - // if (lastBlock != null) { // indicate that we are appending to an existing block bytesCurBlock = lastBlock.getBlockSize(); @@ -1347,9 +1348,9 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src, int buffersize, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException { - final DFSOutputStream out = new DFSOutputStream(dfsClient, src, buffersize, + final DFSOutputStream out = new DFSOutputStream(dfsClient, src, progress, lastBlock, stat, checksum); - out.streamer.start(); + out.start(); return out; } @@ -1716,6 +1717,10 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable { isClosed(); } + private synchronized void start() { + streamer.start(); + } + /** * Aborts this output stream and releases any system * resources associated with this stream. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index d7ce8e57d67..9621c979473 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -150,6 +150,8 @@ public interface ClientProtocol { * @param replication block replication factor. * @param blockSize maximum block size. * + * @return the status of the created file, it could be null if the server + * doesn't support returning the file status * @throws AccessControlException If access is denied * @throws AlreadyBeingCreatedException if the path does not exist. * @throws DSQuotaExceededException If file creation violates disk space @@ -168,13 +170,14 @@ public interface ClientProtocol { * RuntimeExceptions: * @throws InvalidPathException Path src is invalid */ - public void create(String src, FsPermission masked, String clientName, - EnumSetWritable flag, boolean createParent, - short replication, long blockSize) throws AccessControlException, - AlreadyBeingCreatedException, DSQuotaExceededException, - FileAlreadyExistsException, FileNotFoundException, - NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, - UnresolvedLinkException, IOException; + public HdfsFileStatus create(String src, FsPermission masked, + String clientName, EnumSetWritable flag, + boolean createParent, short replication, long blockSize) + throws AccessControlException, AlreadyBeingCreatedException, + DSQuotaExceededException, FileAlreadyExistsException, + FileNotFoundException, NSQuotaExceededException, + ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, + IOException; /** * Append to the end of the file. @@ -296,6 +299,7 @@ public interface ClientProtocol { * @param previous previous block * @param excludeNodes a list of nodes that should not be * allocated for the current block + * @param fileId the id uniquely identifying a file * * @return LocatedBlock allocated block information. * @@ -310,7 +314,7 @@ public interface ClientProtocol { */ @Idempotent public LocatedBlock addBlock(String src, String clientName, - ExtendedBlock previous, DatanodeInfo[] excludeNodes) + ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId) throws AccessControlException, FileNotFoundException, NotReplicatedYetException, SafeModeException, UnresolvedLinkException, IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 26b6e984d53..7d1bd32ba82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -40,6 +40,7 @@ public class HdfsFileStatus { private FsPermission permission; private String owner; private String group; + private long fileId; public static final byte[] EMPTY_NAME = new byte[0]; @@ -55,11 +56,12 @@ public class HdfsFileStatus { * @param owner the owner of the path * @param group the group of the path * @param path the local name in java UTF8 encoding the same as that in-memory + * @param fileId the file id */ public HdfsFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, - byte[] symlink, byte[] path) { + byte[] symlink, byte[] path, long fileId) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -75,6 +77,7 @@ public class HdfsFileStatus { this.group = (group == null) ? "" : group; this.symlink = symlink; this.path = path; + this.fileId = fileId; } /** @@ -223,4 +226,8 @@ public class HdfsFileStatus { final public byte[] getSymlinkInBytes() { return symlink; } + + final public long getFileId() { + return fileId; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 167b66b1e16..987ba42091c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -44,19 +44,19 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { * @param group group * @param symlink symbolic link * @param path local path name in java UTF8 format + * @param fileId the file id * @param locations block locations */ public HdfsLocatedFileStatus(long length, boolean isdir, - int block_replication, - long blocksize, long modification_time, long access_time, - FsPermission permission, String owner, String group, - byte[] symlink, byte[] path, LocatedBlocks locations) { - super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, owner, group, symlink, path); + int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, String owner, String group, + byte[] symlink, byte[] path, long fileId, LocatedBlocks locations) { + super(length, isdir, block_replication, blocksize, modification_time, + access_time, permission, owner, group, symlink, path, fileId); this.locations = locations; - } + } - public LocatedBlocks getBlockLocations() { - return locations; - } + public LocatedBlocks getBlockLocations() { + return locations; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index bb71b217817..af422155508 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -268,14 +268,19 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements public CreateResponseProto create(RpcController controller, CreateRequestProto req) throws ServiceException { try { - server.create(req.getSrc(), PBHelper.convert(req.getMasked()), - req.getClientName(), PBHelper.convert(req.getCreateFlag()), - req.getCreateParent(), (short) req.getReplication(), - req.getBlockSize()); + HdfsFileStatus result = server.create(req.getSrc(), + PBHelper.convert(req.getMasked()), req.getClientName(), + PBHelper.convert(req.getCreateFlag()), req.getCreateParent(), + (short) req.getReplication(), req.getBlockSize()); + + if (result != null) { + return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result)) + .build(); + } + return VOID_CREATE_RESPONSE; } catch (IOException e) { throw new ServiceException(e); } - return VOID_CREATE_RESPONSE; } @Override @@ -348,13 +353,14 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements try { List excl = req.getExcludeNodesList(); - LocatedBlock result = server.addBlock(req.getSrc(), req.getClientName(), + LocatedBlock result = server.addBlock( + req.getSrc(), + req.getClientName(), req.hasPrevious() ? PBHelper.convert(req.getPrevious()) : null, - (excl == null || - excl.size() == 0) ? null : - PBHelper.convert(excl.toArray(new DatanodeInfoProto[excl.size()]))); - return AddBlockResponseProto.newBuilder().setBlock( - PBHelper.convert(result)).build(); + (excl == null || excl.size() == 0) ? null : PBHelper.convert(excl + .toArray(new DatanodeInfoProto[excl.size()])), req.getFileId()); + return AddBlockResponseProto.newBuilder() + .setBlock(PBHelper.convert(result)).build(); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 10ef41acbf3..8e510e6d3ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Append import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto; @@ -100,6 +101,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.io.EnumSetWritable; @@ -193,13 +195,14 @@ public class ClientNamenodeProtocolTranslatorPB implements } @Override - public void create(String src, FsPermission masked, String clientName, - EnumSetWritable flag, boolean createParent, - short replication, long blockSize) throws AccessControlException, - AlreadyBeingCreatedException, DSQuotaExceededException, - FileAlreadyExistsException, FileNotFoundException, - NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, - UnresolvedLinkException, IOException { + public HdfsFileStatus create(String src, FsPermission masked, + String clientName, EnumSetWritable flag, + boolean createParent, short replication, long blockSize) + throws AccessControlException, AlreadyBeingCreatedException, + DSQuotaExceededException, FileAlreadyExistsException, + FileNotFoundException, NSQuotaExceededException, + ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, + IOException { CreateRequestProto req = CreateRequestProto.newBuilder() .setSrc(src) .setMasked(PBHelper.convert(masked)) @@ -210,7 +213,8 @@ public class ClientNamenodeProtocolTranslatorPB implements .setBlockSize(blockSize) .build(); try { - rpcProxy.create(null, req); + CreateResponseProto res = rpcProxy.create(null, req); + return res.hasFs() ? PBHelper.convert(res.getFs()) : null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } @@ -294,15 +298,15 @@ public class ClientNamenodeProtocolTranslatorPB implements throw ProtobufHelper.getRemoteException(e); } } - + @Override public LocatedBlock addBlock(String src, String clientName, - ExtendedBlock previous, DatanodeInfo[] excludeNodes) + ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId) throws AccessControlException, FileNotFoundException, NotReplicatedYetException, SafeModeException, UnresolvedLinkException, IOException { - AddBlockRequestProto.Builder req = AddBlockRequestProto.newBuilder().setSrc(src) - .setClientName(clientName); + AddBlockRequestProto.Builder req = AddBlockRequestProto.newBuilder() + .setSrc(src).setClientName(clientName).setFileId(fileId); if (previous != null) req.setPrevious(PBHelper.convert(previous)); if (excludeNodes != null) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index eec66821653..e2af429ef88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -106,6 +106,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; +import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; @@ -1045,6 +1046,7 @@ public class PBHelper { fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), + fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID, fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null); } @@ -1069,6 +1071,7 @@ public class PBHelper { setPermission(PBHelper.convert(fs.getPermission())). setOwner(fs.getOwner()). setGroup(fs.getGroup()). + setFileId(fs.getFileId()). setPath(ByteString.copyFrom(fs.getLocalNameInBytes())); if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index bc384f3e77d..9becf41174d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1985,7 +1985,8 @@ public class FSDirectory implements Closeable { node.getUserName(), node.getGroupName(), node.isSymlink() ? ((INodeSymlink)node).getSymlink() : null, - path); + path, + node.getId()); } /** @@ -2022,6 +2023,7 @@ public class FSDirectory implements Closeable { node.getGroupName(), node.isSymlink() ? ((INodeSymlink)node).getSymlink() : null, path, + node.getId(), loc); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 06c4beb345c..2a45afa1747 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1772,16 +1772,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * Create a new file entry in the namespace. * * For description of parameters and exceptions thrown see - * {@link ClientProtocol#create()} + * {@link ClientProtocol#create()}, except it returns valid file status + * upon success */ - void startFile(String src, PermissionStatus permissions, String holder, - String clientMachine, EnumSet flag, boolean createParent, - short replication, long blockSize) throws AccessControlException, - SafeModeException, FileAlreadyExistsException, UnresolvedLinkException, + HdfsFileStatus startFile(String src, PermissionStatus permissions, + String holder, String clientMachine, EnumSet flag, + boolean createParent, short replication, long blockSize) + throws AccessControlException, SafeModeException, + FileAlreadyExistsException, UnresolvedLinkException, FileNotFoundException, ParentNotDirectoryException, IOException { try { - startFileInt(src, permissions, holder, clientMachine, flag, createParent, - replication, blockSize); + return startFileInt(src, permissions, holder, clientMachine, flag, + createParent, replication, blockSize); } catch (AccessControlException e) { if (isAuditEnabled() && isExternalInvocation()) { logAuditEvent(false, UserGroupInformation.getCurrentUser(), @@ -1792,18 +1794,21 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } - private void startFileInt(String src, PermissionStatus permissions, String holder, - String clientMachine, EnumSet flag, boolean createParent, - short replication, long blockSize) throws AccessControlException, - SafeModeException, FileAlreadyExistsException, UnresolvedLinkException, + private HdfsFileStatus startFileInt(String src, PermissionStatus permissions, + String holder, String clientMachine, EnumSet flag, + boolean createParent, short replication, long blockSize) + throws AccessControlException, SafeModeException, + FileAlreadyExistsException, UnresolvedLinkException, FileNotFoundException, ParentNotDirectoryException, IOException { boolean skipSync = false; + final HdfsFileStatus stat; writeLock(); try { checkOperation(OperationCategory.WRITE); startFileInternal(src, permissions, holder, clientMachine, flag, createParent, replication, blockSize); + stat = dir.getFileInfo(src, false); } catch (StandbyException se) { skipSync = true; throw se; @@ -1817,11 +1822,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } if (isAuditEnabled() && isExternalInvocation()) { - final HdfsFileStatus stat = dir.getFileInfo(src, false); logAuditEvent(UserGroupInformation.getCurrentUser(), getRemoteIp(), "create", src, null, stat); } + return stat; } /** @@ -2192,11 +2197,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * are replicated. Will return an empty 2-elt array if we want the * client to "try again later". */ - LocatedBlock getAdditionalBlock(String src, - String clientName, - ExtendedBlock previous, - HashMap excludedNodes - ) + LocatedBlock getAdditionalBlock(String src, long fileId, String clientName, + ExtendedBlock previous, HashMap excludedNodes) throws LeaseExpiredException, NotReplicatedYetException, QuotaExceededException, SafeModeException, UnresolvedLinkException, IOException { @@ -2215,7 +2217,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { LocatedBlock[] onRetryBlock = new LocatedBlock[1]; final INode[] inodes = analyzeFileState( - src, clientName, previous, onRetryBlock).getINodes(); + src, fileId, clientName, previous, onRetryBlock).getINodes(); final INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) inodes[inodes.length - 1]; @@ -2245,7 +2247,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // while chooseTarget() was executing. LocatedBlock[] onRetryBlock = new LocatedBlock[1]; INodesInPath inodesInPath = - analyzeFileState(src, clientName, previous, onRetryBlock); + analyzeFileState(src, fileId, clientName, previous, onRetryBlock); INode[] inodes = inodesInPath.getINodes(); final INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) inodes[inodes.length - 1]; @@ -2277,6 +2279,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } INodesInPath analyzeFileState(String src, + long fileId, String clientName, ExtendedBlock previous, LocatedBlock[] onRetryBlock) @@ -2298,7 +2301,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, dir.rootDir.getExistingPathINodes(src, true); final INode[] inodes = inodesInPath.getINodes(); final INodeFileUnderConstruction pendingFile - = checkLease(src, clientName, inodes[inodes.length - 1]); + = checkLease(src, fileId, clientName, inodes[inodes.length - 1]); BlockInfo lastBlockInFile = pendingFile.getLastBlock(); if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) { // The block that the client claims is the current last block @@ -2467,14 +2470,17 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } // make sure that we still have the lease on this file. - private INodeFileUnderConstruction checkLease(String src, String holder) - throws LeaseExpiredException, UnresolvedLinkException { + private INodeFileUnderConstruction checkLease(String src, String holder) + throws LeaseExpiredException, UnresolvedLinkException, + FileNotFoundException { assert hasReadOrWriteLock(); - return checkLease(src, holder, dir.getINode(src)); + return checkLease(src, INodeId.GRANDFATHER_INODE_ID, holder, + dir.getINode(src)); } - - private INodeFileUnderConstruction checkLease(String src, String holder, - INode file) throws LeaseExpiredException { + + private INodeFileUnderConstruction checkLease(String src, long fileId, + String holder, INode file) throws LeaseExpiredException, + FileNotFoundException { assert hasReadOrWriteLock(); if (file == null || !(file instanceof INodeFile)) { Lease lease = leaseManager.getLease(holder); @@ -2495,6 +2501,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throw new LeaseExpiredException("Lease mismatch on " + src + " owned by " + pendingFile.getClientName() + " but is accessed by " + holder); } + INodeId.checkId(fileId, pendingFile); return pendingFile; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java index 11157b2eb3d..293afb82c3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java @@ -17,18 +17,21 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.FileNotFoundException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.SequentialNumber; /** - * An id which uniquely identifies an inode + * An id which uniquely identifies an inode. Id 1 to 1000 are reserved for + * potential future usage. The id won't be recycled and is not expected to wrap + * around in a very long time. Root inode id is always 1001. Id 0 is used for + * backward compatibility support. */ @InterfaceAudience.Private -class INodeId extends SequentialNumber { +public class INodeId extends SequentialNumber { /** - * The last reserved inode id. Reserve id 1 to 1000 for potential future - * usage. The id won't be recycled and is not expected to wrap around in a - * very long time. Root inode id will be 1001. + * The last reserved inode id. */ public static final long LAST_RESERVED_ID = 1000L; @@ -38,6 +41,19 @@ class INodeId extends SequentialNumber { */ public static final long GRANDFATHER_INODE_ID = 0; + /** + * To check if the request id is the same as saved id. Don't check fileId + * with GRANDFATHER_INODE_ID for backward compatibility. + */ + public static void checkId(long requestId, INode inode) + throws FileNotFoundException { + if (requestId != GRANDFATHER_INODE_ID && requestId != inode.getId()) { + throw new FileNotFoundException( + "ID mismatch. Request id and saved id: " + requestId + " , " + + inode.getId()); + } + } + INodeId() { super(LAST_RESERVED_ID); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 654198d43c3..90e79363b17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -422,13 +422,10 @@ class NameNodeRpcServer implements NamenodeProtocols { } @Override // ClientProtocol - public void create(String src, - FsPermission masked, - String clientName, - EnumSetWritable flag, - boolean createParent, - short replication, - long blockSize) throws IOException { + public HdfsFileStatus create(String src, FsPermission masked, + String clientName, EnumSetWritable flag, + boolean createParent, short replication, long blockSize) + throws IOException { String clientMachine = getClientMachine(); if (stateChangeLog.isDebugEnabled()) { stateChangeLog.debug("*DIR* NameNode.create: file " @@ -438,12 +435,13 @@ class NameNodeRpcServer implements NamenodeProtocols { throw new IOException("create: Pathname too long. Limit " + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } - namesystem.startFile(src, - new PermissionStatus(UserGroupInformation.getCurrentUser().getShortUserName(), - null, masked), - clientName, clientMachine, flag.get(), createParent, replication, blockSize); + HdfsFileStatus fileStatus = namesystem.startFile(src, new PermissionStatus( + UserGroupInformation.getCurrentUser().getShortUserName(), null, masked), + clientName, clientMachine, flag.get(), createParent, replication, + blockSize); metrics.incrFilesCreated(); metrics.incrCreateFileOps(); + return fileStatus; } @Override // ClientProtocol @@ -482,26 +480,24 @@ class NameNodeRpcServer implements NamenodeProtocols { throws IOException { namesystem.setOwner(src, username, groupname); } - - @Override // ClientProtocol - public LocatedBlock addBlock(String src, - String clientName, - ExtendedBlock previous, - DatanodeInfo[] excludedNodes) + + @Override + public LocatedBlock addBlock(String src, String clientName, + ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId) throws IOException { - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*BLOCK* NameNode.addBlock: file " - +src+" for "+clientName); + if (stateChangeLog.isDebugEnabled()) { + stateChangeLog.debug("*BLOCK* NameNode.addBlock: file " + src + + " fileId=" + fileId + " for " + clientName); } HashMap excludedNodesSet = null; if (excludedNodes != null) { excludedNodesSet = new HashMap(excludedNodes.length); - for (Node node:excludedNodes) { + for (Node node : excludedNodes) { excludedNodesSet.put(node, node); } } - LocatedBlock locatedBlock = - namesystem.getAdditionalBlock(src, clientName, previous, excludedNodesSet); + LocatedBlock locatedBlock = namesystem.getAdditionalBlock(src, fileId, + clientName, previous, excludedNodesSet); if (locatedBlock != null) metrics.incrAddBlockOps(); return locatedBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index f251e34c13a..e04fb694bdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -219,6 +219,7 @@ public class JsonUtil { m.put("modificationTime", status.getModificationTime()); m.put("blockSize", status.getBlockSize()); m.put("replication", status.getReplication()); + m.put("fileId", status.getFileId()); return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m); } @@ -243,9 +244,10 @@ public class JsonUtil { final long mTime = (Long) m.get("modificationTime"); final long blockSize = (Long) m.get("blockSize"); final short replication = (short) (long) (Long) m.get("replication"); + final long fileId = (Long) m.get("fileId"); return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication, blockSize, mTime, aTime, permission, owner, group, - symlink, DFSUtil.string2Bytes(localName)); + symlink, DFSUtil.string2Bytes(localName), fileId); } /** Convert an ExtendedBlock to a Json map. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index db95f17ec86..419ae45b738 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -67,7 +67,8 @@ message CreateRequestProto { required uint64 blockSize = 7; } -message CreateResponseProto { // void response +message CreateResponseProto { + optional HdfsFileStatusProto fs = 1; } message AppendRequestProto { @@ -119,6 +120,7 @@ message AddBlockRequestProto { required string clientName = 2; optional ExtendedBlockProto previous = 3; repeated DatanodeInfoProto excludeNodes = 4; + optional uint64 fileId = 5 [default = 0]; // default as a bogus id } message AddBlockResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 0bc26290a16..d96d5b2d6f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -170,6 +170,9 @@ message HdfsFileStatusProto { optional uint32 block_replication = 10 [default = 0]; // only 16bits used optional uint64 blocksize = 11 [default = 0]; optional LocatedBlocksProto locations = 12; // suppled only if asked by client + + // Optional field for fileId + optional uint64 fileId = 13 [default = 0]; // default as an invalid id } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index dc77b251946..a7292e2c8e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -23,7 +23,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anyShort; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -49,13 +52,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsUtils; import org.apache.hadoop.hdfs.protocol.Block; @@ -64,12 +67,14 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; @@ -208,7 +213,7 @@ public class TestDFSClientRetries { * Verify that client will correctly give up after the specified number * of times trying to add a block */ - @SuppressWarnings("serial") + @SuppressWarnings({ "serial", "unchecked" }) @Test public void testNotYetReplicatedErrors() throws IOException { @@ -235,7 +240,22 @@ public class TestDFSClientRetries { when(mockNN.addBlock(anyString(), anyString(), any(ExtendedBlock.class), - any(DatanodeInfo[].class))).thenAnswer(answer); + any(DatanodeInfo[].class), + anyLong())).thenAnswer(answer); + + Mockito.doReturn( + new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( + (short) 777), "owner", "group", new byte[0], new byte[0], + 1010)).when(mockNN).getFileInfo(anyString()); + + Mockito.doReturn( + new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( + (short) 777), "owner", "group", new byte[0], new byte[0], + 1010)) + .when(mockNN) + .create(anyString(), (FsPermission) anyObject(), anyString(), + (EnumSetWritable) anyObject(), anyBoolean(), + anyShort(), anyLong()); final DFSClient client = new DFSClient(null, mockNN, conf, null); OutputStream os = client.create("testfile", true); @@ -369,7 +389,8 @@ public class TestDFSClientRetries { return ret2; } }).when(spyNN).addBlock(Mockito.anyString(), Mockito.anyString(), - Mockito.any(), Mockito.any()); + Mockito. any(), Mockito. any(), + Mockito.anyLong()); doAnswer(new Answer() { @@ -410,7 +431,8 @@ public class TestDFSClientRetries { // Make sure the mock was actually properly injected. Mockito.verify(spyNN, Mockito.atLeastOnce()).addBlock( Mockito.anyString(), Mockito.anyString(), - Mockito.any(), Mockito.any()); + Mockito. any(), Mockito. any(), + Mockito.anyLong()); Mockito.verify(spyNN, Mockito.atLeastOnce()).complete( Mockito.anyString(), Mockito.anyString(), Mockito.any()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index e6ce7c43f19..5a01ea57a8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.EnumSetWritable; @@ -517,8 +518,8 @@ public class TestFileCreation { + "The file has " + locations.locatedBlockCount() + " blocks."); // add one block to the file - LocatedBlock location = client.getNamenode().addBlock(file1.toString(), - client.clientName, null, null); + LocatedBlock location = client.getNamenode().addBlock(file1.toString(), + client.clientName, null, null, INodeId.GRANDFATHER_INODE_ID); System.out.println("testFileCreationError2: " + "Added block " + location.getBlock()); @@ -568,8 +569,8 @@ public class TestFileCreation { final Path f = new Path("/foo.txt"); createFile(dfs, f, 3); try { - cluster.getNameNodeRpc().addBlock(f.toString(), - client.clientName, null, null); + cluster.getNameNodeRpc().addBlock(f.toString(), client.clientName, + null, null, INodeId.GRANDFATHER_INODE_ID); fail(); } catch(IOException ioe) { FileSystem.LOG.info("GOOD!", ioe); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java index 1940b6dcd03..969e4a1fd41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hdfs; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.anyShort; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; @@ -29,14 +33,18 @@ import java.security.PrivilegedExceptionAction; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; @@ -256,6 +264,7 @@ public class TestLease { } } + @SuppressWarnings("unchecked") @Test public void testFactory() throws Exception { final String[] groups = new String[]{"supergroup"}; @@ -264,6 +273,20 @@ public class TestLease { ugi[i] = UserGroupInformation.createUserForTesting("user" + i, groups); } + Mockito.doReturn( + new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( + (short) 777), "owner", "group", new byte[0], new byte[0], + 1010)).when(mcp).getFileInfo(anyString()); + Mockito + .doReturn( + new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( + (short) 777), "owner", "group", new byte[0], new byte[0], + 1010)) + .when(mcp) + .create(anyString(), (FsPermission) anyObject(), anyString(), + (EnumSetWritable) anyObject(), anyBoolean(), + anyShort(), anyLong()); + final Configuration conf = new Configuration(); final DFSClient c1 = createDFSClientAs(ugi[0], conf); FSDataOutputStream out1 = createFsOut(c1, "/out1"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index a744b0b2f12..891c6699454 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -1058,7 +1058,8 @@ public class NNThroughputBenchmark { throws IOException { ExtendedBlock prevBlock = null; for(int jdx = 0; jdx < blocksPerFile; jdx++) { - LocatedBlock loc = nameNodeProto.addBlock(fileName, clientName, prevBlock, null); + LocatedBlock loc = nameNodeProto.addBlock(fileName, clientName, + prevBlock, null, INodeId.GRANDFATHER_INODE_ID); prevBlock = loc.getBlock(); for(DatanodeInfo dnInfo : loc.getLocations()) { int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getXferAddr()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java index 6bfdcc9b3c1..793cec6e93a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java @@ -107,7 +107,8 @@ public class TestAddBlockRetry { count++; if(count == 1) { // run second addBlock() LOG.info("Starting second addBlock for " + src); - nn.addBlock(src, "clientName", null, null); + nn.addBlock(src, "clientName", null, null, + INodeId.GRANDFATHER_INODE_ID); LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE); assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size()); lb2 = lbs.get(0); @@ -128,7 +129,7 @@ public class TestAddBlockRetry { // start first addBlock() LOG.info("Starting first addBlock for " + src); - nn.addBlock(src, "clientName", null, null); + nn.addBlock(src, "clientName", null, null, INodeId.GRANDFATHER_INODE_ID); // check locations LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 0d1486886c5..9d5efb33498 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -24,8 +24,10 @@ import static org.junit.Assert.fail; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.EnumSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options; @@ -39,6 +41,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.EnumSetWritable; import org.junit.Test; public class TestINodeFile { @@ -376,7 +380,7 @@ public class TestINodeFile { * @throws IOException */ @Test - public void TestInodeId() throws IOException { + public void testInodeId() throws IOException { Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, @@ -396,9 +400,14 @@ public class TestINodeFile { assertTrue(fs.mkdirs(path)); assertTrue(fsn.getLastInodeId() == 1002); - Path filePath = new Path("/test1/file"); - fs.create(filePath); + // Use namenode rpc to create a file + NamenodeProtocols nnrpc = cluster.getNameNodeRpc(); + HdfsFileStatus fileStatus = nnrpc.create("/test1/file", new FsPermission( + (short) 0755), "client", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, + (short) 1, 128 * 1024 * 1024L); assertTrue(fsn.getLastInodeId() == 1003); + assertTrue(fileStatus.getFileId() == 1003); // Rename doesn't increase inode id Path renamedPath = new Path("/test2"); @@ -412,4 +421,44 @@ public class TestINodeFile { cluster.waitActive(); assertTrue(fsn.getLastInodeId() == 1003); } + + @Test + public void testWriteToRenamedFile() throws IOException { + + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + Path path = new Path("/test1"); + assertTrue(fs.mkdirs(path)); + + int size = conf.getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512); + byte[] data = new byte[size]; + + // Create one file + Path filePath = new Path("/test1/file"); + FSDataOutputStream fos = fs.create(filePath); + + // Rename /test1 to test2, and recreate /test1/file + Path renamedPath = new Path("/test2"); + fs.rename(path, renamedPath); + fs.create(filePath, (short) 1); + + // Add new block should fail since /test1/file has a different fileId + try { + fos.write(data, 0, data.length); + // make sure addBlock() request gets to NN immediately + fos.hflush(); + + fail("Write should fail after rename"); + } catch (Exception e) { + /* Ignore */ + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java index aa2393d7c01..4e77c07fed5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.util.Time; import org.junit.Assert; import org.junit.Test; @@ -42,9 +43,10 @@ public class TestJsonUtil { public void testHdfsFileStatus() { final long now = Time.now(); final String parent = "/dir"; - final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L<<26, - now, now + 10, new FsPermission((short)0644), "user", "group", - DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo")); + final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26, + now, now + 10, new FsPermission((short) 0644), "user", "group", + DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"), + INodeId.GRANDFATHER_INODE_ID); final FileStatus fstatus = toFileStatus(status, parent); System.out.println("status = " + status); System.out.println("fstatus = " + fstatus); From ea94b7b9ddbe154afc06e2b932d247716d55bcc8 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 6 Feb 2013 22:47:10 +0000 Subject: [PATCH 38/69] MAPREDUCE-4822. Unnecessary conversions in History Events. Contributed by Chu Tong git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443257 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/jobhistory/MapAttemptFinishedEvent.java | 2 +- .../jobhistory/ReduceAttemptFinishedEvent.java | 4 ++-- .../mapreduce/jobhistory/TaskAttemptFinishedEvent.java | 4 ++-- .../hadoop/mapreduce/jobhistory/TaskFinishedEvent.java | 10 +++------- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 37ccdda2c0e..a54ba67f187 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -696,6 +696,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the number of map completion event type conversions. (Jason Lowe via sseth) + MAPREDUCE-4822. Unnecessary conversions in History Events. (Chu Tong via + jlowe) + BUG FIXES MAPREDUCE-4458. Warn if java.library.path is used for AM or Task diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index d4f9fa316b9..62df2aae929 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -178,7 +178,7 @@ public class MapAttemptFinishedEvent implements HistoryEvent { /** Get the task type */ public TaskType getTaskType() { - return TaskType.valueOf(taskType.toString()); + return taskType; } /** Get the task status */ public String getTaskStatus() { return taskStatus.toString(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index 10b8c1f0139..a779fca6ec8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -176,11 +176,11 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { public TaskID getTaskId() { return attemptId.getTaskID(); } /** Get the attempt id */ public TaskAttemptID getAttemptId() { - return TaskAttemptID.forName(attemptId.toString()); + return attemptId; } /** Get the task type */ public TaskType getTaskType() { - return TaskType.valueOf(taskType.toString()); + return taskType; } /** Get the task status */ public String getTaskStatus() { return taskStatus.toString(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java index a62ca38e4a8..78b9ca914f6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java @@ -105,11 +105,11 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { public TaskID getTaskId() { return attemptId.getTaskID(); } /** Get the task attempt id */ public TaskAttemptID getAttemptId() { - return TaskAttemptID.forName(attemptId.toString()); + return attemptId; } /** Get the task type */ public TaskType getTaskType() { - return TaskType.valueOf(taskType.toString()); + return taskType; } /** Get the task status */ public String getTaskStatus() { return taskStatus.toString(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index 55de80ca63f..edbf0099ff9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -95,14 +95,10 @@ public class TaskFinishedEvent implements HistoryEvent { } /** Get task id */ - public TaskID getTaskId() { return TaskID.forName(taskid.toString()); } + public TaskID getTaskId() { return taskid; } /** Get successful task attempt id */ public TaskAttemptID getSuccessfulTaskAttemptId() { - if(successfulAttemptId != null) - { - return TaskAttemptID.forName(successfulAttemptId.toString()); - } - return null; + return successfulAttemptId; } /** Get the task finish time */ public long getFinishTime() { return finishTime; } @@ -110,7 +106,7 @@ public class TaskFinishedEvent implements HistoryEvent { public Counters getCounters() { return counters; } /** Get task type */ public TaskType getTaskType() { - return TaskType.valueOf(taskType.toString()); + return taskType; } /** Get task status */ public String getTaskStatus() { return status.toString(); } From 90c3700c5286cf2a5dcc850a386e2c674eb13acd Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 7 Feb 2013 03:04:18 +0000 Subject: [PATCH 39/69] HDFS-4458. In DFSUtil.getNameServiceUris(..), convert default fs URI using NetUtils.createSocketAddr(..) for being consistent with other addresses. Contributed by Binglin Chang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443297 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../src/main/java/org/apache/hadoop/hdfs/DFSUtil.java | 7 +++++++ .../test/java/org/apache/hadoop/hdfs/TestDFSUtil.java | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6c865bfa687..9b6b86a3cae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -311,6 +311,10 @@ Release 2.0.4-beta - UNRELEASED BUG FIXES + HDFS-4458. In DFSUtil.getNameServiceUris(..), convert default fs URI using + NetUtils.createSocketAddr(..) for being consistent with other addresses. + (Binglin Chang via szetszwo) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 39f839246fc..15d8864e905 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -766,6 +766,13 @@ public class DFSUtil { // Add the default URI if it is an HDFS URI. URI defaultUri = FileSystem.getDefaultUri(conf); + // checks if defaultUri is ip:port format + // and convert it to hostname:port format + if (defaultUri != null && (defaultUri.getPort() != -1)) { + defaultUri = createUri(defaultUri.getScheme(), + NetUtils.createSocketAddr(defaultUri.getHost(), + defaultUri.getPort())); + } if (defaultUri != null && HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) && !nonPreferredUris.contains(defaultUri)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index d59acfa6ec4..8482f81ddb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -619,6 +619,16 @@ public class TestDFSUtil { assertEquals(1, uris.size()); assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR))); + + // Make sure when config FS_DEFAULT_NAME_KEY using IP address, + // it will automatically convert it to hostname + conf = new HdfsConfiguration(); + conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:8020"); + uris = DFSUtil.getNameServiceUris(conf); + assertEquals(1, uris.size()); + for (URI uri : uris) { + assertFalse(uri.getHost().equals("127.0.0.1")); + } } @Test From 411bb0bd991344c0cd01bda1de27962f0470e944 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 7 Feb 2013 03:13:14 +0000 Subject: [PATCH 40/69] HADOOP-9283. Add support for running the Hadoop client on AIX. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443303 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../hadoop/security/UserGroupInformation.java | 65 ++++++++++++++----- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 642a686593a..bcba456bab8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -345,6 +345,8 @@ Release 2.0.4-beta - UNRELEASED NEW FEATURES + HADOOP-9283. Add support for running the Hadoop client on AIX. (atm) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index b7f87e5ea89..7983013539c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -301,17 +301,25 @@ public class UserGroupInformation { private static String OS_LOGIN_MODULE_NAME; private static Class OS_PRINCIPAL_CLASS; + private static final boolean windows = System.getProperty("os.name").startsWith("Windows"); private static final boolean is64Bit = System.getProperty("os.arch").contains("64"); + private static final boolean ibmJava = System.getProperty("java.vendor").contains("IBM"); + private static final boolean aix = System.getProperty("os.name").equals("AIX"); + /* Return the OS login module class name */ private static String getOSLoginModuleName() { - if (System.getProperty("java.vendor").contains("IBM")) { - return windows ? (is64Bit - ? "com.ibm.security.auth.module.Win64LoginModule" - : "com.ibm.security.auth.module.NTLoginModule") - : "com.ibm.security.auth.module.LinuxLoginModule"; + if (ibmJava) { + if (windows) { + return is64Bit ? "com.ibm.security.auth.module.Win64LoginModule" + : "com.ibm.security.auth.module.NTLoginModule"; + } else if (aix) { + return "com.ibm.security.auth.module.AIXLoginModule"; + } else { + return "com.ibm.security.auth.module.LinuxLoginModule"; + } } else { return windows ? "com.sun.security.auth.module.NTLoginModule" : "com.sun.security.auth.module.UnixLoginModule"; @@ -323,11 +331,14 @@ public class UserGroupInformation { private static Class getOsPrincipalClass() { ClassLoader cl = ClassLoader.getSystemClassLoader(); try { - if (System.getProperty("java.vendor").contains("IBM")) { + if (ibmJava) { if (windows) { return (Class) (is64Bit ? cl.loadClass("com.ibm.security.auth.UsernamePrincipal") : cl.loadClass("com.ibm.security.auth.NTUserPrincipal")); + } else if (aix) { + return (Class) + cl.loadClass("com.ibm.security.auth.AIXPrincipal"); } else { return (Class) (is64Bit ? cl.loadClass("com.ibm.security.auth.UsernamePrincipal") @@ -418,12 +429,21 @@ public class UserGroupInformation { private static final Map USER_KERBEROS_OPTIONS = new HashMap(); static { - USER_KERBEROS_OPTIONS.put("doNotPrompt", "true"); - USER_KERBEROS_OPTIONS.put("useTicketCache", "true"); - USER_KERBEROS_OPTIONS.put("renewTGT", "true"); + if (ibmJava) { + USER_KERBEROS_OPTIONS.put("useDefaultCcache", "true"); + } else { + USER_KERBEROS_OPTIONS.put("doNotPrompt", "true"); + USER_KERBEROS_OPTIONS.put("useTicketCache", "true"); + USER_KERBEROS_OPTIONS.put("renewTGT", "true"); + } String ticketCache = System.getenv("KRB5CCNAME"); if (ticketCache != null) { - USER_KERBEROS_OPTIONS.put("ticketCache", ticketCache); + if (ibmJava) { + // The first value searched when "useDefaultCcache" is used. + System.setProperty("KRB5CCNAME", ticketCache); + } else { + USER_KERBEROS_OPTIONS.put("ticketCache", ticketCache); + } } USER_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); } @@ -434,10 +454,14 @@ public class UserGroupInformation { private static final Map KEYTAB_KERBEROS_OPTIONS = new HashMap(); static { - KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true"); - KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); - KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true"); - KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true"); + if (ibmJava) { + KEYTAB_KERBEROS_OPTIONS.put("credsType", "both"); + } else { + KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true"); + KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); + KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true"); + KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true"); + } KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); } private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = @@ -462,7 +486,12 @@ public class UserGroupInformation { } else if (USER_KERBEROS_CONFIG_NAME.equals(appName)) { return USER_KERBEROS_CONF; } else if (KEYTAB_KERBEROS_CONFIG_NAME.equals(appName)) { - KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile); + if (ibmJava) { + KEYTAB_KERBEROS_OPTIONS.put("useKeytab", + prependFileAuthority(keytabFile)); + } else { + KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile); + } KEYTAB_KERBEROS_OPTIONS.put("principal", keytabPrincipal); return KEYTAB_KERBEROS_CONF; } @@ -470,6 +499,11 @@ public class UserGroupInformation { } } + private static String prependFileAuthority(String keytabPath) { + return keytabPath.startsWith("file://") ? keytabPath + : "file://" + keytabPath; + } + /** * Represents a javax.security configuration that is created at runtime. */ @@ -666,6 +700,7 @@ public class UserGroupInformation { } loginUser.spawnAutoRenewalThreadForUserCreds(); } catch (LoginException le) { + LOG.debug("failure to login", le); throw new IOException("failure to login", le); } if (LOG.isDebugEnabled()) { From 1eaf9944e3a881c65565562a7756e172cf853cf4 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 7 Feb 2013 03:17:51 +0000 Subject: [PATCH 41/69] Move HDFS-4458 to 2.0.3-alpha in CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443309 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9b6b86a3cae..1e13a49e683 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -311,10 +311,6 @@ Release 2.0.4-beta - UNRELEASED BUG FIXES - HDFS-4458. In DFSUtil.getNameServiceUris(..), convert default fs URI using - NetUtils.createSocketAddr(..) for being consistent with other addresses. - (Binglin Chang via szetszwo) - Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES @@ -784,6 +780,10 @@ Release 2.0.3-alpha - 2013-02-06 HDFS-4468. Use the new StringUtils methods added by HADOOP-9252 and fix TestHDFSCLI and TestQuota. (szetszwo) + HDFS-4458. In DFSUtil.getNameServiceUris(..), convert default fs URI using + NetUtils.createSocketAddr(..) for being consistent with other addresses. + (Binglin Chang via szetszwo) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. From 35832053bf46f77a6350ef8e716a67f2a374b1a0 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 7 Feb 2013 07:07:11 +0000 Subject: [PATCH 42/69] MAPREDUCE-4671. AM does not tell the RM about container requests which are no longer needed. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443323 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../v2/app/rm/RMContainerRequestor.java | 35 ++++++++++++++----- .../v2/app/TestRMContainerAllocator.java | 21 +++++++++-- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a54ba67f187..d2dcb9f44d4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -164,6 +164,9 @@ Release 2.0.4-beta - UNRELEASED BUG FIXES + MAPREDUCE-4671. AM does not tell the RM about container requests which are + no longer needed. (Bikas Saha via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 59c9795e2d8..5f3a7f5f960 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -72,7 +72,10 @@ public abstract class RMContainerRequestor extends RMCommunicator { remoteRequestsTable = new TreeMap>>(); - private final Set ask = new TreeSet(); + // use custom comparator to make sure ResourceRequest objects differing only in + // numContainers dont end up as duplicates + private final Set ask = new TreeSet( + new org.apache.hadoop.yarn.util.BuilderUtils.ResourceRequestComparator()); private final Set release = new TreeSet(); private boolean nodeBlacklistingEnabled; @@ -235,7 +238,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { ResourceRequest zeroedRequest = BuilderUtils.newResourceRequest(req); zeroedRequest.setNumContainers(0); // to be sent to RM on next heartbeat - ask.add(zeroedRequest); + addResourceRequestToAsk(zeroedRequest); } } // if all requests were still in ask queue @@ -320,7 +323,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1); // Note this down for next interaction with ResourceManager - ask.add(remoteRequest); + addResourceRequestToAsk(remoteRequest); if (LOG.isDebugEnabled()) { LOG.debug("addResourceRequest:" + " applicationId=" + applicationId.getId() + " priority=" + priority.getPriority() @@ -353,7 +356,12 @@ public abstract class RMContainerRequestor extends RMCommunicator { + remoteRequest.getNumContainers() + " #asks=" + ask.size()); } - remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1); + if(remoteRequest.getNumContainers() > 0) { + // based on blacklisting comments above we can end up decrementing more + // than requested. so guard for that. + remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1); + } + if (remoteRequest.getNumContainers() == 0) { reqMap.remove(capability); if (reqMap.size() == 0) { @@ -362,13 +370,12 @@ public abstract class RMContainerRequestor extends RMCommunicator { if (remoteRequests.size() == 0) { remoteRequestsTable.remove(priority); } - //remove from ask if it may have - ask.remove(remoteRequest); - } else { - ask.add(remoteRequest);//this will override the request if ask doesn't - //already have it. } + // send the updated resource request to RM + // send 0 container count requests also to cancel previous requests + addResourceRequestToAsk(remoteRequest); + if (LOG.isDebugEnabled()) { LOG.info("AFTER decResourceRequest:" + " applicationId=" + applicationId.getId() + " priority=" + priority.getPriority() @@ -376,6 +383,16 @@ public abstract class RMContainerRequestor extends RMCommunicator { + remoteRequest.getNumContainers() + " #asks=" + ask.size()); } } + + private void addResourceRequestToAsk(ResourceRequest remoteRequest) { + // because objects inside the resource map can be deleted ask can end up + // containing an object that matches new resource object but with different + // numContainers. So exisintg values must be replaced explicitly + if(ask.contains(remoteRequest)) { + ask.remove(remoteRequest); + } + ask.add(remoteRequest); + } protected void release(ContainerId containerId) { release.add(containerId); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index bd00e1b1608..47845a0aa6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -167,6 +167,7 @@ public class TestRMContainerAllocator { List assigned = allocator.schedule(); dispatcher.await(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); + Assert.assertEquals(4, rm.getMyFifoScheduler().lastAsk.size()); // send another request with different resource and priority ContainerRequestEvent event3 = createReq(jobId, 3, 1024, @@ -178,7 +179,8 @@ public class TestRMContainerAllocator { assigned = allocator.schedule(); dispatcher.await(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); - + Assert.assertEquals(3, rm.getMyFifoScheduler().lastAsk.size()); + // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat nodeManager2.nodeHeartbeat(true); // Node heartbeat @@ -187,8 +189,14 @@ public class TestRMContainerAllocator { assigned = allocator.schedule(); dispatcher.await(); + Assert.assertEquals(0, rm.getMyFifoScheduler().lastAsk.size()); checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 }, assigned, false); + + // check that the assigned container requests are cancelled + assigned = allocator.schedule(); + dispatcher.await(); + Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size()); } @Test @@ -422,7 +430,7 @@ public class TestRMContainerAllocator { } private static class MyResourceManager extends MockRM { - + public MyResourceManager(Configuration conf) { super(conf); } @@ -446,6 +454,10 @@ public class TestRMContainerAllocator { protected ResourceScheduler createScheduler() { return new MyFifoScheduler(this.getRMContext()); } + + MyFifoScheduler getMyFifoScheduler() { + return (MyFifoScheduler) scheduler; + } } @Test @@ -1194,7 +1206,9 @@ public class TestRMContainerAllocator { assert (false); } } - + + List lastAsk = null; + // override this to copy the objects otherwise FifoScheduler updates the // numContainers in same objects as kept by RMContainerAllocator @Override @@ -1208,6 +1222,7 @@ public class TestRMContainerAllocator { .getNumContainers()); askCopy.add(reqCopy); } + lastAsk = ask; return super.allocate(applicationAttemptId, askCopy, release); } } From 54a9c6f964482d0a58075a482721ae48c3acd3b8 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 7 Feb 2013 14:52:58 +0000 Subject: [PATCH 43/69] HADOOP-9253. Capture ulimit info in the logs at service start time. Contributed by Arpit Gupta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443517 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../hadoop-common/src/main/bin/hadoop-daemon.sh | 15 +++++++++++++-- .../hadoop-yarn/bin/yarn-daemon.sh | 6 +++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index bcba456bab8..af7766ba8dd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -348,6 +348,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9283. Add support for running the Hadoop client on AIX. (atm) IMPROVEMENTS + HADOOP-9253. Capture ulimit info in the logs at service start time. + (Arpit Gupta via suresh) OPTIMIZATIONS diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh index 3e5f20ba20c..bf03d195e06 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh @@ -83,7 +83,8 @@ fi if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then export HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR export HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR - export HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER + export HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER + starting_secure_dn="true" fi if [ "$HADOOP_IDENT_STRING" = "" ]; then @@ -154,7 +155,17 @@ case $startStop in ;; esac echo $! > $pid - sleep 1; head "$log" + sleep 1 + # capture the ulimit output + if [ "true" = "$starting_secure_dn" ]; then + echo "ulimit -a for secure datanode user $HADOOP_SECURE_DN_USER" >> $log + # capture the ulimit info for the appropriate user + su --shell=/bin/bash $HADOOP_SECURE_DN_USER -c 'ulimit -a' >> $log 2>&1 + else + echo "ulimit -a for user $USER" >> $log + ulimit -a >> $log 2>&1 + fi + head -30 "$log" sleep 3; if ! ps -p $! > /dev/null ; then exit 1 diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh index 2df1044670f..de670facfac 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh @@ -123,7 +123,11 @@ case $startStop in cd "$HADOOP_YARN_HOME" nohup nice -n $YARN_NICENESS "$HADOOP_YARN_HOME"/bin/yarn --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & echo $! > $pid - sleep 1; head "$log" + sleep 1 + # capture the ulimit output + echo "ulimit -a" >> $log + ulimit -a >> $log 2>&1 + head -30 "$log" ;; (stop) From bdfe23235e73a37e319e9f383a6d96e6004bf798 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 7 Feb 2013 20:21:38 +0000 Subject: [PATCH 44/69] YARN-383. AMRMClientImpl should handle null rmClient in stop(). Contributed by Hitesh Shah. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443699 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../dev-support/findbugs-exclude.xml | 7 +++++++ .../dev-support/findbugs-exclude.xml | 19 ------------------- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 5 +++++ .../hadoop/yarn/client/AMRMClientImpl.java | 4 +++- hadoop-yarn-project/pom.xml | 11 ----------- 6 files changed, 18 insertions(+), 31 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/dev-support/findbugs-exclude.xml diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 620215692d4..e0e575b4918 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -26,6 +26,9 @@ Release 2.0.4-beta - UNRELEASED BUG FIXES + YARN-383. AMRMClientImpl should handle null rmClient in stop() + (Hitesh Shah via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index bee8ce24e32..2d63dad48b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -249,4 +249,11 @@ + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/dev-support/findbugs-exclude.xml deleted file mode 100644 index 0e037a2ad04..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/dev-support/findbugs-exclude.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 97ff1f7b5bf..b2a59b93467 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -24,6 +24,11 @@ 3.0.0-SNAPSHOT hadoop-yarn-client + + + ${project.parent.basedir} + + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java index 15c250e8d5f..42b5adbbbff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java @@ -147,7 +147,9 @@ public class AMRMClientImpl extends AbstractService implements AMRMClient { @Override public synchronized void stop() { - RPC.stopProxy(this.rmClient); + if (this.rmClient != null) { + RPC.stopProxy(this.rmClient); + } super.stop(); } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index a0eed4838c2..4366b387e13 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -33,7 +33,6 @@ true 600000 once - ${basedir} yarn true @@ -200,16 +199,6 @@ - org.codehaus.mojo - findbugs-maven-plugin - - true - true - ${mr.basedir}/dev-support/findbugs-exclude.xml - Max - - - org.apache.rat apache-rat-plugin From 362da383fd273b2f403f0504a6ad7837421f6854 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 7 Feb 2013 20:28:20 +0000 Subject: [PATCH 45/69] YARN-385. Add missing fields - location and #containers to ResourceRequestPBImpl's toString(). Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443702 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e0e575b4918..6795817ddfa 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -29,6 +29,9 @@ Release 2.0.4-beta - UNRELEASED YARN-383. AMRMClientImpl should handle null rmClient in stop() (Hitesh Shah via sseth) + YARN-385. Add missing fields - location and #containers to + ResourceRequestPBImpl's toString(). (Sandy Ryza via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index f3834a4db6e..ba064c73f6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -166,6 +166,7 @@ public class ResourceRequestPBImpl extends ResourceRequest { @Override public String toString() { return "{Priority: " + getPriority() + ", Capability: " + getCapability() - + "}"; + + ", # Containers: " + getNumContainers() + + ", Location: " + getHostName() + "}"; } } \ No newline at end of file From c3d09010c759b20f6d2d5ab0ec36a40ac855951b Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 7 Feb 2013 20:38:29 +0000 Subject: [PATCH 46/69] HADOOP-9277. Improve javadoc for FileContext. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443710 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../org/apache/hadoop/fs/FileContext.java | 116 ++++++++---------- 2 files changed, 55 insertions(+), 63 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index af7766ba8dd..c96dc1aac61 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -149,6 +149,8 @@ Trunk (Unreleased) HADOOP-8924. Add maven plugin alternative to shell script to save package-info.java. (Chris Nauroth via suresh) + HADOOP-9277. Improve javadoc for FileContext. (Andrew Wang via suresh) + BUG FIXES HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 978bb1ba0dd..d4ff03785c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -57,70 +57,60 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ShutdownHookManager; /** - * The FileContext class provides an interface to the application writer for - * using the Hadoop file system. - * It provides a set of methods for the usual operation: create, open, - * list, etc + * The FileContext class provides an interface for users of the Hadoop + * file system. It exposes a number of file system operations, e.g. create, + * open, list. * - *

- * *** Path Names *** - *

+ *

Path Names

* - * The Hadoop file system supports a URI name space and URI names. - * It offers a forest of file systems that can be referenced using fully - * qualified URIs. - * Two common Hadoop file systems implementations are + * The Hadoop file system supports a URI namespace and URI names. This enables + * multiple types of file systems to be referenced using fully-qualified URIs. + * Two common Hadoop file system implementations are *
    - *
  • the local file system: file:///path - *
  • the hdfs file system hdfs://nnAddress:nnPort/path + *
  • the local file system: file:///path + *
  • the HDFS file system: hdfs://nnAddress:nnPort/path *
* - * While URI names are very flexible, it requires knowing the name or address - * of the server. For convenience one often wants to access the default system - * in one's environment without knowing its name/address. This has an - * additional benefit that it allows one to change one's default fs - * (e.g. admin moves application from cluster1 to cluster2). + * The Hadoop file system also supports additional naming schemes besides URIs. + * Hadoop has the concept of a default file system, which implies a + * default URI scheme and authority. This enables slash-relative names + * relative to the default FS, which are more convenient for users and + * application writers. The default FS is typically set by the user's + * environment, though it can also be manually specified. *

* - * To facilitate this, Hadoop supports a notion of a default file system. - * The user can set his default file system, although this is - * typically set up for you in your environment via your default config. - * A default file system implies a default scheme and authority; slash-relative - * names (such as /for/bar) are resolved relative to that default FS. - * Similarly a user can also have working-directory-relative names (i.e. names - * not starting with a slash). While the working directory is generally in the - * same default FS, the wd can be in a different FS. + * Hadoop also supports working-directory-relative names, which are paths + * relative to the current working directory (similar to Unix). The working + * directory can be in a different file system than the default FS. *

- * Hence Hadoop path names can be one of: - *

    - *
  • fully qualified URI: scheme://authority/path - *
  • slash relative names: /path relative to the default file system - *
  • wd-relative names: path relative to the working dir - *
+ * Thus, Hadoop path names can be specified as one of the following: + *
    + *
  • a fully-qualified URI: scheme://authority/path (e.g. + * hdfs://nnAddress:nnPort/foo/bar) + *
  • a slash-relative name: path relative to the default file system (e.g. + * /foo/bar) + *
  • a working-directory-relative name: path relative to the working dir (e.g. + * foo/bar) + *
* Relative paths with scheme (scheme:foo/bar) are illegal. * - *

- * ****The Role of the FileContext and configuration defaults**** - *

- * The FileContext provides file namespace context for resolving file names; - * it also contains the umask for permissions, In that sense it is like the - * per-process file-related state in Unix system. - * These two properties - *

    - *
  • default file system i.e your slash) - *
  • umask - *
- * in general, are obtained from the default configuration file - * in your environment, (@see {@link Configuration}). - * - * No other configuration parameters are obtained from the default config as - * far as the file context layer is concerned. All file system instances - * (i.e. deployments of file systems) have default properties; we call these - * server side (SS) defaults. Operation like create allow one to select many - * properties: either pass them in as explicit parameters or use - * the SS properties. - *

- * The file system related SS defaults are + *

Role of FileContext and Configuration Defaults

+ * + * The FileContext is the analogue of per-process file-related state in Unix. It + * contains two properties: + * + *
    + *
  • the default file system (for resolving slash-relative names) + *
  • the umask (for file permissions) + *
+ * In general, these properties are obtained from the default configuration file + * in the user's environment (see {@link Configuration}). + * + * Further file system properties are specified on the server-side. File system + * operations default to using these server-side defaults unless otherwise + * specified. + *

+ * The file system related server-side defaults are: *

    *
  • the home directory (default is "/user/userName") *
  • the initial wd (only for local fs) @@ -131,34 +121,34 @@ import org.apache.hadoop.util.ShutdownHookManager; *
  • checksum option. (checksumType and bytesPerChecksum) *
* - *

- * *** Usage Model for the FileContext class *** - *

+ *

Example Usage

+ * * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. * Unspecified values come from core-defaults.xml in the release jar. *
    *
  • myFContext = FileContext.getFileContext(); // uses the default config * // which has your default FS *
  • myFContext.create(path, ...); - *
  • myFContext.setWorkingDir(path) + *
  • myFContext.setWorkingDir(path); *
  • myFContext.open (path, ...); + *
  • ... *
* Example 2: Get a FileContext with a specific URI as the default FS *
    - *
  • myFContext = FileContext.getFileContext(URI) + *
  • myFContext = FileContext.getFileContext(URI); *
  • myFContext.create(path, ...); - * ... - *
+ *
  • ... + * * Example 3: FileContext with local file system as the default *
      - *
    • myFContext = FileContext.getLocalFSFileContext() + *
    • myFContext = FileContext.getLocalFSFileContext(); *
    • myFContext.create(path, ...); *
    • ... *
    * Example 4: Use a specific config, ignoring $HADOOP_CONFIG * Generally you should not need use a config unless you are doing *
      - *
    • configX = someConfigSomeOnePassedToYou. + *
    • configX = someConfigSomeOnePassedToYou; *
    • myFContext = getFileContext(configX); // configX is not changed, * // is passed down *
    • myFContext.create(path, ...); From 7114a61318000a22402f61aa7e7732c85d388a28 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 7 Feb 2013 21:52:55 +0000 Subject: [PATCH 47/69] HDFS-4470. Several HDFS tests attempt file operations on invalid HDFS paths when running on Windows. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443744 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/fs/TestListFiles.java | 38 ++++++++++++++----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/TestFileLengthOnClusterRestart.java | 2 +- .../apache/hadoop/hdfs/TestLargeBlock.java | 3 +- .../hadoop/hdfs/TestListFilesInDFS.java | 1 + .../TestRBWBlockInvalidation.java | 2 +- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java index aae013fd775..df519c84e8c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java @@ -45,19 +45,39 @@ public class TestListFiles { final protected static Configuration conf = new Configuration(); protected static FileSystem fs; - final protected static Path TEST_DIR = getTestDir(); + protected static Path TEST_DIR; final private static int FILE_LEN = 10; - final private static Path FILE1 = new Path(TEST_DIR, "file1"); - final private static Path DIR1 = new Path(TEST_DIR, "dir1"); - final private static Path FILE2 = new Path(DIR1, "file2"); - final private static Path FILE3 = new Path(DIR1, "file3"); + private static Path FILE1; + private static Path DIR1; + private static Path FILE2; + private static Path FILE3; + + static { + setTestPaths(new Path( + System.getProperty("test.build.data", "build/test/data/work-dir/localfs"), + "main_")); + } protected static Path getTestDir() { - return new Path( - System.getProperty("test.build.data","build/test/data/work-dir/localfs"), - "main_"); + return TEST_DIR; } - + + /** + * Sets the root testing directory and reinitializes any additional test paths + * that are under the root. This method is intended to be called from a + * subclass's @BeforeClass method if there is a need to override the testing + * directory. + * + * @param testDir Path root testing directory + */ + protected static void setTestPaths(Path testDir) { + TEST_DIR = testDir; + FILE1 = new Path(TEST_DIR, "file1"); + DIR1 = new Path(TEST_DIR, "dir1"); + FILE2 = new Path(DIR1, "file2"); + FILE3 = new Path(DIR1, "file3"); + } + @BeforeClass public static void testSetUp() throws Exception { fs = FileSystem.getLocal(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1e13a49e683..6b31b482622 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -310,6 +310,9 @@ Release 2.0.4-beta - UNRELEASED OPTIMIZATIONS BUG FIXES + + HDFS-4470. Several HDFS tests attempt file operations on invalid HDFS + paths when running on Windows. (Chris Nauroth via suresh) Release 2.0.3-alpha - 2013-02-06 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java index f63ba9a53a7..8bafee67f85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileLengthOnClusterRestart.java @@ -43,7 +43,7 @@ public class TestFileLengthOnClusterRestart { .numDataNodes(2).build(); HdfsDataInputStream in = null; try { - Path path = new Path(MiniDFSCluster.getBaseDirectory(), "test"); + Path path = new Path("/tmp/TestFileLengthOnClusterRestart", "test"); DistributedFileSystem dfs = (DistributedFileSystem) cluster .getFileSystem(); FSDataOutputStream out = dfs.create(path); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java index 9563361094c..64c5ef4c056 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java @@ -183,8 +183,7 @@ public class TestLargeBlock { try { // create a new file in test data directory - Path file1 = new Path(System.getProperty("test.build.data") + "/" + - Long.toString(blockSize) + ".dat"); + Path file1 = new Path("/tmp/TestLargeBlock", blockSize + ".dat"); FSDataOutputStream stm = createFile(fs, file1, 1, blockSize); LOG.info("File " + file1 + " created with file size " + fileSize + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListFilesInDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListFilesInDFS.java index ec9e7e2e481..d68563dec87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListFilesInDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListFilesInDFS.java @@ -38,6 +38,7 @@ public class TestListFilesInDFS extends TestListFiles { @BeforeClass public static void testSetUp() throws Exception { + setTestPaths(new Path("/tmp/TestListFilesInDFS")); cluster = new MiniDFSCluster.Builder(conf).build(); fs = cluster.getFileSystem(); fs.delete(TEST_DIR, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java index c07fae4773b..0b7eaeeed9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java @@ -67,7 +67,7 @@ public class TestRBWBlockInvalidation { try { final FSNamesystem namesystem = cluster.getNamesystem(); FileSystem fs = cluster.getFileSystem(); - Path testPath = new Path(MiniDFSCluster.getBaseDirectory(), "foo1"); + Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1"); out = fs.create(testPath, (short) 2); out.writeBytes("HDFS-3157: " + testPath); out.hsync(); From a63d50f79a8cc348e1f5dd84545aeb7766147a4d Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 7 Feb 2013 23:53:49 +0000 Subject: [PATCH 48/69] YARN-377. Use the new StringUtils methods added by HADOOP-9252 and fix TestContainersMonitor. Contributed by Chris Nauroth git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443796 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../monitor/ContainersMonitorImpl.java | 18 +++++++++--------- .../monitor/TestContainersMonitor.java | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6795817ddfa..129594cf45a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -32,6 +32,9 @@ Release 2.0.4-beta - UNRELEASED YARN-385. Add missing fields - location and #containers to ResourceRequestPBImpl's toString(). (Sandy Ryza via sseth) + YARN-377. Use the new StringUtils methods added by HADOOP-9252 and fix + TestContainersMonitor. (Chris Nauroth via szetszwo) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 62d6afc557d..d73f52c588d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -28,7 +28,7 @@ import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -125,10 +125,10 @@ public class ContainersMonitorImpl extends AbstractService implements this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) { LOG.warn("NodeManager configured with " + - StringUtils.humanReadableInt(maxPmemAllottedForContainers) + + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers, "", 1) + " physical memory allocated to containers, which is more than " + "80% of the total physical memory available (" + - StringUtils.humanReadableInt(totalPhysicalMemoryOnNM) + + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "", 1) + "). Thrashing might happen."); } @@ -493,12 +493,12 @@ public class ContainersMonitorImpl extends AbstractService implements private String formatUsageString(long currentVmemUsage, long vmemLimit, long currentPmemUsage, long pmemLimit) { - return String.format("%sb of %sb physical memory used; " + - "%sb of %sb virtual memory used", - StringUtils.humanReadableInt(currentPmemUsage), - StringUtils.humanReadableInt(pmemLimit), - StringUtils.humanReadableInt(currentVmemUsage), - StringUtils.humanReadableInt(vmemLimit)); + return String.format("%sB of %sB physical memory used; " + + "%sB of %sB virtual memory used", + TraditionalBinaryPrefix.long2String(currentPmemUsage, "", 1), + TraditionalBinaryPrefix.long2String(pmemLimit, "", 1), + TraditionalBinaryPrefix.long2String(currentVmemUsage, "", 1), + TraditionalBinaryPrefix.long2String(vmemLimit, "", 1)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 16db4a7fd91..41456fde9bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -267,8 +267,8 @@ public class TestContainersMonitor extends BaseContainerManagerTest { String expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: " - + "[0-9.]+m?b of [0-9.]+m?b physical memory used; " - + "[0-9.]+m?b of [0-9.]+m?b virtual memory used. " + + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for " + cId + " :\n"; Pattern pat = Pattern.compile(expectedMsgPattern); From 6f0c56cb9563a87de75dc0fbbff93b5855b61f56 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 8 Feb 2013 00:28:00 +0000 Subject: [PATCH 49/69] HDFS-4471. Namenode WebUI file browsing does not work with wildcard addresses configured. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1443807 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/NamenodeJspHelper.java | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6b31b482622..79872e1557f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -314,6 +314,9 @@ Release 2.0.4-beta - UNRELEASED HDFS-4470. Several HDFS tests attempt file operations on invalid HDFS paths when running on Windows. (Chris Nauroth via suresh) + HDFS-4471. Namenode WebUI file browsing does not work with wildcard + addresses configured. (Andrew Wang via atm) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index 005ba6a51ec..64700b721ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -25,6 +25,7 @@ import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.URI; import java.net.URLEncoder; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -453,7 +454,13 @@ class NamenodeJspHelper { nodeToRedirect = nn.getHttpAddress().getHostName(); redirectPort = nn.getHttpAddress().getPort(); } - String addr = nn.getNameNodeAddressHostPortString(); + + InetSocketAddress rpcAddr = nn.getNameNodeAddress(); + String rpcHost = rpcAddr.getAddress().isAnyLocalAddress() + ? URI.create(request.getRequestURL().toString()).getHost() + : rpcAddr.getAddress().getHostAddress(); + String addr = rpcHost + ":" + rpcAddr.getPort(); + String fqdn = InetAddress.getByName(nodeToRedirect).getCanonicalHostName(); redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":" + redirectPort + "/browseDirectory.jsp?namenodeInfoPort=" From d4931241ab1494a88af441553055a099ef5d8244 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Fri, 8 Feb 2013 15:55:32 +0000 Subject: [PATCH 50/69] YARN-362. Unexpected extra results when using webUI table search. Contributed by Ravi Prakash git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1444085 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../webapps/static/yarn.dt.plugins.js | 25 ++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 129594cf45a..27d4df3e0d5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -322,6 +322,9 @@ Release 0.23.7 - UNRELEASED YARN-364. AggregatedLogDeletionService can take too long to delete logs (jlowe) + YARN-362. Unexpected extra results when using webUI table search (Ravi + Prakash via jlowe) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js index 3f42c7cc2f5..d0bde290ae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js @@ -74,19 +74,19 @@ jQuery.fn.dataTableExt.oApi.fnSetFilteringDelay = function ( oSettings, iDelay ) } function renderHadoopDate(data, type, full) { - if (type === 'display') { + if (type === 'display' || type === 'filter') { if(data === '0') { return "N/A"; } return new Date(parseInt(data)).toUTCString(); } - // 'filter', 'sort', 'type' and undefined all just use the number + // 'sort', 'type' and undefined all just use the number // If date is 0, then for purposes of sorting it should be consider max_int return data === '0' ? '9007199254740992' : data; } function renderHadoopElapsedTime(data, type, full) { - if (type === 'display') { + if (type === 'display' || type === 'filter') { var timeDiff = parseInt(data); if(timeDiff < 0) return "N/A"; @@ -110,24 +110,37 @@ function renderHadoopElapsedTime(data, type, full) { toReturn += "sec"; return toReturn; } - // 'filter', 'sort', 'type' and undefined all just use the number + // 'sort', 'type' and undefined all just use the number return data; } function parseHadoopID(data, type, full) { - if (type === 'display' || type === 'filter') { + if (type === 'display') { return data; } + //Return the visible string rather than the entire HTML tag + if (type === 'filter') { + return data.split('>')[1].split('<')[0]; + } //Parse the ID for 'sort', 'type' and undefined //The number after the last '_' and before the end tag '<' var splits = data.split('_'); return splits[parseInt(splits.length-1)].split('<')[0]; } +//JSON array element is "20000 attempt_1360183373897_0001_m_000002_0" +function parseHadoopAttemptID(data, type, full) { + if (type === 'display' || type === 'filter') { + return data.split(' ')[1]; + } + //For sorting use the order as defined in the JSON element + return data.split(' ')[0]; +} + function parseHadoopProgress(data, type, full) { if (type === 'display') { return data; } //Return the title attribute for 'sort', 'filter', 'type' and undefined return data.split("'")[1]; -} +} \ No newline at end of file From 6e72c6731aa547e21d9034ad36789fcaa724bd2b Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sat, 9 Feb 2013 19:53:36 +0000 Subject: [PATCH 51/69] HADOOP-9294. GetGroupsTestBase fails on Windows. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1444415 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/tools/GetGroupsTestBase.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c96dc1aac61..90d2f9e0d4e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -350,6 +350,7 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9283. Add support for running the Hadoop client on AIX. (atm) IMPROVEMENTS + HADOOP-9253. Capture ulimit info in the logs at service start time. (Arpit Gupta via suresh) @@ -357,6 +358,8 @@ Release 2.0.4-beta - UNRELEASED BUG FIXES + HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/GetGroupsTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/GetGroupsTestBase.java index f223f1b18e9..a31700778dc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/GetGroupsTestBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/GetGroupsTestBase.java @@ -108,7 +108,7 @@ public abstract class GetGroupsTestBase { for (String group : user.getGroupNames()) { expectedOutput += " " + group; } - return expectedOutput + "\n"; + return expectedOutput + System.getProperty("line.separator"); } private String runTool(Configuration conf, String[] args, boolean success) From 969e84decbc976bd98f1050aead695d15a024ab6 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 12 Feb 2013 00:50:00 +0000 Subject: [PATCH 52/69] HDFS-4342. Directories configured in dfs.namenode.edits.dir.required but not in dfs.namenode.edits.dir are silently ignored. Contributed by Arpit Agarwal git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445006 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 92 ++++++++++++------- .../hadoop/hdfs/server/namenode/NameNode.java | 28 +++++- .../server/namenode/TestNameEditsConfigs.java | 82 +++++++++++++++++ 5 files changed, 168 insertions(+), 39 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 79872e1557f..5cf92d3232b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -317,6 +317,10 @@ Release 2.0.4-beta - UNRELEASED HDFS-4471. Namenode WebUI file browsing does not work with wildcard addresses configured. (Andrew Wang via atm) + HDFS-4342. Directories configured in dfs.namenode.edits.dir.required + but not in dfs.namenode.edits.dir are silently ignored. (Arpit Agarwal + via szetszwo) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 24cdba7e1bf..b1a4f283f5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -235,6 +235,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_SHARED_EDITS_DIR_KEY = "dfs.namenode.shared.edits.dir"; public static final String DFS_NAMENODE_EDITS_PLUGIN_PREFIX = "dfs.namenode.edits.journal-plugin"; public static final String DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY = "dfs.namenode.edits.dir.required"; + public static final String DFS_NAMENODE_EDITS_DIR_DEFAULT = "file:///tmp/hadoop/dfs/name"; public static final String DFS_CLIENT_READ_PREFETCH_SIZE_KEY = "dfs.client.read.prefetch.size"; public static final String DFS_CLIENT_RETRY_WINDOW_BASE= "dfs.client.retry.window.base"; public static final String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 2a45afa1747..b626612ec2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -127,6 +127,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.ServiceFailedException; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -422,11 +423,61 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } /** + * Check the supplied configuration for correctness. + * @param conf Supplies the configuration to validate. + * @throws IOException if the configuration could not be queried. + * @throws IllegalArgumentException if the configuration is invalid. + */ + private static void checkConfiguration(Configuration conf) + throws IOException { + + final Collection namespaceDirs = + FSNamesystem.getNamespaceDirs(conf); + final Collection editsDirs = + FSNamesystem.getNamespaceEditsDirs(conf); + final Collection requiredEditsDirs = + FSNamesystem.getRequiredNamespaceEditsDirs(conf); + final Collection sharedEditsDirs = + FSNamesystem.getSharedEditsDirs(conf); + + for (URI u : requiredEditsDirs) { + if (u.toString().compareTo( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_DEFAULT) == 0) { + continue; + } + + // Each required directory must also be in editsDirs or in + // sharedEditsDirs. + if (!editsDirs.contains(u) && + !sharedEditsDirs.contains(u)) { + throw new IllegalArgumentException( + "Required edits directory " + u.toString() + " not present in " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY + ". " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY + "=" + + editsDirs.toString() + "; " + + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY + "=" + + requiredEditsDirs.toString() + ". " + + DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "=" + + sharedEditsDirs.toString() + "."); + } + } + + if (namespaceDirs.size() == 1) { + LOG.warn("Only one image storage directory (" + + DFS_NAMENODE_NAME_DIR_KEY + ") configured. Beware of dataloss" + + " due to lack of redundant storage directories!"); + } + if (editsDirs.size() == 1) { + LOG.warn("Only one namespace edits storage directory (" + + DFS_NAMENODE_EDITS_DIR_KEY + ") configured. Beware of dataloss" + + " due to lack of redundant storage directories!"); + } + } /** * Instantiates an FSNamesystem loaded from the image and edits * directories specified in the passed Configuration. - * + * * @param conf the Configuration which specifies the storage directories * from which to load * @return an FSNamesystem which contains the loaded namespace @@ -434,39 +485,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, */ public static FSNamesystem loadFromDisk(Configuration conf) throws IOException { - Collection namespaceDirs = FSNamesystem.getNamespaceDirs(conf); - List namespaceEditsDirs = - FSNamesystem.getNamespaceEditsDirs(conf); - return loadFromDisk(conf, namespaceDirs, namespaceEditsDirs); - } - /** - * Instantiates an FSNamesystem loaded from the image and edits - * directories passed. - * - * @param conf the Configuration which specifies the storage directories - * from which to load - * @param namespaceDirs directories to load the fsimages - * @param namespaceEditsDirs directories to load the edits from - * @return an FSNamesystem which contains the loaded namespace - * @throws IOException if loading fails - */ - public static FSNamesystem loadFromDisk(Configuration conf, - Collection namespaceDirs, List namespaceEditsDirs) - throws IOException { - - if (namespaceDirs.size() == 1) { - LOG.warn("Only one image storage directory (" - + DFS_NAMENODE_NAME_DIR_KEY + ") configured. Beware of dataloss" - + " due to lack of redundant storage directories!"); - } - if (namespaceEditsDirs.size() == 1) { - LOG.warn("Only one namespace edits storage directory (" - + DFS_NAMENODE_EDITS_DIR_KEY + ") configured. Beware of dataloss" - + " due to lack of redundant storage directories!"); - } - - FSImage fsImage = new FSImage(conf, namespaceDirs, namespaceEditsDirs); + checkConfiguration(conf); + FSImage fsImage = new FSImage(conf, + FSNamesystem.getNamespaceDirs(conf), + FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem namesystem = new FSNamesystem(conf, fsImage); StartupOption startOpt = NameNode.getStartupOption(conf); if (startOpt == StartupOption.RECOVER) { @@ -913,7 +936,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, "\n\t\t- use Backup Node as a persistent and up-to-date storage " + "of the file system meta-data."); } else if (dirNames.isEmpty()) { - dirNames = Collections.singletonList("file:///tmp/hadoop/dfs/name"); + dirNames = Collections.singletonList( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_DEFAULT); } return Util.stringCollectionAsURIs(dirNames); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 99f804d1bc5..f3c95fa9feb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -78,6 +78,7 @@ import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -780,6 +781,26 @@ public class NameNode { return initializeSharedEdits(conf, force, false); } + /** + * Clone the supplied configuration but remove the shared edits dirs. + * + * @param conf Supplies the original configuration. + * @return Cloned configuration without the shared edit dirs. + * @throws IOException on failure to generate the configuration. + */ + private static Configuration getConfigurationWithoutSharedEdits( + Configuration conf) + throws IOException { + List editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); + String editsDirsString = Joiner.on(",").join(editsDirs); + + Configuration confWithoutShared = new Configuration(conf); + confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); + confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + editsDirsString); + return confWithoutShared; + } + /** * Format a new shared edits dir and copy in enough edit log segments so that * the standby NN can start up. @@ -809,11 +830,8 @@ public class NameNode { NNStorage existingStorage = null; try { - Configuration confWithoutShared = new Configuration(conf); - confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); - FSNamesystem fsns = FSNamesystem.loadFromDisk(confWithoutShared, - FSNamesystem.getNamespaceDirs(conf), - FSNamesystem.getNamespaceEditsDirs(conf, false)); + FSNamesystem fsns = + FSNamesystem.loadFromDisk(getConfigurationWithoutSharedEdits(conf)); existingStorage = fsns.getFSImage().getStorage(); NamespaceInfo nsInfo = existingStorage.getNamespaceInfo(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java index 63388be3207..b2368a1458f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java @@ -308,6 +308,88 @@ public class TestNameEditsConfigs { new File(storageDir, "current"), NameNodeDirType.IMAGE_AND_EDITS); } + /** + * Test edits.dir.required configuration options. + * 1. Directory present in dfs.namenode.edits.dir.required but not in + * dfs.namenode.edits.dir. Expected to fail. + * 2. Directory present in both dfs.namenode.edits.dir.required and + * dfs.namenode.edits.dir. Expected to succeed. + * 3. Directory present only in dfs.namenode.edits.dir. Expected to + * succeed. + */ + @Test + public void testNameEditsRequiredConfigs() throws IOException { + MiniDFSCluster cluster = null; + File nameAndEditsDir = new File(base_dir, "name_and_edits"); + File nameAndEditsDir2 = new File(base_dir, "name_and_edits2"); + + // 1 + // Bad configuration. Add a directory to dfs.namenode.edits.dir.required + // without adding it to dfs.namenode.edits.dir. + try { + Configuration conf = new HdfsConfiguration(); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, + nameAndEditsDir2.toURI().toString()); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + fail("Successfully started cluster but should not have been able to."); + } catch (IllegalArgumentException iae) { // expect to fail + LOG.info("EXPECTED: cluster start failed due to bad configuration" + iae); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + cluster = null; + } + + // 2 + // Good configuration. Add a directory to both dfs.namenode.edits.dir.required + // and dfs.namenode.edits.dir. + try { + Configuration conf = new HdfsConfiguration(); + conf.setStrings( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString(), + nameAndEditsDir2.toURI().toString()); + conf.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, + nameAndEditsDir2.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + + // 3 + // Good configuration. Adds a directory to dfs.namenode.edits.dir but not to + // dfs.namenode.edits.dir.required. + try { + Configuration conf = new HdfsConfiguration(); + conf.setStrings( + DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, + nameAndEditsDir.toURI().toString(), + nameAndEditsDir2.toURI().toString()); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_DATA_NODES) + .manageNameDfsDirs(false) + .build(); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** * Test various configuration options of dfs.namenode.name.dir and dfs.namenode.edits.dir * This test tries to simulate failure scenarios. From 7e615c7f7519de412588734aacde2d5445bb8135 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 12 Feb 2013 20:16:08 +0000 Subject: [PATCH 53/69] MAPREDUCE-4994. -jt generic command line option does not work. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445330 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/util/GenericOptionsParser.java | 7 ++++++- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../hadoop/mapred/LocalClientProtocolProvider.java | 12 +----------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java index 808fcde2a8d..b3eabf04524 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java @@ -268,7 +268,12 @@ public class GenericOptionsParser { } if (line.hasOption("jt")) { - conf.set("mapred.job.tracker", line.getOptionValue("jt"), + String optionValue = line.getOptionValue("jt"); + if (optionValue.equalsIgnoreCase("local")) { + conf.set("mapreduce.framework.name", optionValue); + } + + conf.set("yarn.resourcemanager.address", optionValue, "from -jt command line option"); } if (line.hasOption("conf")) { diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d2dcb9f44d4..66709834984 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -167,6 +167,8 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4671. AM does not tell the RM about container requests which are no longer needed. (Bikas Saha via sseth) + MAPREDUCE-4994. -jt generic command line option does not work. (sandyr via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java index cead4349db2..9dfffd5c284 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; @InterfaceAudience.Private public class LocalClientProtocolProvider extends ClientProtocolProvider { @@ -38,16 +37,7 @@ public class LocalClientProtocolProvider extends ClientProtocolProvider { if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) { return null; } - String tracker = conf.get(JTConfig.JT_IPC_ADDRESS, "local"); - if ("local".equals(tracker)) { - conf.setInt("mapreduce.job.maps", 1); - return new LocalJobRunner(conf); - } else { - - throw new IOException("Invalid \"" + JTConfig.JT_IPC_ADDRESS - + "\" configuration value for LocalJobRunner : \"" - + tracker + "\""); - } + return new LocalJobRunner(conf); } @Override From c2d6407070338f26baaa7d67c968501a987dfe1a Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 13 Feb 2013 01:15:48 +0000 Subject: [PATCH 54/69] MAPREDUCE-4989. JSONify DataTables input data for Attempts page. Contributed by Ravi Prakash git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445448 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/webapp/TaskPage.java | 101 +++++++------- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 126 +++++++++--------- .../mapreduce/v2/hs/webapp/HsTasksBlock.java | 6 +- .../mapreduce/v2/hs/webapp/HsTasksPage.java | 36 ++--- 5 files changed, 132 insertions(+), 140 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 66709834984..edbb59af34e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -696,6 +696,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes (Aleksey Gorshkov via bobby) + MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi + Prakash via jlowe) + OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java index 90f082a2312..6ee62725e0a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java @@ -27,18 +27,11 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit; import java.util.Collection; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.SubView; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; @@ -60,7 +53,7 @@ public class TaskPage extends AppView { h2($(TITLE)); return; } - TBODY> tbody = html. + html. table("#attempts"). thead(). tr(). @@ -72,49 +65,46 @@ public class TaskPage extends AppView { th(".tsh", "Started"). th(".tsh", "Finished"). th(".tsh", "Elapsed"). - th(".note", "Note")._()._(). - tbody(); + th(".note", "Note")._()._(); + // Write all the data into a JavaScript array of arrays for JQuery + // DataTables to display + StringBuilder attemptsTableData = new StringBuilder("[\n"); + for (TaskAttempt attempt : getTaskAttempts()) { TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true); - String taid = ta.getId(); String progress = percent(ta.getProgress() / 100); - ContainerId containerId = ta.getAssignedContainerId(); String nodeHttpAddr = ta.getNode(); - long startTime = ta.getStartTime(); - long finishTime = ta.getFinishTime(); - long elapsed = ta.getElapsedTime(); String diag = ta.getNote() == null ? "" : ta.getNote(); - TR>> row = tbody.tr(); - TD>>> nodeTd = row. - td(".id", taid). - td(".progress", progress). - td(".state", ta.getState()).td(); - if (nodeHttpAddr == null) { - nodeTd._("N/A"); - } else { - nodeTd. - a(".nodelink", url(HttpConfig.getSchemePrefix(), - nodeHttpAddr), nodeHttpAddr); - } - nodeTd._(); - if (containerId != null) { - String containerIdStr = ta.getAssignedContainerIdStr(); - row.td(). - a(".logslink", url(HttpConfig.getSchemePrefix(), - nodeHttpAddr, "node", "containerlogs", - containerIdStr, app.getJob().getUserName()), "logs")._(); - } else { - row.td()._("N/A")._(); - } + attemptsTableData.append("[\"") + .append(ta.getId()).append("\",\"") + .append(progress).append("\",\"") + .append(ta.getState().toString()).append("\",\"") - row. - td(".ts", Times.format(startTime)). - td(".ts", Times.format(finishTime)). - td(".dt", StringUtils.formatTime(elapsed)). - td(".note", diag)._(); + .append(nodeHttpAddr == null ? "N/A" : + "" + + nodeHttpAddr + "") + .append("\",\"") + + .append(ta.getAssignedContainerId() == null ? "N/A" : + "logs") + .append("\",\"") + + .append(ta.getStartTime()).append("\",\"") + .append(ta.getFinishTime()).append("\",\"") + .append(ta.getElapsedTime()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + diag))).append("\"],\n"); } - tbody._()._(); + //Remove the last comma and close off the array of arrays + if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1); + } + attemptsTableData.append("]"); + html.script().$type("text/javascript"). + _("var attemptsTableData=" + attemptsTableData)._(); } protected boolean isValidRequest() { @@ -140,9 +130,24 @@ public class TaskPage extends AppView { } private String attemptsTableInit() { - return tableInit(). - // Sort by id upon page load - append(", aaSorting: [[0, 'asc']]"). - append("}").toString(); + return tableInit() + .append(", 'aaData': attemptsTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") + .append("\n,aoColumnDefs:[\n") + + //logs column should not filterable (it includes container ID which may pollute searches) + .append("\n{'aTargets': [ 4 ]") + .append(", 'bSearchable': false }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6") + .append(" ], 'mRender': renderHadoopDate }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 7") + .append(" ], 'mRender': renderHadoopElapsedTime }]") + + // Sort by id upon page load + .append("\n, aaSorting: [[0, 'asc']]") + .append("}").toString(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java index 9807b1f4a93..f01ddc31ea9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java @@ -29,6 +29,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit; import java.util.Collection; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -110,13 +111,17 @@ public class HsTaskPage extends HsView { th(".note", "Note"); TBODY> tbody = headRow._()._().tbody(); - for (TaskAttempt ta : getTaskAttempts()) { + // Write all the data into a JavaScript array of arrays for JQuery + // DataTables to display + StringBuilder attemptsTableData = new StringBuilder("[\n"); + + for (TaskAttempt ta : getTaskAttempts()) { String taid = MRApps.toString(ta.getID()); String nodeHttpAddr = ta.getNodeHttpAddress(); String containerIdString = ta.getAssignedContainerID().toString(); String nodeIdString = ta.getAssignedContainerMgrAddress(); - String nodeRackName = ta.getNodeRackName(); + String nodeRackName = ta.getNodeRackName(); long attemptStartTime = ta.getLaunchTime(); long shuffleFinishTime = -1; @@ -138,58 +143,43 @@ public class HsTaskPage extends HsView { long attemptElapsed = Times.elapsed(attemptStartTime, attemptFinishTime, false); int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000); - - TR>> row = tbody.tr(); - TD>>> td = row.td(); - td.br().$title(String.valueOf(sortId))._(). // sorting - _(taid)._().td(ta.getState().toString()).td().a(".nodelink", - HttpConfig.getSchemePrefix()+ nodeHttpAddr, - nodeRackName + "/" + nodeHttpAddr); - td._(); - row.td(). - a(".logslink", - url("logs", nodeIdString, containerIdString, taid, app.getJob() - .getUserName()), "logs")._(); - - row.td(). - br().$title(String.valueOf(attemptStartTime))._(). - _(Times.format(attemptStartTime))._(); + attemptsTableData.append("[\"") + .append(sortId + " ").append(taid).append("\",\"") + .append(ta.getState().toString()).append("\",\"") + + .append("") + .append(nodeRackName + "/" + nodeHttpAddr + "\",\"") + + .append("logs\",\"") + + .append(attemptStartTime).append("\",\""); if(type == TaskType.REDUCE) { - row.td(). - br().$title(String.valueOf(shuffleFinishTime))._(). - _(Times.format(shuffleFinishTime))._(); - row.td(). - br().$title(String.valueOf(sortFinishTime))._(). - _(Times.format(sortFinishTime))._(); + attemptsTableData.append(shuffleFinishTime).append("\",\"") + .append(sortFinishTime).append("\",\""); } - row. - td(). - br().$title(String.valueOf(attemptFinishTime))._(). - _(Times.format(attemptFinishTime))._(); - + attemptsTableData.append(attemptFinishTime).append("\",\""); + if(type == TaskType.REDUCE) { - row.td(). - br().$title(String.valueOf(elapsedShuffleTime))._(). - _(formatTime(elapsedShuffleTime))._(); - row.td(). - br().$title(String.valueOf(elapsedSortTime))._(). - _(formatTime(elapsedSortTime))._(); - row.td(). - br().$title(String.valueOf(elapsedReduceTime))._(). - _(formatTime(elapsedReduceTime))._(); + attemptsTableData.append(elapsedShuffleTime).append("\",\"") + .append(elapsedSortTime).append("\",\"") + .append(elapsedReduceTime).append("\",\""); } - - row. - td(). - br().$title(String.valueOf(attemptElapsed))._(). - _(formatTime(attemptElapsed))._(). - td(".note", Joiner.on('\n').join(ta.getDiagnostics())); - row._(); + attemptsTableData.append(attemptElapsed).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n"); } - - + //Remove the last comma and close off the array of arrays + if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1); + } + attemptsTableData.append("]"); + html.script().$type("text/javascript"). + _("var attemptsTableData=" + attemptsTableData)._(); + TR>> footRow = tbody._().tfoot().tr(); footRow. th().input("search_init").$type(InputType.text). @@ -237,10 +227,6 @@ public class HsTaskPage extends HsView { footRow._()._()._(); } - private String formatTime(long elapsed) { - return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed); - } - /** * @return true if this is a valid request else false. */ @@ -292,24 +278,34 @@ public class HsTaskPage extends HsView { TaskId taskID = MRApps.toTaskID($(TASK_ID)); type = taskID.getTaskType(); } - StringBuilder b = tableInit(). - append(",aoColumnDefs:["); + StringBuilder b = tableInit() + .append(", 'aaData': attemptsTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") + .append("\n,aoColumnDefs:[\n") - b.append("{'sType':'title-numeric', 'aTargets': [ 0"); - if(type == TaskType.REDUCE) { - b.append(", 7, 8, 9, 10"); - } else { //MAP - b.append(", 5"); - } - b.append(" ] }]"); + //logs column should not filterable (it includes container ID which may pollute searches) + .append("\n{'aTargets': [ 3 ]") + .append(", 'bSearchable': false }") - // Sort by id upon page load - b.append(", aaSorting: [[0, 'asc']]"); + .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]") + .append(", 'mRender': parseHadoopAttemptID }") - b.append("}"); - return b.toString(); + .append("\n, {'sType':'numeric', 'aTargets': [ 4, 5") + //Column numbers are different for maps and reduces + .append(type == TaskType.REDUCE ? ", 6, 7" : "") + .append(" ], 'mRender': renderHadoopDate }") + + .append("\n, {'sType':'numeric', 'aTargets': [") + .append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6") + .append(" ], 'mRender': renderHadoopElapsedTime }]") + + // Sort by id upon page load + .append("\n, aaSorting: [[0, 'asc']]") + .append("}"); + return b.toString(); } - + private String attemptsPostTableInit() { return "var asInitVals = new Array();\n" + "$('tfoot input').keyup( function () \n{"+ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java index a76e4bebf3a..b42f77e7bdd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java @@ -140,6 +140,7 @@ public class HsTasksBlock extends HtmlBlock { attemptFinishTime = ta.getFinishTime(); attemptElapsed = ta.getElapsedTime(); } + tasksTableData.append("[\"") .append("") .append(tid).append("\",\"") @@ -205,9 +206,4 @@ public class HsTasksBlock extends HtmlBlock { footRow._()._()._(); } - - private String formatTime(long elapsed) { - return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed); - } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java index 4cc88212119..b278ed41cdb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java @@ -67,33 +67,25 @@ public class HsTasksPage extends HsView { type = MRApps.taskType(symbol); } StringBuilder b = tableInit(). - append(", 'aaData': tasksTableData"); - b.append(", bDeferRender: true"); - b.append(", bProcessing: true"); + append(", 'aaData': tasksTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") - b.append("\n, aoColumnDefs: [\n"); - b.append("{'sType':'numeric', 'aTargets': [ 0 ]"); - b.append(", 'mRender': parseHadoopID }"); + .append("\n, aoColumnDefs: [\n") + .append("{'sType':'numeric', 'aTargets': [ 0 ]") + .append(", 'mRender': parseHadoopID }") - b.append(", {'sType':'numeric', 'aTargets': [ 4"); - if(type == TaskType.REDUCE) { - b.append(", 9, 10, 11, 12"); - } else { //MAP - b.append(", 7"); - } - b.append(" ], 'mRender': renderHadoopElapsedTime }"); + .append(", {'sType':'numeric', 'aTargets': [ 4") + .append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7") + .append(" ], 'mRender': renderHadoopElapsedTime }") - b.append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5"); - if(type == TaskType.REDUCE) { - b.append(", 6, 7, 8"); - } else { //MAP - b.append(", 6"); - } - b.append(" ], 'mRender': renderHadoopDate }]"); + .append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5") + .append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6") + .append(" ], 'mRender': renderHadoopDate }]") // Sort by id upon page load - b.append("\n, aaSorting: [[0, 'asc']]"); - b.append("}"); + .append("\n, aaSorting: [[0, 'asc']]") + .append("}"); return b.toString(); } From 390d1fbef889092fc8f4296bd9e547f3200c8b37 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 13 Feb 2013 02:11:36 +0000 Subject: [PATCH 55/69] MAPREDUCE-4992. AM hangs in RecoveryService when recovering tasks with speculative attempts. Contributed by Robert Parker git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445456 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/recover/RecoveryService.java | 15 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 169 +++++++++++++++++- .../jobhistory/JobHistoryParser.java | 8 + 4 files changed, 192 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index edbb59af34e..8ce787dcb27 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -712,6 +712,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4458. Warn if java.library.path is used for AM or Task (Robert Parker via jeagles) + MAPREDUCE-4992. AM hangs in RecoveryService when recovering tasks with + speculative attempts (Robert Parker via jlowe) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java index 4ab61c52351..aca752721a7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java @@ -21,9 +21,12 @@ package org.apache.hadoop.mapreduce.v2.app.recover; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,6 +38,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; @@ -108,7 +112,7 @@ public class RecoveryService extends CompositeService implements Recovery { private JobInfo jobInfo = null; private final Map completedTasks = new HashMap(); - + private final List pendingTaskScheduleEvents = new ArrayList(); @@ -193,6 +197,14 @@ public class RecoveryService extends CompositeService implements Recovery { .getAllTasks(); for (TaskInfo taskInfo : taskInfos.values()) { if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) { + Iterator> taskAttemptIterator = + taskInfo.getAllTaskAttempts().entrySet().iterator(); + while (taskAttemptIterator.hasNext()) { + Map.Entry currentEntry = taskAttemptIterator.next(); + if (!jobInfo.getAllCompletedTaskAttempts().containsKey(currentEntry.getKey())) { + taskAttemptIterator.remove(); + } + } completedTasks .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo); LOG.info("Read from history task " @@ -215,6 +227,7 @@ public class RecoveryService extends CompositeService implements Recovery { JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir)); + LOG.info("Trying file " + histDirPath.toString()); FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf); // read the previous history file historyFile = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 8d6ca2b9b24..2f92cb9369c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -50,11 +50,15 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.EventHandler; import org.junit.Test; @@ -734,12 +738,173 @@ public class TestRecovery { app.verifyCompleted(); validateOutput(); } - + + /** + * AM with 2 maps and 1 reduce. For 1st map, one attempt fails, one attempt + * completely disappears because of failed launch, one attempt gets killed and + * one attempt succeeds. AM crashes after the first tasks finishes and + * recovers completely and succeeds in the second generation. + * + * @throws Exception + */ + @Test + public void testSpeculative() throws Exception { + + int runCount = 0; + long am1StartTimeEst = System.currentTimeMillis(); + MRApp app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), true, ++runCount); + Configuration conf = new Configuration(); + conf.setBoolean("mapred.mapper.new-api", true); + conf.setBoolean("mapred.reducer.new-api", true); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); + Job job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + long jobStartTime = job.getReport().getStartTime(); + //all maps would be running + Assert.assertEquals("No of tasks not correct", + 3, job.getTasks().size()); + + Iterator it = job.getTasks().values().iterator(); + Task mapTask1 = it.next(); + Task mapTask2 = it.next(); + Task reduceTask = it.next(); + + // all maps must be running + app.waitForState(mapTask1, TaskState.RUNNING); + app.waitForState(mapTask2, TaskState.RUNNING); + + // Launch a Speculative Task for the first Task + app.getContext().getEventHandler().handle( + new TaskEvent(mapTask1.getID(), TaskEventType.T_ADD_SPEC_ATTEMPT)); + int timeOut = 0; + while (mapTask1.getAttempts().size() != 2 && timeOut++ < 10) { + Thread.sleep(1000); + LOG.info("Waiting for next attempt to start"); + } + Iterator t1it = mapTask1.getAttempts().values().iterator(); + TaskAttempt task1Attempt1 = t1it.next(); + TaskAttempt task1Attempt2 = t1it.next(); + TaskAttempt task2Attempt = mapTask2.getAttempts().values().iterator().next(); + + ContainerId t1a2contId = task1Attempt2.getAssignedContainerID(); + + LOG.info(t1a2contId.toString()); + LOG.info(task1Attempt1.getID().toString()); + LOG.info(task1Attempt2.getID().toString()); + + // Launch container for speculative attempt + app.getContext().getEventHandler().handle( + new TaskAttemptContainerLaunchedEvent(task1Attempt2.getID(), runCount)); + + //before sending the TA_DONE, event make sure attempt has come to + //RUNNING state + app.waitForState(task1Attempt1, TaskAttemptState.RUNNING); + app.waitForState(task1Attempt2, TaskAttemptState.RUNNING); + app.waitForState(task2Attempt, TaskAttemptState.RUNNING); + + // reduces must be in NEW state + Assert.assertEquals("Reduce Task state not correct", + TaskState.RUNNING, reduceTask.getReport().getTaskState()); + + //send the done signal to the map 1 attempt 1 + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + task1Attempt1.getID(), + TaskAttemptEventType.TA_DONE)); + + app.waitForState(task1Attempt1, TaskAttemptState.SUCCEEDED); + + //wait for first map task to complete + app.waitForState(mapTask1, TaskState.SUCCEEDED); + long task1StartTime = mapTask1.getReport().getStartTime(); + long task1FinishTime = mapTask1.getReport().getFinishTime(); + + //stop the app + app.stop(); + + //rerun + //in rerun the 1st map will be recovered from previous run + long am2StartTimeEst = System.currentTimeMillis(); + app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), false, ++runCount); + conf = new Configuration(); + conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, true); + conf.setBoolean("mapred.mapper.new-api", true); + conf.setBoolean("mapred.reducer.new-api", true); + conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + //all maps would be running + Assert.assertEquals("No of tasks not correct", + 3, job.getTasks().size()); + it = job.getTasks().values().iterator(); + mapTask1 = it.next(); + mapTask2 = it.next(); + reduceTask = it.next(); + + // first map will be recovered, no need to send done + app.waitForState(mapTask1, TaskState.SUCCEEDED); + + app.waitForState(mapTask2, TaskState.RUNNING); + + task2Attempt = mapTask2.getAttempts().values().iterator().next(); + //before sending the TA_DONE, event make sure attempt has come to + //RUNNING state + app.waitForState(task2Attempt, TaskAttemptState.RUNNING); + + //send the done signal to the 2nd map task + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + mapTask2.getAttempts().values().iterator().next().getID(), + TaskAttemptEventType.TA_DONE)); + + //wait to get it completed + app.waitForState(mapTask2, TaskState.SUCCEEDED); + + //wait for reduce to be running before sending done + app.waitForState(reduceTask, TaskState.RUNNING); + + //send the done signal to the reduce + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + reduceTask.getAttempts().values().iterator().next().getID(), + TaskAttemptEventType.TA_DONE)); + + app.waitForState(job, JobState.SUCCEEDED); + app.verifyCompleted(); + Assert.assertEquals("Job Start time not correct", + jobStartTime, job.getReport().getStartTime()); + Assert.assertEquals("Task Start time not correct", + task1StartTime, mapTask1.getReport().getStartTime()); + Assert.assertEquals("Task Finish time not correct", + task1FinishTime, mapTask1.getReport().getFinishTime()); + Assert.assertEquals(2, job.getAMInfos().size()); + int attemptNum = 1; + // Verify AMInfo + for (AMInfo amInfo : job.getAMInfos()) { + Assert.assertEquals(attemptNum++, amInfo.getAppAttemptId() + .getAttemptId()); + Assert.assertEquals(amInfo.getAppAttemptId(), amInfo.getContainerId() + .getApplicationAttemptId()); + Assert.assertEquals(MRApp.NM_HOST, amInfo.getNodeManagerHost()); + Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort()); + Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort()); + } + long am1StartTimeReal = job.getAMInfos().get(0).getStartTime(); + long am2StartTimeReal = job.getAMInfos().get(1).getStartTime(); + Assert.assertTrue(am1StartTimeReal >= am1StartTimeEst + && am1StartTimeReal <= am2StartTimeEst); + Assert.assertTrue(am2StartTimeReal >= am2StartTimeEst + && am2StartTimeReal <= System.currentTimeMillis()); + + } + private void writeBadOutput(TaskAttempt attempt, Configuration conf) throws Exception { TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, TypeConverter.fromYarn(attempt.getID())); - + TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat .getRecordWriter(tContext); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 36dc77ef2b2..eb8db667e8a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -246,6 +246,7 @@ public class JobHistoryParser implements HistoryEventHandler { attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.counters = event.getCounters(); attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleReduceAttemptFinishedEvent @@ -262,6 +263,7 @@ public class JobHistoryParser implements HistoryEventHandler { attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleMapAttemptFinishedEvent(MapAttemptFinishedEvent event) { @@ -276,6 +278,7 @@ public class JobHistoryParser implements HistoryEventHandler { attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleTaskAttemptFailedEvent( @@ -306,6 +309,7 @@ public class JobHistoryParser implements HistoryEventHandler { taskInfo.successfulAttemptId = null; } } + info.completedTaskAttemptsMap.put(event.getTaskAttemptId(), attemptInfo); } private void handleTaskAttemptStartedEvent(TaskAttemptStartedEvent event) { @@ -443,6 +447,7 @@ public class JobHistoryParser implements HistoryEventHandler { Map jobACLs; Map tasksMap; + Map completedTaskAttemptsMap; List amInfos; AMInfo latestAmInfo; boolean uberized; @@ -456,6 +461,7 @@ public class JobHistoryParser implements HistoryEventHandler { finishedMaps = finishedReduces = 0; username = jobname = jobConfPath = jobQueueName = ""; tasksMap = new HashMap(); + completedTaskAttemptsMap = new HashMap(); jobACLs = new HashMap(); priority = JobPriority.NORMAL; } @@ -530,6 +536,8 @@ public class JobHistoryParser implements HistoryEventHandler { public Counters getReduceCounters() { return reduceCounters; } /** @return the map of all tasks in this job */ public Map getAllTasks() { return tasksMap; } + /** @return the map of all completed task attempts in this job */ + public Map getAllCompletedTaskAttempts() { return completedTaskAttemptsMap; } /** @return the priority of this job */ public String getPriority() { return priority.toString(); } public Map getJobACLs() { return jobACLs; } From 576f7893859a13d7cbbe8c14348182a0434757cf Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 13 Feb 2013 14:49:35 +0000 Subject: [PATCH 56/69] HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445635 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-project/src/site/site.xml | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 90d2f9e0d4e..f7e95c19844 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1341,6 +1341,9 @@ Release 0.23.7 - UNRELEASED BUG FIXES + HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via + tgraves) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index a199ce35022..8b2a138d043 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -52,13 +52,25 @@ + + + +
    • + + + + + + + + @@ -72,6 +84,7 @@ + From 628922eaeff5b43eb0dfd37bd718d90c445c2340 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 13 Feb 2013 15:15:28 +0000 Subject: [PATCH 57/69] HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage option (Andy Isaacson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445656 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/apt/CommandsManual.apt.vm | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f7e95c19844..3048a903501 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1344,6 +1344,9 @@ Release 0.23.7 - UNRELEASED HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via tgraves) + HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage + option (Andy Isaacson via tgraves) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm index a198bb66404..d3d22c91490 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm @@ -350,10 +350,11 @@ Administration Commands Runs a HDFS dfsadmin client. - Usage: << ...] [-clrQuota ...] [-help [cmd]]>>> + Usage: << ...] [-clrQuota ...] [-restoreFailedStorage true|false|check] [-help [cmd]]>>> *-----------------+-----------------------------------------------------------+ || COMMAND_OPTION || Description +*-----------------+-----------------------------------------------------------+ | -report | Reports basic filesystem information and statistics. *-----------------+-----------------------------------------------------------+ | -safemode enter / leave / get / wait | Safe mode maintenance command. Safe @@ -403,6 +404,10 @@ Administration Commands | 2. user is not an administrator. It does not fault if the | directory has no quota. *-----------------+-----------------------------------------------------------+ +| -restoreFailedStorage true / false / check | This option will turn on/off automatic attempt to restore failed storage replicas. + | If a failed storage becomes available again the system will attempt to restore + | edits and/or fsimage during checkpoint. 'check' option will return current setting. +*-----------------+-----------------------------------------------------------+ | -help [cmd] | Displays help for the given command or all commands if none | is specified. *-----------------+-----------------------------------------------------------+ From 666667464ad773449cd76b567312495577b3214d Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 13 Feb 2013 15:40:01 +0000 Subject: [PATCH 58/69] YARN-249. Capacity Scheduler web page should show list of active users per queue like it used to (in 1.x) (Ravi Prakash via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445673 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/webapp/ResponseInfo.java | 18 +- .../hadoop/yarn/webapp/view/InfoBlock.java | 6 +- .../scheduler/capacity/LeafQueue.java | 13 + .../scheduler/capacity/UserInfo.java | 60 ++ .../webapp/CapacitySchedulerPage.java | 40 +- .../webapp/JAXBContextResolver.java | 6 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 7 + .../dao/CapacitySchedulerQueueInfo.java | 6 + .../webapp/dao/ResourceInfo.java | 48 ++ .../resourcemanager/webapp/dao/UsersInfo.java | 46 ++ .../yarn/server/resourcemanager/MockRM.java | 45 +- .../server/resourcemanager/TestRMRestart.java | 2 +- .../TestRMWebServicesCapacitySched.java | 148 +++- .../src/site/apt/ResourceManagerRest.apt.vm | 698 +++++++++++------- 15 files changed, 857 insertions(+), 289 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UsersInfo.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 27d4df3e0d5..957aaf54543 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -310,6 +310,9 @@ Release 0.23.7 - UNRELEASED YARN-133 Update web services docs for RM clusterMetrics (Ravi Prakash via kihwal) + YARN-249. Capacity Scheduler web page should show list of active users per + queue like it used to (in 1.x) (Ravi Prakash via tgraves) + OPTIMIZATIONS YARN-357. App submission should not be synchronized (daryn) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ResponseInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ResponseInfo.java index 84c3b650e21..d38545fa373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ResponseInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ResponseInfo.java @@ -34,19 +34,21 @@ public class ResponseInfo implements Iterable { public final String key; public final String url; public final Object value; + public final boolean isRaw; - Item(String key, String url, Object value) { + Item(String key, String url, Object value, boolean isRaw) { this.key = key; this.url = url; this.value = value; + this.isRaw = isRaw; } - public static Item of(String key, Object value) { - return new Item(key, null, value); + public static Item of(String key, Object value, boolean isRaw) { + return new Item(key, null, value, isRaw); } public static Item of(String key, String url, Object value) { - return new Item(key, url, value); + return new Item(key, url, value, false); } } @@ -71,7 +73,7 @@ public class ResponseInfo implements Iterable { } public ResponseInfo _(String key, Object value) { - items.add(Item.of(key, value)); + items.add(Item.of(key, value, false)); return this; } @@ -80,6 +82,12 @@ public class ResponseInfo implements Iterable { return this; } + //Value is raw HTML and shouldn't be escaped + public ResponseInfo _r(String key, Object value) { + items.add(Item.of(key, value, true)); + return this; + } + public void clear() { items.clear(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java index 7cefe1d2497..88b7297c133 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java @@ -46,7 +46,11 @@ public class InfoBlock extends HtmlBlock { th(item.key); String value = String.valueOf(item.value); if (item.url == null) { - tr.td(value); + if (!item.isRaw) { + tr.td(value); + } else { + tr.td()._r(value)._(); + } } else { tr. td(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 7656ace5b0f..719cf1e578b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -571,6 +571,19 @@ public class LeafQueue implements CSQueue { return user; } + /** + * @return an ArrayList of UserInfo objects who are active in this queue + */ + public synchronized ArrayList getUsers() { + ArrayList usersToReturn = new ArrayList(); + for (Map.Entry entry: users.entrySet()) { + usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone( + entry.getValue().consumed), entry.getValue().getActiveApplications(), + entry.getValue().getPendingApplications())); + } + return usersToReturn; + } + @Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java new file mode 100644 index 00000000000..65c911bbb9e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class UserInfo { + protected String username; + protected ResourceInfo resourcesUsed; + protected int numPendingApplications; + protected int numActiveApplications; + + UserInfo() {} + + UserInfo(String username, Resource resUsed, int activeApps, int pendingApps) { + this.username = username; + this.resourcesUsed = new ResourceInfo(resUsed); + this.numActiveApplications = activeApps; + this.numPendingApplications = pendingApps; + } + + public String getUsername() { + return username; + } + + public ResourceInfo getResourcesUsed() { + return resourcesUsed; + } + + public int getNumPendingApplications() { + return numPendingApplications; + } + + public int getNumActiveApplications() { + return numActiveApplications; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 110bf8d17a9..52a1bc77e25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -22,12 +22,15 @@ import static org.apache.hadoop.yarn.util.StringHelper.join; import java.util.ArrayList; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.webapp.ResponseInfo; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; @@ -63,8 +66,42 @@ class CapacitySchedulerPage extends RmView { lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo; } + //Return a string describing one resource as a percentage of another + private String getPercentage(ResourceInfo numerator, ResourceInfo denominator) { + StringBuilder percentString = new StringBuilder("Memory: "); + if (numerator != null) { + percentString.append(numerator.getMemory()); + } + if (denominator.getMemory() != 0) { + percentString.append(" (") + .append(StringUtils.format("%.2f", numerator.getMemory() * 100.0 / + denominator.getMemory()) + "%)"); + } + percentString.append(", vCores: "); + if (numerator != null) { + percentString.append(numerator.getvCores()); + } + if (denominator.getvCores() != 0) { + percentString.append(" (") + .append(StringUtils.format("%.2f", numerator.getvCores() * 100.0 / + denominator.getvCores()) + "%)"); + } + return percentString.toString(); + } + @Override protected void render(Block html) { + StringBuilder activeUserList = new StringBuilder(""); + ResourceInfo usedResources = lqinfo.getResourcesUsed(); + ArrayList users = lqinfo.getUsers().getUsersList(); + for (UserInfo entry: users) { + activeUserList.append(entry.getUsername()).append(" <") + .append(getPercentage(entry.getResourcesUsed(), usedResources)) + .append(", Active Apps: " + entry.getNumActiveApplications()) + .append(", Pending Apps: " + entry.getNumPendingApplications()) + .append(">
    "); //Force line break + } + ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). _("Queue State:", lqinfo.getQueueState()). _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)). @@ -81,7 +118,8 @@ class CapacitySchedulerPage extends RmView { _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)). _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). - _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())); + _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())). + _r("Active users: ", activeUserList.toString()); html._(InfoBlock.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 12e77a7c49c..5f50ed18f12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -30,6 +30,7 @@ import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; @@ -42,9 +43,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsIn import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UsersInfo; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; @Singleton @@ -61,7 +64,8 @@ public class JAXBContextResolver implements ContextResolver { SchedulerTypeInfo.class, NodeInfo.class, UserMetricsInfo.class, CapacitySchedulerInfo.class, ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, NodesInfo.class, - RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class}; + RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, + ResourceInfo.class, UsersInfo.class, UserInfo.class}; public JAXBContextResolver() throws Exception { this.types = new HashSet(Arrays.asList(cTypes)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 5b2624ee984..d90e9631b95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -35,6 +35,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected int maxActiveApplications; protected int maxActiveApplicationsPerUser; protected int userLimit; + protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; CapacitySchedulerLeafQueueInfo() { @@ -50,6 +51,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { maxActiveApplications = q.getMaximumActiveApplications(); maxActiveApplicationsPerUser = q.getMaximumActiveApplicationsPerUser(); userLimit = q.getUserLimit(); + users = new UsersInfo(q.getUsers()); userLimitFactor = q.getUserLimitFactor(); } @@ -85,6 +87,11 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { return userLimit; } + //Placing here because of JERSEY-1199 + public UsersInfo getUsers() { + return users; + } + public float getUserLimitFactor() { return userLimitFactor; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 6757227c63a..2cfa660a355 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -48,6 +48,7 @@ public class CapacitySchedulerQueueInfo { protected String queueName; protected QueueState state; protected CapacitySchedulerQueueInfoList queues; + protected ResourceInfo resourcesUsed; CapacitySchedulerQueueInfo() { }; @@ -69,6 +70,7 @@ public class CapacitySchedulerQueueInfo { usedResources = q.getUsedResources().toString(); queueName = q.getQueueName(); state = q.getState(); + resourcesUsed = new ResourceInfo(q.getUsedResources()); } public float getCapacity() { @@ -119,6 +121,10 @@ public class CapacitySchedulerQueueInfo { return this.queues; } + public ResourceInfo getResourcesUsed() { + return resourcesUsed; + } + /** * Limit a value to a specified range. * @param val the value to be capped diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java new file mode 100644 index 00000000000..84276bf105a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Resource; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceInfo { + int memory; + int vCores; + + public ResourceInfo() { + } + + public ResourceInfo(Resource res) { + memory = res.getMemory(); + vCores = res.getVirtualCores(); + } + + public int getMemory() { + return memory; + } + + public int getvCores() { + return vCores; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UsersInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UsersInfo.java new file mode 100644 index 00000000000..3ee7edfca85 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UsersInfo.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class UsersInfo { + @XmlElement(name="user") + protected ArrayList usersList = new ArrayList(); + + public UsersInfo() { + } + + public UsersInfo(ArrayList usersList) { + this.usersList = usersList; + } + + public ArrayList getUsersList() { + return usersList; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 0bc3211a819..12391c60978 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -18,16 +18,19 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.security.PrivilegedAction; import java.util.Map; import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -37,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -118,21 +122,27 @@ public class MockRM extends ResourceManager { } public RMApp submitApp(int masterMemory) throws Exception { - return submitApp(masterMemory, "", ""); + return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() + .getShortUserName()); } // client public RMApp submitApp(int masterMemory, String name, String user) throws Exception { - return submitApp(masterMemory, name, user, null, false); + return submitApp(masterMemory, name, user, null, false, null); } public RMApp submitApp(int masterMemory, String name, String user, Map acls) throws Exception { - return submitApp(masterMemory, name, user, acls, false); + return submitApp(masterMemory, name, user, acls, false, null); } public RMApp submitApp(int masterMemory, String name, String user, - Map acls, boolean unmanaged) throws Exception { + Map acls, String queue) throws Exception { + return submitApp(masterMemory, name, user, acls, false, queue); + } + + public RMApp submitApp(int masterMemory, String name, String user, + Map acls, boolean unmanaged, String queue) throws Exception { ClientRMProtocol client = getClientRMService(); GetNewApplicationResponse resp = client.getNewApplication(Records .newRecord(GetNewApplicationRequest.class)); @@ -148,6 +158,9 @@ public class MockRM extends ResourceManager { if(unmanaged) { sub.setUnmanagedAM(true); } + if (queue != null) { + sub.setQueue(queue); + } ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); Resource capability = Records.newRecord(Resource.class); @@ -157,7 +170,29 @@ public class MockRM extends ResourceManager { sub.setAMContainerSpec(clc); req.setApplicationSubmissionContext(sub); - client.submitApplication(req); + UserGroupInformation fakeUser = + UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"}); + PrivilegedAction action = + new PrivilegedAction() { + ClientRMProtocol client; + SubmitApplicationRequest req; + @Override + public SubmitApplicationResponse run() { + try { + return client.submitApplication(req); + } catch (YarnRemoteException e) { + e.printStackTrace(); + } + return null; + } + PrivilegedAction setClientReq( + ClientRMProtocol client, SubmitApplicationRequest req) { + this.client = client; + this.req = req; + return this; + } + }.setClientReq(client, req); + fakeUser.doAs(action); // make sure app is immediately available after submit waitForState(appId, RMAppState.ACCEPTED); return getRMContext().getRMApps().get(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index d4f97380c3d..babe79a2de8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -152,7 +152,7 @@ public class TestRMRestart { .getApplicationId()); // create unmanaged app - RMApp appUnmanaged = rm1.submitApp(200, "", "", null, true); + RMApp appUnmanaged = rm1.submitApp(200, "someApp", "someUser", null, true, null); ApplicationAttemptId unmanagedAttemptId = appUnmanaged.getCurrentAppAttempt().getAppAttemptId(); // assert appUnmanaged info is saved diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 00bdafbe8ae..3fc08831898 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -27,10 +27,12 @@ import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -44,6 +46,7 @@ import org.junit.Before; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; @@ -355,10 +358,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 11; + int numExpectedElements = 12; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 20; + numExpectedElements = 22; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -397,6 +400,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { lqi.userLimit = info.getInt("userLimit"); lqi.userLimitFactor = (float) info.getDouble("userLimitFactor"); verifyLeafQueueGeneric(q, lqi); + // resourcesUsed and users (per-user resources used) are checked in + // testPerUserResource() } } @@ -464,4 +469,143 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { assertEquals("userLimitFactor doesn't match", csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f); } + + //Return a child Node of node with the tagname or null if none exists + private Node getChildNodeByName(Node node, String tagname) { + NodeList nodeList = node.getChildNodes(); + for (int i=0; i < nodeList.getLength(); ++i) { + if (nodeList.item(i).getNodeName().equals(tagname)) { + return nodeList.item(i); + } + } + return null; + } + + /** + * Test per user resources and resourcesUsed elements in the web services XML + * @throws Exception + */ + @Test + public void testPerUserResourcesXML() throws Exception { + //Start RM so that it accepts app submissions + rm.start(); + try { + rm.submitApp(10, "app1", "user1", null, "b1"); + rm.submitApp(20, "app2", "user2", null, "b1"); + + //Get the XML from ws/v1/cluster/scheduler + WebResource r = resource(); + ClientResponse response = r.path("ws/v1/cluster/scheduler") + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilder db = DocumentBuilderFactory.newInstance() + .newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + //Parse the XML we got + Document dom = db.parse(is); + + //Get all users elements (1 for each leaf queue) + NodeList allUsers = dom.getElementsByTagName("users"); + for (int i=0; i", res.toString()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index f55f925bede..d48312a9360 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -378,6 +378,8 @@ ResourceManager REST API's. *---------------+--------------+-------------------------------+ | queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information| *---------------+--------------+-------------------------------+ +| resourcesUsed | A single resource object | The total amount of resources used by this queue | +*---------------+--------------+-------------------------------+ ** Elements of the queues object for a Leaf queue - contains all elements in parent plus the following: @@ -404,6 +406,32 @@ ResourceManager REST API's. *---------------+--------------+-------------------------------+ | userLimitFactor | float | The user limit factor set in the configuration | *---------------+--------------+-------------------------------+ +| users | array of users(JSON)/zero or more user objects(XML) | A collection of user objects containing resources used | +*---------------+--------------+-------------------------------+ + +** Elements of the user object for users: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| username | String | The username of the user using the resources | +*---------------+--------------+-------------------------------+ +| resourcesUsed | A single resource object | The amount of resources used by the user in this queue | +*---------------+--------------+-------------------------------+ +| numActiveApplications | int | The number of active applications for this user in this queue | +*---------------+--------------+-------------------------------+ +| numPendingApplications | int | The number of pending applications for this user in this queue | +*---------------+--------------+-------------------------------+ + +** Elements of the resource object for resourcesUsed in user and queues: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| memory | int | The amount of memory used (in MB) | +*---------------+--------------+-------------------------------+ +| vCores | int | The number of virtual cores | +*---------------+--------------+-------------------------------+ *** Response Examples @@ -428,199 +456,262 @@ ResourceManager REST API's. +---+ { - "scheduler" : { - "schedulerInfo" : { - "queueName" : "root", - "maxCapacity" : 100, - "type" : "capacityScheduler", - "queues" : { - "queue" : [ - { - "numPendingApplications" : 0, - "queueName" : "default", - "userLimitFactor" : 1, - "maxApplications" : 1, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 90, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 90, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 70, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 70, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 1 - }, - { - "queueName" : "test", - "absoluteCapacity" : 20, - "usedCapacity" : 0, - "capacity" : 20, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 100, - "queues" : { - "queue" : [ - { - "queueName" : "a1", - "absoluteCapacity" : 12, - "usedCapacity" : 0, - "capacity" : 60.000004, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 100, - "queues" : { - "queue" : [ - { - "numPendingApplications" : 0, - "queueName" : "a11", - "userLimitFactor" : 1, - "maxApplications" : 0, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 100, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 10.200001, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 85, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 0 - }, - { - "numPendingApplications" : 0, - "queueName" : "a12", - "userLimitFactor" : 1, - "maxApplications" : 0, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 100, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 1.8000001, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 15.000001, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 0 - } - ] - }, - "absoluteUsedCapacity" : 0 - }, - { - "numPendingApplications" : 0, - "queueName" : "a2", - "userLimitFactor" : 1, - "maxApplications" : 0, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 100, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 8.000001, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 40, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 0 - } - ] - }, - "absoluteUsedCapacity" : 0 - }, - { - "queueName" : "test2", - "absoluteCapacity" : 10, - "usedCapacity" : 0, - "capacity" : 10, - "state" : "RUNNING", - "maxCapacity" : 15.000001, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 15.000001, - "queues" : { - "queue" : [ - { - "numPendingApplications" : 0, - "queueName" : "a3", - "userLimitFactor" : 1, - "maxApplications" : 0, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 15.000001, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 9, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 90, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 0 - }, - { - "numPendingApplications" : 0, - "queueName" : "a4", - "userLimitFactor" : 1, - "maxApplications" : 0, - "usedCapacity" : 0, - "numContainers" : 0, - "state" : "RUNNING", - "maxCapacity" : 100, - "numApplications" : 0, - "usedResources" : "memory: 0", - "absoluteMaxCapacity" : 15.000001, - "maxActiveApplications" : 1, - "numActiveApplications" : 0, - "absoluteUsedCapacity" : 0, - "userLimit" : 100, - "absoluteCapacity" : 1.0000001, - "maxActiveApplicationsPerUser" : 1, - "capacity" : 10, - "type" : "capacitySchedulerLeafQueueInfo", - "maxApplicationsPerUser" : 0 - } - ] - }, - "absoluteUsedCapacity" : 0 - } - ] - }, - "usedCapacity" : 0, - "capacity" : 100 - } - } + "scheduler": { + "schedulerInfo": { + "capacity": 100.0, + "maxCapacity": 100.0, + "queueName": "root", + "queues": { + "queue": [ + { + "absoluteCapacity": 10.5, + "absoluteMaxCapacity": 50.0, + "absoluteUsedCapacity": 0.0, + "capacity": 10.5, + "maxCapacity": 50.0, + "numApplications": 0, + "queueName": "a", + "queues": { + "queue": [ + { + "absoluteCapacity": 3.15, + "absoluteMaxCapacity": 25.0, + "absoluteUsedCapacity": 0.0, + "capacity": 30.000002, + "maxCapacity": 50.0, + "numApplications": 0, + "queueName": "a1", + "queues": { + "queue": [ + { + "absoluteCapacity": 2.6775, + "absoluteMaxCapacity": 25.0, + "absoluteUsedCapacity": 0.0, + "capacity": 85.0, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 1, + "maxApplications": 267, + "maxApplicationsPerUser": 267, + "maxCapacity": 100.0, + "numActiveApplications": 0, + "numApplications": 0, + "numContainers": 0, + "numPendingApplications": 0, + "queueName": "a1a", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 1.0, + "users": null + }, + { + "absoluteCapacity": 0.47250003, + "absoluteMaxCapacity": 25.0, + "absoluteUsedCapacity": 0.0, + "capacity": 15.000001, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 1, + "maxApplications": 47, + "maxApplicationsPerUser": 47, + "maxCapacity": 100.0, + "numActiveApplications": 0, + "numApplications": 0, + "numContainers": 0, + "numPendingApplications": 0, + "queueName": "a1b", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 1.0, + "users": null + } + ] + }, + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "usedCapacity": 0.0, + "usedResources": "" + }, + { + "absoluteCapacity": 7.35, + "absoluteMaxCapacity": 50.0, + "absoluteUsedCapacity": 0.0, + "capacity": 70.0, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 100, + "maxApplications": 735, + "maxApplicationsPerUser": 73500, + "maxCapacity": 100.0, + "numActiveApplications": 0, + "numApplications": 0, + "numContainers": 0, + "numPendingApplications": 0, + "queueName": "a2", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 100.0, + "users": null + } + ] + }, + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "usedCapacity": 0.0, + "usedResources": "" + }, + { + "absoluteCapacity": 89.5, + "absoluteMaxCapacity": 100.0, + "absoluteUsedCapacity": 0.0, + "capacity": 89.5, + "maxCapacity": 100.0, + "numApplications": 2, + "queueName": "b", + "queues": { + "queue": [ + { + "absoluteCapacity": 53.7, + "absoluteMaxCapacity": 100.0, + "absoluteUsedCapacity": 0.0, + "capacity": 60.000004, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 100, + "maxApplications": 5370, + "maxApplicationsPerUser": 537000, + "maxCapacity": 100.0, + "numActiveApplications": 1, + "numApplications": 2, + "numContainers": 0, + "numPendingApplications": 1, + "queueName": "b1", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 100.0, + "users": { + "user": [ + { + "numActiveApplications": 0, + "numPendingApplications": 1, + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "username": "user2" + }, + { + "numActiveApplications": 1, + "numPendingApplications": 0, + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "username": "user1" + } + ] + } + }, + { + "absoluteCapacity": 35.3525, + "absoluteMaxCapacity": 100.0, + "absoluteUsedCapacity": 0.0, + "capacity": 39.5, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 100, + "maxApplications": 3535, + "maxApplicationsPerUser": 353500, + "maxCapacity": 100.0, + "numActiveApplications": 0, + "numApplications": 0, + "numContainers": 0, + "numPendingApplications": 0, + "queueName": "b2", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 100.0, + "users": null + }, + { + "absoluteCapacity": 0.4475, + "absoluteMaxCapacity": 100.0, + "absoluteUsedCapacity": 0.0, + "capacity": 0.5, + "maxActiveApplications": 1, + "maxActiveApplicationsPerUser": 100, + "maxApplications": 44, + "maxApplicationsPerUser": 4400, + "maxCapacity": 100.0, + "numActiveApplications": 0, + "numApplications": 0, + "numContainers": 0, + "numPendingApplications": 0, + "queueName": "b3", + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "type": "capacitySchedulerLeafQueueInfo", + "usedCapacity": 0.0, + "usedResources": "", + "userLimit": 100, + "userLimitFactor": 100.0, + "users": null + } + ] + }, + "resourcesUsed": { + "memory": 0, + "vCores": 0 + }, + "state": "RUNNING", + "usedCapacity": 0.0, + "usedResources": "" + } + ] + }, + "type": "capacityScheduler", + "usedCapacity": 0.0 + } + } } +---+ @@ -653,48 +744,27 @@ ResourceManager REST API's. 100.0 root - - 70.0 - 0.0 - 90.0 - 70.0 - 90.0 - 0.0 - 0 - memory: 0 - default - RUNNING - 0 - 0 - 0 - 1 - 1 - 1 - 1 - 100 - 1.0 - - 20.0 + 10.5 0.0 - 100.0 - 20.0 - 100.0 + 50.0 + 10.5 + 50.0 0.0 0 - memory: 0 - test + <memory:0, vCores:0> + a RUNNING - 60.000004 + 30.000002 0.0 - 100.0 - 12.0 - 100.0 + 50.0 + 3.15 + 25.0 0.0 0 - memory: 0 + <memory:0, vCores:0> a1 RUNNING @@ -702,124 +772,206 @@ ResourceManager REST API's. 85.0 0.0 100.0 - 10.200001 - 100.0 + 2.6775 + 25.0 0.0 0 - memory: 0 - a11 + <memory:0, vCores:0> + a1a RUNNING + + 0 + 0 + 0 0 0 - 0 - 0 + 267 + 267 1 1 100 + 1.0 15.000001 0.0 100.0 - 1.8000001 - 100.0 + 0.47250003 + 25.0 0.0 0 - memory: 0 - a12 + <memory:0, vCores:0> + a1b RUNNING + + 0 + 0 + 0 0 0 - 0 - 0 + 47 + 47 1 1 100 + 1.0 + + 0 + 0 + - 40.0 + 70.0 0.0 100.0 - 8.000001 - 100.0 + 7.35 + 50.0 0.0 0 - memory: 0 + <memory:0, vCores:0> a2 RUNNING + + 0 + 0 + 0 0 0 - 0 - 0 + 735 + 73500 1 - 1 + 100 100 - 1.0 + + 100.0 + + 0 + 0 + - 10.0 + 89.5 0.0 - 15.000001 - 10.0 - 15.000001 + 100.0 + 89.5 + 100.0 0.0 - 0 - memory: 0 - test2 + 2 + <memory:0, vCores:0> + b RUNNING - 90.0 + 60.000004 0.0 100.0 - 9.0 - 15.000001 + 53.7 + 100.0 0.0 - 0 - memory: 0 - a3 + 2 + <memory:0, vCores:0> + b1 RUNNING - 0 - 0 + + 0 + 0 + + 1 + 1 0 - 0 - 0 + 5370 + 537000 1 - 1 + 100 100 - 1.0 + + + user2 + + 0 + 0 + + 1 + 0 + + + user1 + + 0 + 0 + + 0 + 1 + + + 100.0 - 10.0 + 39.5 0.0 100.0 - 1.0000001 - 15.000001 + 35.3525 + 100.0 0.0 0 - memory: 0 - a4 + <memory:0, vCores:0> + b2 RUNNING + + 0 + 0 + 0 0 0 - 0 - 0 + 3535 + 353500 1 - 1 + 100 100 - 1.0 + + 100.0 + + + 0.5 + 0.0 + 100.0 + 0.4475 + 100.0 + 0.0 + 0 + <memory:0, vCores:0> + b3 + RUNNING + + 0 + 0 + + 0 + 0 + 0 + 44 + 4400 + 1 + 100 + 100 + + 100.0 + + 0 + 0 + From 43204f0c836ed35270bdac99bf3bafdb78ae6b57 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 13 Feb 2013 19:20:31 +0000 Subject: [PATCH 59/69] MAPREDUCE-5000. Fixes getCounters when speculating by fixing the selection of the best attempt for a task. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445871 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/impl/TaskImpl.java | 4 ++ .../v2/app/job/impl/TestTaskImpl.java | 60 ++++++++++++++++++- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8ce787dcb27..12ea8aaf432 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -169,6 +169,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4994. -jt generic command line option does not work. (sandyr via tucu) + MAPREDUCE-5000. Fixes getCounters when speculating by fixing the selection + of the best attempt for a task. (Jason Lowe via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index 28950e96cc5..c45197e1579 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -539,6 +539,10 @@ public abstract class TaskImpl implements Task, EventHandler { //select the nextAttemptNumber with best progress // always called inside the Read Lock private TaskAttempt selectBestAttempt() { + if (successfulAttempt != null) { + return attempts.get(successfulAttempt); + } + float progress = 0f; TaskAttempt result = null; for (TaskAttempt at : attempts.values()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java index c0cfc072284..3f03dc3fd32 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java @@ -35,6 +35,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Task; import org.apache.hadoop.mapred.TaskUmbilicalProtocol; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; @@ -52,7 +55,6 @@ import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent; -import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; @@ -143,6 +145,7 @@ public class TestTaskImpl { private float progress = 0; private TaskAttemptState state = TaskAttemptState.NEW; private TaskType taskType; + private Counters attemptCounters = TaskAttemptImpl.EMPTY_COUNTERS; public MockTaskAttemptImpl(TaskId taskId, int id, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, Path jobFile, int partition, @@ -178,7 +181,15 @@ public class TestTaskImpl { public TaskAttemptState getState() { return state; } - + + @Override + public Counters getCounters() { + return attemptCounters; + } + + public void setCounters(Counters counters) { + attemptCounters = counters; + } } private class MockTask extends Task { @@ -687,4 +698,49 @@ public class TestTaskImpl { TaskEventType.T_ATTEMPT_KILLED)); assertEquals(TaskState.FAILED, mockTask.getState()); } + + @Test + public void testCountersWithSpeculation() { + mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(), + remoteJobConfFile, conf, taskAttemptListener, jobToken, + credentials, clock, + completedTasksFromPreviousRun, startCount, + metrics, appContext, TaskType.MAP) { + @Override + protected int getMaxAttempts() { + return 1; + } + }; + TaskId taskId = getNewTaskID(); + scheduleTaskAttempt(taskId); + launchTaskAttempt(getLastAttempt().getAttemptId()); + updateLastAttemptState(TaskAttemptState.RUNNING); + MockTaskAttemptImpl baseAttempt = getLastAttempt(); + + // add a speculative attempt + mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), + TaskEventType.T_ADD_SPEC_ATTEMPT)); + launchTaskAttempt(getLastAttempt().getAttemptId()); + updateLastAttemptState(TaskAttemptState.RUNNING); + MockTaskAttemptImpl specAttempt = getLastAttempt(); + assertEquals(2, taskAttempts.size()); + + Counters specAttemptCounters = new Counters(); + Counter cpuCounter = specAttemptCounters.findCounter( + TaskCounter.CPU_MILLISECONDS); + cpuCounter.setValue(1000); + specAttempt.setCounters(specAttemptCounters); + + // have the spec attempt succeed but second attempt at 1.0 progress as well + commitTaskAttempt(specAttempt.getAttemptId()); + specAttempt.setProgress(1.0f); + specAttempt.setState(TaskAttemptState.SUCCEEDED); + mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(), + TaskEventType.T_ATTEMPT_SUCCEEDED)); + assertEquals(TaskState.SUCCEEDED, mockTask.getState()); + baseAttempt.setProgress(1.0f); + + Counters taskCounters = mockTask.getCounters(); + assertEquals("wrong counters for task", specAttemptCounters, taskCounters); + } } From 7bc6040ebb24130d0888847e1d1f5ae3893506d1 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 13 Feb 2013 19:40:36 +0000 Subject: [PATCH 60/69] YARN-391. Formatting fixes for LCEResourceHandler classes. Contributed by Steve Loughran) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445882 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../util/CgroupsLCEResourcesHandler.java | 32 +++++++++---------- .../util/DefaultLCEResourcesHandler.java | 4 +-- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 957aaf54543..ee2161c007f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -35,6 +35,9 @@ Release 2.0.4-beta - UNRELEASED YARN-377. Use the new StringUtils methods added by HADOOP-9252 and fix TestContainersMonitor. (Chris Nauroth via szetszwo) + YARN-391. Formatting fixes for LCEResourceHandler classes. + (Steve Loughran via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index 53a01ebddec..5bcd34f9510 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -80,17 +80,17 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { NM_LINUX_CONTAINER_CGROUPS_MOUNT, false); this.cgroupMountPath = conf.get(YarnConfiguration. NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null); - + // remove extra /'s at end or start of cgroupPrefix if (cgroupPrefix.charAt(0) == '/') { - cgroupPrefix = cgroupPrefix.substring(1); + cgroupPrefix = cgroupPrefix.substring(1); } int len = cgroupPrefix.length(); if (cgroupPrefix.charAt(len - 1) == '/') { - cgroupPrefix = cgroupPrefix.substring(0, len - 1); + cgroupPrefix = cgroupPrefix.substring(0, len - 1); } - + // mount cgroups if requested if (cgroupMount && cgroupMountPath != null) { ArrayList cgroupKVs = new ArrayList(); @@ -98,14 +98,14 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { CONTROLLER_CPU); lce.mountCgroups(cgroupKVs, cgroupPrefix); } - + initializeControllerPaths(); } boolean isCpuWeightEnabled() { return this.cpuWeightEnabled; - } + } /* * Next four functions are for an individual cgroup. @@ -155,7 +155,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { } } } - } + } private void deleteCgroup(String controller, String groupName) { String path = pathForCgroup(controller, groupName); @@ -165,7 +165,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { if (! new File(path).delete()) { LOG.warn("Unable to delete cgroup at: " + path); } - } + } /* * Next three functions operate on all the resources we are enforcing. @@ -178,7 +178,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { private void setupLimits(ContainerId containerId, Resource containerResource) throws IOException { String containerName = containerId.toString(); - + if (isCpuWeightEnabled()) { createCgroup(CONTROLLER_CPU, containerName); updateCgroup(CONTROLLER_CPU, containerName, "shares", @@ -202,7 +202,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { if (isCpuWeightEnabled()) { deleteCgroup(CONTROLLER_CPU, containerName); - } + } } /* @@ -222,7 +222,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { String containerName = containerId.toString(); StringBuilder sb = new StringBuilder("cgroups="); - + if (isCpuWeightEnabled()) { sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/cgroup.procs"); sb.append(","); @@ -231,7 +231,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { if (sb.charAt(sb.length() - 1) == ',') { sb.deleteCharAt(sb.length() - 1); } - + return sb.toString(); } @@ -255,8 +255,8 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { BufferedReader in = null; try { - in = new BufferedReader(new FileReader(new File(MTAB_FILE))); - + in = new BufferedReader(new FileReader(new File(MTAB_FILE))); + for (String str = in.readLine(); str != null; str = in.readLine()) { Matcher m = MTAB_FILE_FORMAT.matcher(str); @@ -316,6 +316,6 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { } else { throw new IOException("Not able to enforce cpu weights; cannot find " + "cgroup for cpu controller in " + MTAB_FILE); - } + } } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java index fcb166ffbf0..9fb87074e2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java @@ -33,7 +33,7 @@ public class DefaultLCEResourcesHandler implements LCEResourcesHandler { private Configuration conf; public DefaultLCEResourcesHandler() { - } + } public void setConf(Configuration conf) { this.conf = conf; @@ -42,7 +42,7 @@ public class DefaultLCEResourcesHandler implements LCEResourcesHandler { @Override public Configuration getConf() { return conf; - } + } public void init(LinuxContainerExecutor lce) { } From a3ddc8ef968fc175d41f5232e026e9031553d992 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 13 Feb 2013 19:42:13 +0000 Subject: [PATCH 61/69] HADOOP-9305. Add support for running the Hadoop client on 64-bit AIX. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445884 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../hadoop/security/UserGroupInformation.java | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3048a903501..86a6c3421c2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -360,6 +360,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh) + HADOOP-9305. Add support for running the Hadoop client on 64-bit AIX. (atm) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 7983013539c..b01e12b5a44 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -316,7 +316,8 @@ public class UserGroupInformation { return is64Bit ? "com.ibm.security.auth.module.Win64LoginModule" : "com.ibm.security.auth.module.NTLoginModule"; } else if (aix) { - return "com.ibm.security.auth.module.AIXLoginModule"; + return is64Bit ? "com.ibm.security.auth.module.AIX64LoginModule" + : "com.ibm.security.auth.module.AIXLoginModule"; } else { return "com.ibm.security.auth.module.LinuxLoginModule"; } @@ -331,24 +332,24 @@ public class UserGroupInformation { private static Class getOsPrincipalClass() { ClassLoader cl = ClassLoader.getSystemClassLoader(); try { + String principalClass = null; if (ibmJava) { - if (windows) { - return (Class) (is64Bit - ? cl.loadClass("com.ibm.security.auth.UsernamePrincipal") - : cl.loadClass("com.ibm.security.auth.NTUserPrincipal")); - } else if (aix) { - return (Class) - cl.loadClass("com.ibm.security.auth.AIXPrincipal"); + if (is64Bit) { + principalClass = "com.ibm.security.auth.UsernamePrincipal"; } else { - return (Class) (is64Bit - ? cl.loadClass("com.ibm.security.auth.UsernamePrincipal") - : cl.loadClass("com.ibm.security.auth.LinuxPrincipal")); + if (windows) { + principalClass = "com.ibm.security.auth.NTUserPrincipal"; + } else if (aix) { + principalClass = "com.ibm.security.auth.AIXPrincipal"; + } else { + principalClass = "com.ibm.security.auth.LinuxPrincipal"; + } } } else { - return (Class) (windows - ? cl.loadClass("com.sun.security.auth.NTUserPrincipal") - : cl.loadClass("com.sun.security.auth.UnixPrincipal")); + principalClass = windows ? "com.sun.security.auth.NTUserPrincipal" + : "com.sun.security.auth.UnixPrincipal"; } + return (Class) cl.loadClass(principalClass); } catch (ClassNotFoundException e) { LOG.error("Unable to find JAAS classes:" + e.getMessage()); } From 6bcbf492a2397db38c69eef427c62b909a0ac9e3 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 13 Feb 2013 22:13:44 +0000 Subject: [PATCH 62/69] HADOOP-9117. replace protoc ant plugin exec with a maven plugin. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445956 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../dev-support/findbugsExcludeFile.xml | 4 + hadoop-common-project/hadoop-common/pom.xml | 105 +++++++-------- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 125 ++++++++++-------- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 67 +++------- .../hadoop-mapreduce-client-common/pom.xml | 69 +++------- .../maven/plugin/protoc/ProtocMojo.java | 86 ++++++++++++ .../hadoop-yarn/hadoop-yarn-api/pom.xml | 75 +++-------- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 76 +++-------- .../hadoop-yarn-server-common/pom.xml | 64 +++------ .../hadoop-yarn-server-nodemanager/pom.xml | 74 +++-------- 11 files changed, 321 insertions(+), 426 deletions(-) create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 86a6c3421c2..832218e0234 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -354,6 +354,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9253. Capture ulimit info in the logs at service start time. (Arpit Gupta via suresh) + HADOOP-9117. replace protoc ant plugin exec with a maven plugin. (tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index e5f52ba3030..a11383cd035 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -286,6 +286,10 @@     + + + +