From 04d139e2a0e61a62471556255fc9a65792fa373c Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Fri, 20 Dec 2013 16:05:03 +0000 Subject: [PATCH 01/11] HDFS-5681. renewLease should not hold fsn write lock. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552711 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 45aaaccbebe..5949df30176 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -770,6 +770,8 @@ Release 2.4.0 - UNRELEASED HDFS-5341. Reduce fsdataset lock duration during directory scanning. (Qus-Jiawei via kihwal) + HDFS-5681. renewLease should not hold fsn write lock. (daryn via Kihwal) + BUG FIXES HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d293006d97d..83b9b21ab20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3999,13 +3999,13 @@ public class FSNamesystem implements Namesystem, FSClusterStats, */ void renewLease(String holder) throws IOException { checkOperation(OperationCategory.WRITE); - writeLock(); + readLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot renew lease for " + holder); leaseManager.renewLease(holder); } finally { - writeUnlock(); + readUnlock(); } } From 74689ab7caf981459e668e29ded4164c6dab01db Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Fri, 20 Dec 2013 19:22:49 +0000 Subject: [PATCH 02/11] YARN-1180. Update capacity scheduler docs to include types on the configs (Chen He via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552788 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 17 ++++++++++++++++- .../src/site/apt/CapacityScheduler.apt.vm | 5 +++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8e5c4636138..c623dc2fb15 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1756,7 +1756,22 @@ Release 2.0.2-alpha - 2012-09-07 YARN-138. Ensure default values for minimum/maximum container sizes is sane. (harsh & sseth via acmurthy) -Release 0.23.10 - UNRELEASED +Release 0.23.11 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + YARN-1180. Update capacity scheduler docs to include types on the configs + (Chen He via jeagles) + +Release 0.23.10 - 2013-12-09 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 58123cf9254..38b9887e216 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -220,7 +220,7 @@ Hadoop MapReduce Next Generation - Capacity Scheduler | | application, no single user can use more than 33% of the queue resources. | | | With 4 or more users, no user can use more than 25% of the queues | | | resources. A value of 100 implies no user limits are imposed. The default | -| | is 100.| +| | is 100. Value is specified as a integer.| *--------------------------------------+--------------------------------------+ | <<.user-limit-factor>>> | | | | The multiple of the queue capacity which can be configured to allow a | @@ -249,6 +249,7 @@ Hadoop MapReduce Next Generation - Capacity Scheduler | | be rejected. Default is 10000. This can be set for all queues with | | | <<>> and can also be overridden on a | | | per queue basis by setting <<.maximum-applications>>>. | +| | Integer value expected.| *--------------------------------------+--------------------------------------+ | <<>> / | | <<.maximum-am-resource-percent>>> | | @@ -276,7 +277,7 @@ Hadoop MapReduce Next Generation - Capacity Scheduler | | Thus, if the queue is <<>> no applications can be | | | submitted to the entire cluster. | | | Existing applications continue to completion, thus the queue can be -| | gracefully. | +| | gracefully. Value is specified as Enumeration. | *--------------------------------------+--------------------------------------+ | <<.acl_submit_applications>>> | | | | The which controls who can applications to the given queue. | From 1b9b95692412f3866fe716e56bbcf756a15b70e4 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 20 Dec 2013 19:53:59 +0000 Subject: [PATCH 03/11] MAPREDUCE-5692. Add explicit diagnostics when a task attempt is killed due to speculative execution (Gera Shegalov via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552797 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../v2/app/job/impl/TaskAttemptImpl.java | 18 ++++++++++ .../mapreduce/v2/app/job/impl/TaskImpl.java | 14 ++++---- .../v2/TestSpeculativeExecutionWithMRApp.java | 36 +++++++++++++++++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e3b559eaf32..8d9b61e0fb1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -187,6 +187,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5052. Job History UI and web services confusing job start time and job submit time (Chen He via jeagles) + MAPREDUCE-5692. Add explicit diagnostics when a task attempt is killed due + to speculative execution (Gera Shegalov via Sandy Ryza) + OPTIMIZATIONS MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index f3c62a48d5e..5e14ce1cb52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1552,6 +1552,12 @@ public abstract class TaskAttemptImpl implements TaskAttemptEvent event) { //set the finish time taskAttempt.setFinishTime(); + + if (event instanceof TaskAttemptKillEvent) { + taskAttempt.addDiagnosticInfo( + ((TaskAttemptKillEvent) event).getMessage()); + } + //send the deallocate event to ContainerAllocator taskAttempt.eventHandler.handle( new ContainerAllocatorEvent(taskAttempt.attemptId, @@ -1855,6 +1861,12 @@ public abstract class TaskAttemptImpl implements LOG.debug("Not generating HistoryFinish event since start event not " + "generated for taskAttempt: " + taskAttempt.getID()); } + + if (event instanceof TaskAttemptKillEvent) { + taskAttempt.addDiagnosticInfo( + ((TaskAttemptKillEvent) event).getMessage()); + } + // taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.KILLED); Not logging Map/Reduce attempts in case of failure. taskAttempt.eventHandler.handle(new TaskTAttemptEvent( taskAttempt.attemptId, @@ -1872,6 +1884,12 @@ public abstract class TaskAttemptImpl implements // for it taskAttempt.taskAttemptListener.unregister( taskAttempt.attemptId, taskAttempt.jvmID); + + if (event instanceof TaskAttemptKillEvent) { + taskAttempt.addDiagnosticInfo( + ((TaskAttemptKillEvent) event).getMessage()); + } + taskAttempt.reportedStatus.progress = 1.0f; taskAttempt.updateProgressSplits(); //send the cleanup event to containerLauncher diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index 75d833652af..efb46d538b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -69,6 +69,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobMapTaskRescheduledEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptRecoverEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; @@ -100,6 +101,7 @@ import com.google.common.annotations.VisibleForTesting; public abstract class TaskImpl implements Task, EventHandler { private static final Log LOG = LogFactory.getLog(TaskImpl.class); + private static final String SPECULATION = "Speculation: "; protected final JobConf conf; protected final Path jobFile; @@ -906,8 +908,8 @@ public abstract class TaskImpl implements Task, EventHandler { LOG.info(task.commitAttempt + " already given a go for committing the task output, so killing " + attemptID); - task.eventHandler.handle(new TaskAttemptEvent( - attemptID, TaskAttemptEventType.TA_KILL)); + task.eventHandler.handle(new TaskAttemptKillEvent(attemptID, + SPECULATION + task.commitAttempt + " committed first!")); } } } @@ -932,9 +934,8 @@ public abstract class TaskImpl implements Task, EventHandler { // other reasons. !attempt.isFinished()) { LOG.info("Issuing kill to other attempt " + attempt.getID()); - task.eventHandler.handle( - new TaskAttemptEvent(attempt.getID(), - TaskAttemptEventType.TA_KILL)); + task.eventHandler.handle(new TaskAttemptKillEvent(attempt.getID(), + SPECULATION + task.successfulAttempt + " succeeded first!")); } } task.finished(TaskStateInternal.SUCCEEDED); @@ -1199,8 +1200,7 @@ public abstract class TaskImpl implements Task, EventHandler { private void killUnfinishedAttempt(TaskAttempt attempt, String logMsg) { if (attempt != null && !attempt.isFinished()) { eventHandler.handle( - new TaskAttemptEvent(attempt.getID(), - TaskAttemptEventType.TA_KILL)); + new TaskAttemptKillEvent(attempt.getID(), logMsg)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java index 37d09e0da38..9c88c67b664 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce.v2; +import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.Random; @@ -106,17 +107,21 @@ public class TestSpeculativeExecutionWithMRApp { int maxTimeWait = 10; boolean successfullySpeculated = false; + TaskAttempt[] ta = null; while (maxTimeWait > 0 && !successfullySpeculated) { if (taskToBeSpeculated.getAttempts().size() != 2) { Thread.sleep(1000); clock.setTime(System.currentTimeMillis() + 20000); } else { successfullySpeculated = true; + // finish 1st TA, 2nd will be killed + ta = makeFirstAttemptWin(appEventHandler, taskToBeSpeculated); } maxTimeWait--; } Assert .assertTrue("Couldn't speculate successfully", successfullySpeculated); + verifySpeculationMessage(app, ta); } @Test(timeout = 60000) @@ -197,16 +202,47 @@ public class TestSpeculativeExecutionWithMRApp { int maxTimeWait = 5; boolean successfullySpeculated = false; + TaskAttempt[] ta = null; while (maxTimeWait > 0 && !successfullySpeculated) { if (speculatedTask.getAttempts().size() != 2) { Thread.sleep(1000); } else { successfullySpeculated = true; + ta = makeFirstAttemptWin(appEventHandler, speculatedTask); } maxTimeWait--; } Assert .assertTrue("Couldn't speculate successfully", successfullySpeculated); + verifySpeculationMessage(app, ta); + } + + private static TaskAttempt[] makeFirstAttemptWin( + EventHandler appEventHandler, Task speculatedTask) { + + // finish 1st TA, 2nd will be killed + Collection attempts = speculatedTask.getAttempts().values(); + TaskAttempt[] ta = new TaskAttempt[attempts.size()]; + attempts.toArray(ta); + appEventHandler.handle( + new TaskAttemptEvent(ta[0].getID(), TaskAttemptEventType.TA_DONE)); + appEventHandler.handle(new TaskAttemptEvent(ta[0].getID(), + TaskAttemptEventType.TA_CONTAINER_CLEANED)); + return ta; + } + + private static void verifySpeculationMessage(MRApp app, TaskAttempt[] ta) + throws Exception { + app.waitForState(ta[0], TaskAttemptState.SUCCEEDED); + app.waitForState(ta[1], TaskAttemptState.KILLED); + boolean foundSpecMsg = false; + for (String msg : ta[1].getDiagnostics()) { + if (msg.contains("Speculation")) { + foundSpecMsg = true; + break; + } + } + Assert.assertTrue("No speculation diagnostics!", foundSpecMsg); } private TaskAttemptStatus createTaskAttemptStatus(TaskAttemptId id, From 17a3d72d3b5a8b249ac1566b80913356c080919b Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 20 Dec 2013 20:07:46 +0000 Subject: [PATCH 04/11] HDFS-5691. Fix typo in ShortCircuitLocalRead document. Contributed by Akira Ajisaka. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552802 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5949df30176..20d969d848e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -822,6 +822,9 @@ Release 2.4.0 - UNRELEASED HDFS-5676. fix inconsistent synchronization of CachingStrategy (cmccabe) + HDFS-5691. Fix typo in ShortCircuitLocalRead document. + (Akira Ajisaka via suresh) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm index 58f07124161..daa4c2fc36f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm @@ -73,7 +73,7 @@ HDFS Short-Circuit Local Reads This configuration parameter turns on short-circuit local reads. - * dfs.client.read.shortcircuit.skip.checkusm + * dfs.client.read.shortcircuit.skip.checksum If this configuration parameter is set, short-circuit local reads will skip checksums. This is normally not recommended, but it may be useful for From 04dac636953532c9c4ab6692e193e3ffe292edb2 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 20 Dec 2013 21:00:25 +0000 Subject: [PATCH 05/11] MAPREDUCE-5550. Task Status message (reporter.setStatus) not shown in UI with Hadoop 2.0 (Gera Shegalov via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552808 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/v2/app/job/impl/TaskImpl.java | 10 ++++- .../mapreduce/v2/app/webapp/TaskPage.java | 8 ++-- .../mapreduce/v2/app/webapp/TasksBlock.java | 2 + .../mapreduce/v2/app/webapp/TasksPage.java | 4 +- .../v2/app/webapp/dao/TaskAttemptInfo.java | 30 ++++++------- .../mapreduce/v2/app/webapp/dao/TaskInfo.java | 5 +++ .../hadoop/mapreduce/v2/app/MockJobs.java | 27 ++++++++---- .../app/webapp/TestAMWebServicesAttempts.java | 11 ++--- .../v2/app/webapp/TestAMWebServicesTasks.java | 11 +++-- .../mapreduce/v2/api/records/TaskReport.java | 3 +- .../api/records/impl/pb/TaskReportPBImpl.java | 12 ++++++ .../mapreduce/v2/hs/webapp/HsTaskPage.java | 42 +++++++++++-------- .../mapreduce/v2/hs/webapp/TestBlocks.java | 31 +++++++++++--- .../hs/webapp/TestHsWebServicesAttempts.java | 10 ++--- .../v2/hs/webapp/TestHsWebServicesTasks.java | 2 +- 16 files changed, 144 insertions(+), 67 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8d9b61e0fb1..4dcc7f37d9b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -190,6 +190,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5692. Add explicit diagnostics when a task attempt is killed due to speculative execution (Gera Shegalov via Sandy Ryza) + MAPREDUCE-5550. Task Status message (reporter.setStatus) not shown in UI + with Hadoop 2.0 (Gera Shegalov via Sandy Ryza) + OPTIMIZATIONS MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index efb46d538b4..9d14d4e2b6a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -376,11 +376,15 @@ public abstract class TaskImpl implements Task, EventHandler { TaskReport report = recordFactory.newRecordInstance(TaskReport.class); readLock.lock(); try { + TaskAttempt bestAttempt = selectBestAttempt(); report.setTaskId(taskId); report.setStartTime(getLaunchTime()); report.setFinishTime(getFinishTime()); report.setTaskState(getState()); - report.setProgress(getProgress()); + report.setProgress(bestAttempt == null ? 0f : bestAttempt.getProgress()); + report.setStatus(bestAttempt == null + ? "" + : bestAttempt.getReport().getStateString()); for (TaskAttempt attempt : attempts.values()) { if (TaskAttemptState.RUNNING.equals(attempt.getState())) { @@ -400,7 +404,9 @@ public abstract class TaskImpl implements Task, EventHandler { // Add a copy of counters as the last step so that their lifetime on heap // is as small as possible. - report.setCounters(TypeConverter.toYarn(getCounters())); + report.setCounters(TypeConverter.toYarn(bestAttempt == null + ? TaskAttemptImpl.EMPTY_COUNTERS + : bestAttempt.getCounters())); return report; } finally { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java index 866c7f1ebb4..47d9f1ea59f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java @@ -63,6 +63,7 @@ public class TaskPage extends AppView { th(".id", "Attempt"). th(".progress", "Progress"). th(".state", "State"). + th(".status", "Status"). th(".node", "Node"). th(".logs", "Logs"). th(".tsh", "Started"). @@ -84,6 +85,7 @@ public class TaskPage extends AppView { .append(ta.getId()).append("\",\"") .append(progress).append("\",\"") .append(ta.getState().toString()).append("\",\"") + .append(ta.getStatus()).append("\",\"") .append(nodeHttpAddr == null ? "N/A" : "" @@ -144,13 +146,13 @@ public class TaskPage extends AppView { .append("\n,aoColumnDefs:[\n") //logs column should not filterable (it includes container ID which may pollute searches) - .append("\n{'aTargets': [ 4 ]") + .append("\n{'aTargets': [ 5 ]") .append(", 'bSearchable': false }") - .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6") + .append("\n, {'sType':'numeric', 'aTargets': [ 6, 7") .append(" ], 'mRender': renderHadoopDate }") - .append("\n, {'sType':'numeric', 'aTargets': [ 7") + .append("\n, {'sType':'numeric', 'aTargets': [ 8") .append(" ], 'mRender': renderHadoopElapsedTime }]") // Sort by id upon page load diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java index dc6bbb24d48..e2c65e320b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java @@ -59,6 +59,7 @@ public class TasksBlock extends HtmlBlock { tr(). th("Task"). th("Progress"). + th("Status"). th("State"). th("Start Time"). th("Finish Time"). @@ -81,6 +82,7 @@ public class TasksBlock extends HtmlBlock { .append(join(pct, '%')).append("'> ").append("
\",\"") + .append(info.getStatus()).append("\",\"") .append(info.getState()).append("\",\"") .append(info.getStartTime()).append("\",\"") diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java index 3753b1ea39d..0212ae4f741 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java @@ -50,10 +50,10 @@ public class TasksPage extends AppView { .append(", 'mRender': parseHadoopProgress }") - .append("\n, {'sType':'numeric', 'aTargets': [3, 4]") + .append("\n, {'sType':'numeric', 'aTargets': [4, 5]") .append(", 'mRender': renderHadoopDate }") - .append("\n, {'sType':'numeric', 'aTargets': [5]") + .append("\n, {'sType':'numeric', 'aTargets': [6]") .append(", 'mRender': renderHadoopElapsedTime }]") // Sort by id upon page load diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java index c35411b0a2e..d8e89b1cbc9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; @@ -45,6 +46,7 @@ public class TaskAttemptInfo { protected String id; protected String rack; protected TaskAttemptState state; + protected String status; protected String nodeHttpAddress; protected String diagnostics; protected String type; @@ -61,29 +63,23 @@ public class TaskAttemptInfo { } public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) { + final TaskAttemptReport report = ta.getReport(); this.type = type.toString(); this.id = MRApps.toString(ta.getID()); this.nodeHttpAddress = ta.getNodeHttpAddress(); - this.startTime = ta.getLaunchTime(); - this.finishTime = ta.getFinishTime(); - this.assignedContainerId = ConverterUtils.toString(ta - .getAssignedContainerID()); - this.assignedContainer = ta.getAssignedContainerID(); - this.progress = ta.getProgress() * 100; - this.state = ta.getState(); + this.startTime = report.getStartTime(); + this.finishTime = report.getFinishTime(); + this.assignedContainerId = ConverterUtils.toString(report.getContainerId()); + this.assignedContainer = report.getContainerId(); + this.progress = report.getProgress() * 100; + this.status = report.getStateString(); + this.state = report.getTaskAttemptState(); this.elapsedTime = Times .elapsed(this.startTime, this.finishTime, isRunning); if (this.elapsedTime == -1) { this.elapsedTime = 0; } - List diagnostics = ta.getDiagnostics(); - if (diagnostics != null && !diagnostics.isEmpty()) { - StringBuffer b = new StringBuffer(); - for (String diag : diagnostics) { - b.append(diag); - } - this.diagnostics = b.toString(); - } + this.diagnostics = report.getDiagnosticInfo(); this.rack = ta.getNodeRackName(); } @@ -99,6 +95,10 @@ public class TaskAttemptInfo { return this.state.toString(); } + public String getStatus() { + return status; + } + public String getId() { return this.id; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java index 40983400aef..00305a83e40 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java @@ -43,6 +43,7 @@ public class TaskInfo { protected TaskState state; protected String type; protected String successfulAttempt; + protected String status; @XmlTransient int taskNum; @@ -66,6 +67,7 @@ public class TaskInfo { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; + this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); @@ -121,4 +123,7 @@ public class TaskInfo { return null; } + public String getStatus() { + return status; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index c018096c358..eb4919c9cde 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -174,22 +174,37 @@ public class MockJobs extends MockApps { report.setFinishTime(System.currentTimeMillis() + (int) (Math.random() * DT) + 1); report.setProgress((float) Math.random()); + report.setStatus("Moving average: " + Math.random()); report.setCounters(TypeConverter.toYarn(newCounters())); report.setTaskState(TASK_STATES.next()); return report; } public static TaskAttemptReport newTaskAttemptReport(TaskAttemptId id) { + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( + id.getTaskId().getJobId().getAppId(), 0); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class); report.setTaskAttemptId(id); report .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT)); report.setFinishTime(System.currentTimeMillis() + (int) (Math.random() * DT) + 1); + + if (id.getTaskId().getTaskType() == TaskType.REDUCE) { + report.setShuffleFinishTime( + (report.getFinishTime() + report.getStartTime()) / 2); + report.setSortFinishTime( + (report.getFinishTime() + report.getShuffleFinishTime()) / 2); + } + report.setPhase(PHASES.next()); report.setTaskAttemptState(TASK_ATTEMPT_STATES.next()); report.setProgress((float) Math.random()); report.setCounters(TypeConverter.toYarn(newCounters())); + report.setContainerId(containerId); + report.setDiagnosticInfo(DIAGS.next()); + report.setStateString("Moving average " + Math.random()); return report; } @@ -230,8 +245,6 @@ public class MockJobs extends MockApps { taid.setTaskId(tid); taid.setId(i); final TaskAttemptReport report = newTaskAttemptReport(taid); - final List diags = Lists.newArrayList(); - diags.add(DIAGS.next()); return new TaskAttempt() { @Override public NodeId getNodeId() throws UnsupportedOperationException{ @@ -250,12 +263,12 @@ public class MockJobs extends MockApps { @Override public long getLaunchTime() { - return 0; + return report.getStartTime(); } @Override public long getFinishTime() { - return 0; + return report.getFinishTime(); } @Override @@ -313,7 +326,7 @@ public class MockJobs extends MockApps { @Override public List getDiagnostics() { - return diags; + return Lists.newArrayList(report.getDiagnosticInfo()); } @Override @@ -323,12 +336,12 @@ public class MockJobs extends MockApps { @Override public long getShuffleFinishTime() { - return 0; + return report.getShuffleFinishTime(); } @Override public long getSortFinishTime() { - return 0; + return report.getSortFinishTime(); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java index 886270bf1bb..dcd5d2954ba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java @@ -1,3 +1,4 @@ +/** /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -425,9 +426,9 @@ public class TestAMWebServicesAttempts extends JerseyTest { public void verifyAMTaskAttempt(JSONObject info, TaskAttempt att, TaskType ttype) throws JSONException { if (ttype == TaskType.REDUCE) { - assertEquals("incorrect number of elements", 16, info.length()); + assertEquals("incorrect number of elements", 17, info.length()); } else { - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); } verifyTaskAttemptGeneric(att, ttype, info.getString("id"), @@ -532,11 +533,11 @@ public class TestAMWebServicesAttempts extends JerseyTest { assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(), mergeFinishTime); assertEquals("elapsedShuffleTime wrong", - ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime); + ta.getShuffleFinishTime() - ta.getLaunchTime(), elapsedShuffleTime); assertEquals("elapsedMergeTime wrong", - ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime); + ta.getSortFinishTime() - ta.getShuffleFinishTime(), elapsedMergeTime); assertEquals("elapsedReduceTime wrong", - ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime); + ta.getFinishTime() - ta.getSortFinishTime(), elapsedReduceTime); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java index b8ece70f323..8bf1bb7b752 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java @@ -525,12 +525,13 @@ public class TestAMWebServicesTasks extends JerseyTest { public void verifyAMSingleTask(JSONObject info, Task task) throws JSONException { - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements", 9, info.length()); verifyTaskGeneric(task, info.getString("id"), info.getString("state"), info.getString("type"), info.getString("successfulAttempt"), info.getLong("startTime"), info.getLong("finishTime"), - info.getLong("elapsedTime"), (float) info.getDouble("progress")); + info.getLong("elapsedTime"), (float) info.getDouble("progress"), + info.getString("status")); } public void verifyAMTask(JSONArray arr, Job job, String type) @@ -555,7 +556,7 @@ public class TestAMWebServicesTasks extends JerseyTest { public void verifyTaskGeneric(Task task, String id, String state, String type, String successfulAttempt, long startTime, long finishTime, - long elapsedTime, float progress) { + long elapsedTime, float progress, String status) { TaskId taskid = task.getID(); String tid = MRApps.toString(taskid); @@ -572,6 +573,7 @@ public class TestAMWebServicesTasks extends JerseyTest { assertEquals("finishTime wrong", report.getFinishTime(), finishTime); assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime); assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f); + assertEquals("status wrong", report.getStatus(), status); } public void verifyAMSingleTaskXML(Element element, Task task) { @@ -582,7 +584,8 @@ public class TestAMWebServicesTasks extends JerseyTest { WebServicesTestUtils.getXmlLong(element, "startTime"), WebServicesTestUtils.getXmlLong(element, "finishTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), - WebServicesTestUtils.getXmlFloat(element, "progress")); + WebServicesTestUtils.getXmlFloat(element, "progress"), + WebServicesTestUtils.getXmlString(element, "status")); } public void verifyAMTaskXML(NodeList nodes, Job job) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java index 66cca13f5d1..1444a53f3ee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java @@ -24,10 +24,10 @@ public interface TaskReport { public abstract TaskId getTaskId(); public abstract TaskState getTaskState(); public abstract float getProgress(); + public abstract String getStatus(); public abstract long getStartTime(); public abstract long getFinishTime(); public abstract Counters getCounters(); - public abstract List getRunningAttemptsList(); public abstract TaskAttemptId getRunningAttempt(int index); public abstract int getRunningAttemptsCount(); @@ -42,6 +42,7 @@ public interface TaskReport { public abstract void setTaskId(TaskId taskId); public abstract void setTaskState(TaskState taskState); public abstract void setProgress(float progress); + public abstract void setStatus(String status); public abstract void setStartTime(long startTime); public abstract void setFinishTime(long finishTime); public abstract void setCounters(Counters counters); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java index 9801a16f39c..ba1245c2e3d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java @@ -49,6 +49,7 @@ public class TaskReportPBImpl extends ProtoBase implements Task private List runningAttempts = null; private TaskAttemptId successfulAttemptId = null; private List diagnostics = null; + private String status; public TaskReportPBImpl() { @@ -171,11 +172,22 @@ public class TaskReportPBImpl extends ProtoBase implements Task return (p.getProgress()); } + @Override + public String getStatus() { + return status; + } + @Override public void setProgress(float progress) { maybeInitBuilder(); builder.setProgress((progress)); } + + @Override + public void setStatus(String status) { + this.status = status; + } + @Override public TaskState getTaskState() { TaskReportProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java index 4d8a8cfc7b1..fde3a3a03f5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java @@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.webapp.App; +import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; import org.apache.hadoop.yarn.util.Times; @@ -89,6 +90,7 @@ public class HsTaskPage extends HsView { headRow. th(".id", "Attempt"). th(".state", "State"). + th(".status", "Status"). th(".node", "Node"). th(".logs", "Logs"). th(".tsh", "Start Time"); @@ -113,15 +115,16 @@ public class HsTaskPage extends HsView { // DataTables to display StringBuilder attemptsTableData = new StringBuilder("[\n"); - for (TaskAttempt ta : getTaskAttempts()) { - String taid = MRApps.toString(ta.getID()); + for (TaskAttempt attempt : getTaskAttempts()) { + final TaskAttemptInfo ta = new TaskAttemptInfo(attempt, false); + String taid = ta.getId(); - String nodeHttpAddr = ta.getNodeHttpAddress(); - String containerIdString = ta.getAssignedContainerID().toString(); - String nodeIdString = ta.getAssignedContainerMgrAddress(); - String nodeRackName = ta.getNodeRackName(); + String nodeHttpAddr = ta.getNode(); + String containerIdString = ta.getAssignedContainerIdStr(); + String nodeIdString = attempt.getAssignedContainerMgrAddress(); + String nodeRackName = ta.getRack(); - long attemptStartTime = ta.getLaunchTime(); + long attemptStartTime = ta.getStartTime(); long shuffleFinishTime = -1; long sortFinishTime = -1; long attemptFinishTime = ta.getFinishTime(); @@ -129,8 +132,8 @@ public class HsTaskPage extends HsView { long elapsedSortTime = -1; long elapsedReduceTime = -1; if(type == TaskType.REDUCE) { - shuffleFinishTime = ta.getShuffleFinishTime(); - sortFinishTime = ta.getSortFinishTime(); + shuffleFinishTime = attempt.getShuffleFinishTime(); + sortFinishTime = attempt.getSortFinishTime(); elapsedShuffleTime = Times.elapsed(attemptStartTime, shuffleFinishTime, false); elapsedSortTime = @@ -140,11 +143,13 @@ public class HsTaskPage extends HsView { } long attemptElapsed = Times.elapsed(attemptStartTime, attemptFinishTime, false); - int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000); + int sortId = attempt.getID().getId() + + (attempt.getID().getTaskId().getId() * 10000); attemptsTableData.append("[\"") .append(sortId + " ").append(taid).append("\",\"") - .append(ta.getState().toString()).append("\",\"") + .append(ta.getState()).append("\",\"") + .append(ta.getStatus()).append("\",\"") .append("") .append(nodeRackName + "/" + nodeHttpAddr + "\",\"") @@ -167,8 +172,9 @@ public class HsTaskPage extends HsView { .append(elapsedReduceTime).append("\",\""); } attemptsTableData.append(attemptElapsed).append("\",\"") - .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( - Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n"); + .append(StringEscapeUtils.escapeJavaScript( + StringEscapeUtils.escapeHtml(ta.getNote()))) + .append("\"],\n"); } //Remove the last comma and close off the array of arrays if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { @@ -184,6 +190,8 @@ public class HsTaskPage extends HsView { $name("attempt_name").$value("Attempt")._()._(). th().input("search_init").$type(InputType.text). $name("attempt_state").$value("State")._()._(). + th().input("search_init").$type(InputType.text). + $name("attempt_status").$value("Status")._()._(). th().input("search_init").$type(InputType.text). $name("attempt_node").$value("Node")._()._(). th().input("search_init").$type(InputType.text). @@ -283,19 +291,19 @@ public class HsTaskPage extends HsView { .append("\n,aoColumnDefs:[\n") //logs column should not filterable (it includes container ID which may pollute searches) - .append("\n{'aTargets': [ 3 ]") + .append("\n{'aTargets': [ 4 ]") .append(", 'bSearchable': false }") .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]") .append(", 'mRender': parseHadoopAttemptID }") - .append("\n, {'sType':'numeric', 'aTargets': [ 4, 5") + .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6") //Column numbers are different for maps and reduces - .append(type == TaskType.REDUCE ? ", 6, 7" : "") + .append(type == TaskType.REDUCE ? ", 7, 8" : "") .append(" ], 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', 'aTargets': [") - .append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6") + .append(type == TaskType.REDUCE ? "9, 10, 11, 12" : "7") .append(" ], 'mRender': renderHadoopElapsedTime }]") // Sort by id upon page load diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java index 1d22c313a71..241bdb246fd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java @@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskReport; @@ -138,11 +139,31 @@ public class TestBlocks { when(attempt.getAssignedContainerMgrAddress()).thenReturn( "assignedContainerMgrAddress"); when(attempt.getNodeRackName()).thenReturn("nodeRackName"); - when(attempt.getLaunchTime()).thenReturn(100002L); - when(attempt.getFinishTime()).thenReturn(100012L); - when(attempt.getShuffleFinishTime()).thenReturn(100010L); - when(attempt.getSortFinishTime()).thenReturn(100011L); - when(attempt.getState()).thenReturn(TaskAttemptState.SUCCEEDED); + + final long taStartTime = 100002L; + final long taFinishTime = 100012L; + final long taShuffleFinishTime = 100010L; + final long taSortFinishTime = 100011L; + final TaskAttemptState taState = TaskAttemptState.SUCCEEDED; + + when(attempt.getLaunchTime()).thenReturn(taStartTime); + when(attempt.getFinishTime()).thenReturn(taFinishTime); + when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime); + when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime); + when(attempt.getState()).thenReturn(taState); + + TaskAttemptReport taReport = mock(TaskAttemptReport.class); + when(taReport.getStartTime()).thenReturn(taStartTime); + when(taReport.getFinishTime()).thenReturn(taFinishTime); + when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime); + when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime); + when(taReport.getContainerId()).thenReturn(containerId); + when(taReport.getProgress()).thenReturn(1.0f); + when(taReport.getStateString()).thenReturn("Processed 128/128 records"); + when(taReport.getTaskAttemptState()).thenReturn(taState); + when(taReport.getDiagnosticInfo()).thenReturn(""); + + when(attempt.getReport()).thenReturn(taReport); attempts.put(taId, attempt); when(task.getAttempts()).thenReturn(attempts); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java index 8a5a6db88a3..60dc235d684 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java @@ -444,9 +444,9 @@ public class TestHsWebServicesAttempts extends JerseyTest { public void verifyHsTaskAttempt(JSONObject info, TaskAttempt att, TaskType ttype) throws JSONException { if (ttype == TaskType.REDUCE) { - assertEquals("incorrect number of elements", 16, info.length()); + assertEquals("incorrect number of elements", 17, info.length()); } else { - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); } verifyTaskAttemptGeneric(att, ttype, info.getString("id"), @@ -551,11 +551,11 @@ public class TestHsWebServicesAttempts extends JerseyTest { assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(), mergeFinishTime); assertEquals("elapsedShuffleTime wrong", - ta.getLaunchTime() - ta.getShuffleFinishTime(), elapsedShuffleTime); + ta.getShuffleFinishTime() - ta.getLaunchTime(), elapsedShuffleTime); assertEquals("elapsedMergeTime wrong", - ta.getShuffleFinishTime() - ta.getSortFinishTime(), elapsedMergeTime); + ta.getSortFinishTime() - ta.getShuffleFinishTime(), elapsedMergeTime); assertEquals("elapsedReduceTime wrong", - ta.getSortFinishTime() - ta.getFinishTime(), elapsedReduceTime); + ta.getFinishTime() - ta.getSortFinishTime(), elapsedReduceTime); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index e76f37c24c0..ee0ccc6afcf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -538,7 +538,7 @@ public class TestHsWebServicesTasks extends JerseyTest { public void verifyHsSingleTask(JSONObject info, Task task) throws JSONException { - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements", 9, info.length()); verifyTaskGeneric(task, info.getString("id"), info.getString("state"), info.getString("type"), info.getString("successfulAttempt"), From 71b4903ea41a17c9305b25d24be786aed5b6e82f Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 20 Dec 2013 22:06:48 +0000 Subject: [PATCH 06/11] HADOOP-10169. Remove the unnecessary synchronized in JvmMetrics class. Contributed by Liang Xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552820 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../hadoop/metrics2/source/JvmMetrics.java | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 0a6962f6f5a..4ffa26b7a84 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -404,6 +404,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10164. Allow UGI to login with a known Subject (bobby) + HADOOP-10169. Remove the unnecessary synchronized in JvmMetrics class. + (Liang Xie via jing9) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java index 35c92bbe92c..750499139a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java @@ -24,10 +24,8 @@ import java.lang.management.MemoryUsage; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.lang.management.GarbageCollectorMXBean; -import java.util.Map; import java.util.List; - -import com.google.common.collect.Maps; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.log.metrics.EventCounter; @@ -67,7 +65,8 @@ public class JvmMetrics implements MetricsSource { ManagementFactory.getGarbageCollectorMXBeans(); final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); final String processName, sessionId; - final Map gcInfoCache = Maps.newHashMap(); + final ConcurrentHashMap gcInfoCache = + new ConcurrentHashMap(); JvmMetrics(String processName, String sessionId) { this.processName = processName; @@ -123,13 +122,17 @@ public class JvmMetrics implements MetricsSource { .addCounter(GcTimeMillis, timeMillis); } - private synchronized MetricsInfo[] getGcInfo(String gcName) { + private MetricsInfo[] getGcInfo(String gcName) { MetricsInfo[] gcInfo = gcInfoCache.get(gcName); if (gcInfo == null) { gcInfo = new MetricsInfo[2]; - gcInfo[0] = Interns.info("GcCount"+ gcName, "GC Count for "+ gcName); - gcInfo[1] = Interns.info("GcTimeMillis"+ gcName, "GC Time for "+ gcName); - gcInfoCache.put(gcName, gcInfo); + gcInfo[0] = Interns.info("GcCount" + gcName, "GC Count for " + gcName); + gcInfo[1] = Interns + .info("GcTimeMillis" + gcName, "GC Time for " + gcName); + MetricsInfo[] previousGcInfo = gcInfoCache.putIfAbsent(gcName, gcInfo); + if (previousGcInfo != null) { + return previousGcInfo; + } } return gcInfo; } From 86e5db39ba7828c771716bc983861343ba323047 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 20 Dec 2013 23:02:04 +0000 Subject: [PATCH 07/11] HDFS-2933. Fix incorrect target version in CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552837 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 20d969d848e..9d6f98fe215 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -763,6 +763,9 @@ Release 2.4.0 - UNRELEASED HDFS-5540. Fix intermittent failure in TestBlocksWithNotEnoughRacks. (Binglin Chang via junping_du) + HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via + Arpit Agarwal) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) @@ -1137,9 +1140,6 @@ Release 2.1.1-beta - 2013-09-23 HDFS-5047. Supress logging of full stack trace of quota and lease exceptions. (Robert Parker via kihwal) - HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via - Arpit Agarwal) - HDFS-5111. Remove duplicated error message for snapshot commands when processing invalid arguments. (jing9) From b9ae3087c0f83bfeeea47ded8e19932b46fd2350 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 20 Dec 2013 23:27:20 +0000 Subject: [PATCH 08/11] HDFS-5636. Enforce a max TTL per cache pool (awang via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552841 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 17 +- .../hadoop/hdfs/protocol/CacheDirective.java | 8 + .../hdfs/protocol/CacheDirectiveInfo.java | 27 ++- .../hadoop/hdfs/protocol/CachePoolInfo.java | 73 ++++++- .../hadoop/hdfs/protocolPB/PBHelper.java | 6 + .../CacheReplicationMonitor.java | 2 +- .../hdfs/server/namenode/CacheManager.java | 183 +++++++++++----- .../hdfs/server/namenode/CachePool.java | 33 ++- .../hdfs/server/namenode/FSEditLogLoader.java | 4 +- .../server/namenode/FSImageSerialization.java | 25 ++- .../apache/hadoop/hdfs/tools/CacheAdmin.java | 199 ++++++++++++------ .../main/proto/ClientNamenodeProtocol.proto | 1 + .../org/apache/hadoop/hdfs/TestDFSUtil.java | 22 +- .../server/namenode/TestCacheDirectives.java | 185 ++++++++++++++++ .../src/test/resources/editsStored.xml | 143 ++++++------- .../src/test/resources/testCacheAdminConf.xml | 36 +++- 17 files changed, 749 insertions(+), 217 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9d6f98fe215..34678de50bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -241,6 +241,8 @@ Trunk (Unreleased) HDFS-5431. Support cachepool-based limit management in path-based caching (awang via cmccabe) + HDFS-5636. Enforce a max TTL per cache pool. (awang via cmccabe) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index d03ad418c15..112fa063f16 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1546,7 +1546,11 @@ public class DFSUtil { * Converts a time duration in milliseconds into DDD:HH:MM:SS format. */ public static String durationToString(long durationMs) { - Preconditions.checkArgument(durationMs >= 0, "Invalid negative duration"); + boolean negative = false; + if (durationMs < 0) { + negative = true; + durationMs = -durationMs; + } // Chop off the milliseconds long durationSec = durationMs / 1000; final int secondsPerMinute = 60; @@ -1559,7 +1563,12 @@ public class DFSUtil { final long minutes = durationSec / secondsPerMinute; durationSec -= minutes * secondsPerMinute; final long seconds = durationSec; - return String.format("%03d:%02d:%02d:%02d", days, hours, minutes, seconds); + final long milliseconds = durationMs % 1000; + String format = "%03d:%02d:%02d:%02d.%03d"; + if (negative) { + format = "-" + format; + } + return String.format(format, days, hours, minutes, seconds, milliseconds); } /** @@ -1571,9 +1580,9 @@ public class DFSUtil { + ": too short"); } String ttlString = relTime.substring(0, relTime.length()-1); - int ttl; + long ttl; try { - ttl = Integer.parseInt(ttlString); + ttl = Long.parseLong(ttlString); } catch (NumberFormatException e) { throw new IOException("Unable to parse relative time value of " + relTime + ": " + ttlString + " is not a number"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java index b6964cac966..89cf641a02d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java @@ -52,6 +52,14 @@ public final class CacheDirective implements IntrusiveCollection.Element { private Element prev; private Element next; + public CacheDirective(CacheDirectiveInfo info) { + this( + info.getId(), + info.getPath().toUri().getPath(), + info.getReplication(), + info.getExpiration().getAbsoluteMillis()); + } + public CacheDirective(long id, String path, short replication, long expiryTime) { Preconditions.checkArgument(id > 0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java index d7a911123e9..f6b3c34f4ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java @@ -26,6 +26,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; +import com.google.common.base.Preconditions; + /** * Describes a path-based cache directive. */ @@ -138,11 +140,22 @@ public class CacheDirectiveInfo { */ public static class Expiration { - /** Denotes a CacheDirectiveInfo that never expires **/ - public static final int EXPIRY_NEVER = -1; + /** + * The maximum value we accept for a relative expiry. + */ + public static final long MAX_RELATIVE_EXPIRY_MS = + Long.MAX_VALUE / 4; // This helps prevent weird overflow bugs + + /** + * An relative Expiration that never expires. + */ + public static final Expiration NEVER = newRelative(MAX_RELATIVE_EXPIRY_MS); /** * Create a new relative Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param ms how long until the CacheDirective expires, in milliseconds * @return A relative Expiration @@ -153,6 +166,9 @@ public class CacheDirectiveInfo { /** * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param date when the CacheDirective expires * @return An absolute Expiration @@ -163,6 +179,9 @@ public class CacheDirectiveInfo { /** * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param ms when the CacheDirective expires, in milliseconds since the Unix * epoch. @@ -176,6 +195,10 @@ public class CacheDirectiveInfo { private final boolean isRelative; private Expiration(long ms, boolean isRelative) { + if (isRelative) { + Preconditions.checkArgument(ms <= MAX_RELATIVE_EXPIRY_MS, + "Expiration time is too far in the future!"); + } this.ms = ms; this.isRelative = isRelative; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java index 98a7dd8e202..61bbe387b9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; /** * CachePoolInfo describes a cache pool. @@ -42,6 +43,20 @@ import org.apache.hadoop.fs.permission.FsPermission; public class CachePoolInfo { public static final Log LOG = LogFactory.getLog(CachePoolInfo.class); + /** + * Indicates that the pool does not have a maximum relative expiry. + */ + public static final long RELATIVE_EXPIRY_NEVER = + Expiration.MAX_RELATIVE_EXPIRY_MS; + /** + * Default max relative expiry for cache pools. + */ + public static final long DEFAULT_MAX_RELATIVE_EXPIRY = + RELATIVE_EXPIRY_NEVER; + + public static final long LIMIT_UNLIMITED = Long.MAX_VALUE; + public static final long DEFAULT_LIMIT = LIMIT_UNLIMITED; + final String poolName; @Nullable @@ -56,14 +71,24 @@ public class CachePoolInfo { @Nullable Long limit; + @Nullable + Long maxRelativeExpiryMs; + public CachePoolInfo(String poolName) { this.poolName = poolName; } - + + /** + * @return Name of the pool. + */ public String getPoolName() { return poolName; } + /** + * @return The owner of the pool. Along with the group and mode, determines + * who has access to view and modify the pool. + */ public String getOwnerName() { return ownerName; } @@ -73,6 +98,10 @@ public class CachePoolInfo { return this; } + /** + * @return The group of the pool. Along with the owner and mode, determines + * who has access to view and modify the pool. + */ public String getGroupName() { return groupName; } @@ -81,7 +110,11 @@ public class CachePoolInfo { this.groupName = groupName; return this; } - + + /** + * @return Unix-style permissions of the pool. Along with the owner and group, + * determines who has access to view and modify the pool. + */ public FsPermission getMode() { return mode; } @@ -91,6 +124,10 @@ public class CachePoolInfo { return this; } + /** + * @return The maximum aggregate number of bytes that can be cached by + * directives in this pool. + */ public Long getLimit() { return limit; } @@ -100,6 +137,26 @@ public class CachePoolInfo { return this; } + /** + * @return The maximum relative expiration of directives of this pool in + * milliseconds + */ + public Long getMaxRelativeExpiryMs() { + return maxRelativeExpiryMs; + } + + /** + * Set the maximum relative expiration of directives of this pool in + * milliseconds. + * + * @param ms in milliseconds + * @return This builder, for call chaining. + */ + public CachePoolInfo setMaxRelativeExpiryMs(Long ms) { + this.maxRelativeExpiryMs = ms; + return this; + } + public String toString() { return new StringBuilder().append("{"). append("poolName:").append(poolName). @@ -108,6 +165,7 @@ public class CachePoolInfo { append(", mode:").append((mode == null) ? "null" : String.format("0%03o", mode.toShort())). append(", limit:").append(limit). + append(", maxRelativeExpiryMs:").append(maxRelativeExpiryMs). append("}").toString(); } @@ -125,6 +183,7 @@ public class CachePoolInfo { append(groupName, other.groupName). append(mode, other.mode). append(limit, other.limit). + append(maxRelativeExpiryMs, other.maxRelativeExpiryMs). isEquals(); } @@ -136,6 +195,7 @@ public class CachePoolInfo { append(groupName). append(mode). append(limit). + append(maxRelativeExpiryMs). hashCode(); } @@ -146,6 +206,15 @@ public class CachePoolInfo { if ((info.getLimit() != null) && (info.getLimit() < 0)) { throw new InvalidRequestException("Limit is negative."); } + if (info.getMaxRelativeExpiryMs() != null) { + long maxRelativeExpiryMs = info.getMaxRelativeExpiryMs(); + if (maxRelativeExpiryMs < 0l) { + throw new InvalidRequestException("Max relative expiry is negative."); + } + if (maxRelativeExpiryMs > Expiration.MAX_RELATIVE_EXPIRY_MS) { + throw new InvalidRequestException("Max relative expiry is too big."); + } + } validateName(info.poolName); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index b7898da1e44..3b4c82b287e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1816,6 +1816,9 @@ public class PBHelper { if (info.getLimit() != null) { builder.setLimit(info.getLimit()); } + if (info.getMaxRelativeExpiryMs() != null) { + builder.setMaxRelativeExpiry(info.getMaxRelativeExpiryMs()); + } return builder.build(); } @@ -1835,6 +1838,9 @@ public class PBHelper { if (proto.hasLimit()) { info.setLimit(proto.getLimit()); } + if (proto.hasMaxRelativeExpiry()) { + info.setMaxRelativeExpiryMs(proto.getMaxRelativeExpiry()); + } return info; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index c3ae8881c3f..5aa440fb6f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -365,7 +365,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping directive id " + directive.getId() - + " because it has expired (" + directive.getExpiryTime() + ">=" + + " because it has expired (" + directive.getExpiryTime() + "<=" + now + ")"); } continue; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index 82bb4e8f6a7..e25913d9cb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.Iterator; import java.util.LinkedList; @@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirective; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; @@ -322,27 +324,48 @@ public final class CacheManager { * {@link CacheDirectiveInfo.Expiration}. This converts a relative Expiration * into an absolute time based on the local clock. * - * @param directive from which to get the expiry time - * @param defaultValue to use if Expiration is not set - * @return Absolute expiry time in milliseconds since Unix epoch - * @throws InvalidRequestException if the Expiration is invalid + * @param info to validate. + * @param maxRelativeExpiryTime of the info's pool. + * @return the expiration time, or the pool's max absolute expiration if the + * info's expiration was not set. + * @throws InvalidRequestException if the info's Expiration is invalid. */ - private static long validateExpiryTime(CacheDirectiveInfo directive, - long defaultValue) throws InvalidRequestException { - long expiryTime; - CacheDirectiveInfo.Expiration expiration = directive.getExpiration(); - if (expiration != null) { - if (expiration.getMillis() < 0) { - throw new InvalidRequestException("Cannot set a negative expiration: " - + expiration.getMillis()); - } - // Converts a relative duration into an absolute time based on the local - // clock - expiryTime = expiration.getAbsoluteMillis(); - } else { - expiryTime = defaultValue; + private static long validateExpiryTime(CacheDirectiveInfo info, + long maxRelativeExpiryTime) throws InvalidRequestException { + if (LOG.isTraceEnabled()) { + LOG.trace("Validating directive " + info + + " pool maxRelativeExpiryTime " + maxRelativeExpiryTime); } - return expiryTime; + final long now = new Date().getTime(); + final long maxAbsoluteExpiryTime = now + maxRelativeExpiryTime; + if (info == null || info.getExpiration() == null) { + return maxAbsoluteExpiryTime; + } + Expiration expiry = info.getExpiration(); + if (expiry.getMillis() < 0l) { + throw new InvalidRequestException("Cannot set a negative expiration: " + + expiry.getMillis()); + } + long relExpiryTime, absExpiryTime; + if (expiry.isRelative()) { + relExpiryTime = expiry.getMillis(); + absExpiryTime = now + relExpiryTime; + } else { + absExpiryTime = expiry.getMillis(); + relExpiryTime = absExpiryTime - now; + } + // Need to cap the expiry so we don't overflow a long when doing math + if (relExpiryTime > Expiration.MAX_RELATIVE_EXPIRY_MS) { + throw new InvalidRequestException("Expiration " + + expiry.toString() + " is too far in the future!"); + } + // Fail if the requested expiry is greater than the max + if (relExpiryTime > maxRelativeExpiryTime) { + throw new InvalidRequestException("Expiration " + expiry.toString() + + " exceeds the max relative expiration time of " + + maxRelativeExpiryTime + " ms."); + } + return absExpiryTime; } /** @@ -357,6 +380,9 @@ public final class CacheManager { private void checkLimit(CachePool pool, String path, short replication) throws InvalidRequestException { CacheDirectiveStats stats = computeNeeded(path, replication); + if (pool.getLimit() == CachePoolInfo.LIMIT_UNLIMITED) { + return; + } if (pool.getBytesNeeded() + (stats.getBytesNeeded() * replication) > pool .getLimit()) { throw new InvalidRequestException("Caching path " + path + " of size " @@ -461,17 +487,13 @@ public final class CacheManager { } /** - * To be called only from the edit log loading code + * Adds a directive, skipping most error checking. This should only be called + * internally in special scenarios like edit log replay. */ CacheDirectiveInfo addDirectiveFromEditLog(CacheDirectiveInfo directive) throws InvalidRequestException { long id = directive.getId(); - CacheDirective entry = - new CacheDirective( - directive.getId(), - directive.getPath().toUri().getPath(), - directive.getReplication(), - directive.getExpiration().getAbsoluteMillis()); + CacheDirective entry = new CacheDirective(directive); CachePool pool = cachePools.get(directive.getPool()); addInternal(entry, pool); if (nextDirectiveId <= id) { @@ -490,8 +512,7 @@ public final class CacheManager { checkWritePermission(pc, pool); String path = validatePath(info); short replication = validateReplication(info, (short)1); - long expiryTime = validateExpiryTime(info, - CacheDirectiveInfo.Expiration.EXPIRY_NEVER); + long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs()); // Do quota validation if required if (!flags.contains(CacheFlag.FORCE)) { // Can't kick and wait if caching is disabled @@ -513,6 +534,56 @@ public final class CacheManager { return directive.toInfo(); } + /** + * Factory method that makes a new CacheDirectiveInfo by applying fields in a + * CacheDirectiveInfo to an existing CacheDirective. + * + * @param info with some or all fields set. + * @param defaults directive providing default values for unset fields in + * info. + * + * @return new CacheDirectiveInfo of the info applied to the defaults. + */ + private static CacheDirectiveInfo createFromInfoAndDefaults( + CacheDirectiveInfo info, CacheDirective defaults) { + // Initialize the builder with the default values + CacheDirectiveInfo.Builder builder = + new CacheDirectiveInfo.Builder(defaults.toInfo()); + // Replace default with new value if present + if (info.getPath() != null) { + builder.setPath(info.getPath()); + } + if (info.getReplication() != null) { + builder.setReplication(info.getReplication()); + } + if (info.getPool() != null) { + builder.setPool(info.getPool()); + } + if (info.getExpiration() != null) { + builder.setExpiration(info.getExpiration()); + } + return builder.build(); + } + + /** + * Modifies a directive, skipping most error checking. This is for careful + * internal use only. modifyDirective can be non-deterministic since its error + * checking depends on current system time, which poses a problem for edit log + * replay. + */ + void modifyDirectiveFromEditLog(CacheDirectiveInfo info) + throws InvalidRequestException { + // Check for invalid IDs. + Long id = info.getId(); + if (id == null) { + throw new InvalidRequestException("Must supply an ID."); + } + CacheDirective prevEntry = getById(id); + CacheDirectiveInfo newInfo = createFromInfoAndDefaults(info, prevEntry); + removeInternal(prevEntry); + addInternal(new CacheDirective(newInfo), getCachePool(newInfo.getPool())); + } + public void modifyDirective(CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet flags) throws IOException { assert namesystem.hasWriteLock(); @@ -527,33 +598,38 @@ public final class CacheManager { } CacheDirective prevEntry = getById(id); checkWritePermission(pc, prevEntry.getPool()); - String path = prevEntry.getPath(); - if (info.getPath() != null) { - path = validatePath(info); - } - short replication = prevEntry.getReplication(); - replication = validateReplication(info, replication); + // Fill in defaults + CacheDirectiveInfo infoWithDefaults = + createFromInfoAndDefaults(info, prevEntry); + CacheDirectiveInfo.Builder builder = + new CacheDirectiveInfo.Builder(infoWithDefaults); - long expiryTime = prevEntry.getExpiryTime(); - expiryTime = validateExpiryTime(info, expiryTime); - - CachePool pool = prevEntry.getPool(); - if (info.getPool() != null) { - pool = getCachePool(validatePoolName(info)); - checkWritePermission(pc, pool); + // Do validation + validatePath(infoWithDefaults); + validateReplication(infoWithDefaults, (short)-1); + // Need to test the pool being set here to avoid rejecting a modify for a + // directive that's already been forced into a pool + CachePool srcPool = prevEntry.getPool(); + CachePool destPool = getCachePool(validatePoolName(infoWithDefaults)); + if (!srcPool.getPoolName().equals(destPool.getPoolName())) { + checkWritePermission(pc, destPool); if (!flags.contains(CacheFlag.FORCE)) { - // Can't kick and wait if caching is disabled - if (monitor != null) { - monitor.waitForRescan(); - } - checkLimit(pool, path, replication); + checkLimit(destPool, infoWithDefaults.getPath().toUri().getPath(), + infoWithDefaults.getReplication()); } } + // Verify the expiration against the destination pool + validateExpiryTime(infoWithDefaults, destPool.getMaxRelativeExpiryMs()); + + // Indicate changes to the CRM + if (monitor != null) { + monitor.setNeedsRescan(); + } + + // Validation passed removeInternal(prevEntry); - CacheDirective newEntry = - new CacheDirective(id, path, replication, expiryTime); - addInternal(newEntry, pool); + addInternal(new CacheDirective(builder.build()), destPool); } catch (IOException e) { LOG.warn("modifyDirective of " + idString + " failed: ", e); throw e; @@ -562,7 +638,7 @@ public final class CacheManager { info+ "."); } - public void removeInternal(CacheDirective directive) + private void removeInternal(CacheDirective directive) throws InvalidRequestException { assert namesystem.hasWriteLock(); // Remove the corresponding entry in directivesByPath. @@ -734,6 +810,13 @@ public final class CacheManager { monitor.setNeedsRescan(); } } + if (info.getMaxRelativeExpiryMs() != null) { + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); + pool.setMaxRelativeExpiryMs(maxRelativeExpiry); + bld.append(prefix).append("set maxRelativeExpiry to " + + maxRelativeExpiry); + prefix = "; "; + } if (prefix.isEmpty()) { bld.append("no changes."); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java index 3da7437acc9..3a96a05aca8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java @@ -49,8 +49,6 @@ import com.google.common.base.Preconditions; public final class CachePool { public static final Log LOG = LogFactory.getLog(CachePool.class); - public static final long DEFAULT_LIMIT = Long.MAX_VALUE; - @Nonnull private final String poolName; @@ -76,6 +74,12 @@ public final class CachePool { */ private long limit; + /** + * Maximum duration that a CacheDirective in this pool remains valid, + * in milliseconds. + */ + private long maxRelativeExpiryMs; + private long bytesNeeded; private long bytesCached; private long filesNeeded; @@ -122,9 +126,12 @@ public final class CachePool { FsPermission mode = (info.getMode() == null) ? FsPermission.getCachePoolDefault() : info.getMode(); long limit = info.getLimit() == null ? - DEFAULT_LIMIT : info.getLimit(); + CachePoolInfo.DEFAULT_LIMIT : info.getLimit(); + long maxRelativeExpiry = info.getMaxRelativeExpiryMs() == null ? + CachePoolInfo.DEFAULT_MAX_RELATIVE_EXPIRY : + info.getMaxRelativeExpiryMs(); return new CachePool(info.getPoolName(), - ownerName, groupName, mode, limit); + ownerName, groupName, mode, limit, maxRelativeExpiry); } /** @@ -134,11 +141,11 @@ public final class CachePool { static CachePool createFromInfo(CachePoolInfo info) { return new CachePool(info.getPoolName(), info.getOwnerName(), info.getGroupName(), - info.getMode(), info.getLimit()); + info.getMode(), info.getLimit(), info.getMaxRelativeExpiryMs()); } CachePool(String poolName, String ownerName, String groupName, - FsPermission mode, long limit) { + FsPermission mode, long limit, long maxRelativeExpiry) { Preconditions.checkNotNull(poolName); Preconditions.checkNotNull(ownerName); Preconditions.checkNotNull(groupName); @@ -148,6 +155,7 @@ public final class CachePool { this.groupName = groupName; this.mode = new FsPermission(mode); this.limit = limit; + this.maxRelativeExpiryMs = maxRelativeExpiry; } public String getPoolName() { @@ -190,6 +198,15 @@ public final class CachePool { return this; } + public long getMaxRelativeExpiryMs() { + return maxRelativeExpiryMs; + } + + public CachePool setMaxRelativeExpiryMs(long expiry) { + this.maxRelativeExpiryMs = expiry; + return this; + } + /** * Get either full or partial information about this CachePool. * @@ -207,7 +224,8 @@ public final class CachePool { return info.setOwnerName(ownerName). setGroupName(groupName). setMode(new FsPermission(mode)). - setLimit(limit); + setLimit(limit). + setMaxRelativeExpiryMs(maxRelativeExpiryMs); } /** @@ -300,6 +318,7 @@ public final class CachePool { append(", groupName:").append(groupName). append(", mode:").append(mode). append(", limit:").append(limit). + append(", maxRelativeExpiryMs:").append(maxRelativeExpiryMs). append(" }").toString(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index be328f71772..c0d1032d605 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -651,8 +651,8 @@ public class FSEditLogLoader { case OP_MODIFY_CACHE_DIRECTIVE: { ModifyCacheDirectiveInfoOp modifyOp = (ModifyCacheDirectiveInfoOp) op; - fsNamesys.getCacheManager().modifyDirective( - modifyOp.directive, null, EnumSet.of(CacheFlag.FORCE)); + fsNamesys.getCacheManager().modifyDirectiveFromEditLog( + modifyOp.directive); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index 9d3fbcb6f7e..ee316c7d40d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -587,18 +587,22 @@ public class FSImageSerialization { final String groupName = info.getGroupName(); final Long limit = info.getLimit(); final FsPermission mode = info.getMode(); + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); - boolean hasOwner, hasGroup, hasMode, hasLimit; + boolean hasOwner, hasGroup, hasMode, hasLimit, hasMaxRelativeExpiry; hasOwner = ownerName != null; hasGroup = groupName != null; hasMode = mode != null; hasLimit = limit != null; + hasMaxRelativeExpiry = maxRelativeExpiry != null; int flags = (hasOwner ? 0x1 : 0) | (hasGroup ? 0x2 : 0) | (hasMode ? 0x4 : 0) | - (hasLimit ? 0x8 : 0); + (hasLimit ? 0x8 : 0) | + (hasMaxRelativeExpiry ? 0x10 : 0); + writeInt(flags, out); if (hasOwner) { @@ -613,6 +617,9 @@ public class FSImageSerialization { if (hasLimit) { writeLong(limit, out); } + if (hasMaxRelativeExpiry) { + writeLong(maxRelativeExpiry, out); + } } public static CachePoolInfo readCachePoolInfo(DataInput in) @@ -632,7 +639,10 @@ public class FSImageSerialization { if ((flags & 0x8) != 0) { info.setLimit(readLong(in)); } - if ((flags & ~0xF) != 0) { + if ((flags & 0x10) != 0) { + info.setMaxRelativeExpiryMs(readLong(in)); + } + if ((flags & ~0x1F) != 0) { throw new IOException("Unknown flag in CachePoolInfo: " + flags); } return info; @@ -646,6 +656,7 @@ public class FSImageSerialization { final String groupName = info.getGroupName(); final Long limit = info.getLimit(); final FsPermission mode = info.getMode(); + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); if (ownerName != null) { XMLUtils.addSaxString(contentHandler, "OWNERNAME", ownerName); @@ -660,6 +671,10 @@ public class FSImageSerialization { XMLUtils.addSaxString(contentHandler, "LIMIT", Long.toString(limit)); } + if (maxRelativeExpiry != null) { + XMLUtils.addSaxString(contentHandler, "MAXRELATIVEEXPIRY", + Long.toString(maxRelativeExpiry)); + } } public static CachePoolInfo readCachePoolInfo(Stanza st) @@ -678,6 +693,10 @@ public class FSImageSerialization { if (st.hasChildren("LIMIT")) { info.setLimit(Long.parseLong(st.getValue("LIMIT"))); } + if (st.hasChildren("MAXRELATIVEEXPIRY")) { + info.setMaxRelativeExpiryMs( + Long.parseLong(st.getValue("MAXRELATIVEEXPIRY"))); + } return info; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index 0e34db3c0bd..2766b382580 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -35,14 +35,12 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CachePoolStats; -import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.tools.TableListing.Justification; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; @@ -120,6 +118,23 @@ public class CacheAdmin extends Configured implements Tool { return listing; } + /** + * Parses a time-to-live value from a string + * @return The ttl in milliseconds + * @throws IOException if it could not be parsed + */ + private static Long parseTtlString(String maxTtlString) throws IOException { + Long maxTtl = null; + if (maxTtlString != null) { + if (maxTtlString.equalsIgnoreCase("never")) { + maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER; + } else { + maxTtl = DFSUtil.parseRelativeTime(maxTtlString); + } + } + return maxTtl; + } + interface Command { String getName(); String getShortUsage(); @@ -154,7 +169,7 @@ public class CacheAdmin extends Configured implements Tool { listing.addRow("", "The cache replication factor to use. " + "Defaults to 1."); listing.addRow("", "How long the directive is " + - "valid. Can be specified in minutes, hours, and days via e.g. " + + "valid. Can be specified in minutes, hours, and days, e.g. " + "30m, 4h, 2d. Valid units are [smhd]." + " If unspecified, the directive never expires."); return getShortUsage() + "\n" + @@ -309,7 +324,7 @@ public class CacheAdmin extends Configured implements Tool { "added. You must have write permission on the cache pool " + "in order to move a directive into it. (optional)"); listing.addRow("", "How long the directive is " + - "valid. Can be specified in minutes, hours, and days via e.g. " + + "valid. Can be specified in minutes, hours, and days, e.g. " + "30m, 4h, 2d. Valid units are [smhd]." + " If unspecified, the directive never expires."); return getShortUsage() + "\n" + @@ -419,22 +434,27 @@ public class CacheAdmin extends Configured implements Tool { System.err.println("Usage is " + getShortUsage()); return 1; } - DistributedFileSystem dfs = getDFS(conf); - RemoteIterator iter = - dfs.listCacheDirectives( - new CacheDirectiveInfo.Builder(). - setPath(new Path(path)).build()); int exitCode = 0; - while (iter.hasNext()) { - CacheDirectiveEntry entry = iter.next(); - try { - dfs.removeCacheDirective(entry.getInfo().getId()); - System.out.println("Removed cache directive " + - entry.getInfo().getId()); - } catch (IOException e) { - System.err.println(prettifyException(e)); - exitCode = 2; + try { + DistributedFileSystem dfs = getDFS(conf); + RemoteIterator iter = + dfs.listCacheDirectives( + new CacheDirectiveInfo.Builder(). + setPath(new Path(path)).build()); + while (iter.hasNext()) { + CacheDirectiveEntry entry = iter.next(); + try { + dfs.removeCacheDirective(entry.getInfo().getId()); + System.out.println("Removed cache directive " + + entry.getInfo().getId()); + } catch (IOException e) { + System.err.println(prettifyException(e)); + exitCode = 2; + } } + } catch (IOException e) { + System.err.println(prettifyException(e)); + exitCode = 2; } if (exitCode == 0) { System.out.println("Removed every cache directive with path " + @@ -500,41 +520,46 @@ public class CacheAdmin extends Configured implements Tool { addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); - - DistributedFileSystem dfs = getDFS(conf); - RemoteIterator iter = - dfs.listCacheDirectives(builder.build()); - int numEntries = 0; - while (iter.hasNext()) { - CacheDirectiveEntry entry = iter.next(); - CacheDirectiveInfo directive = entry.getInfo(); - CacheDirectiveStats stats = entry.getStats(); - List row = new LinkedList(); - row.add("" + directive.getId()); - row.add(directive.getPool()); - row.add("" + directive.getReplication()); - String expiry; - if (directive.getExpiration().getMillis() == - CacheDirectiveInfo.Expiration.EXPIRY_NEVER) { - expiry = "never"; - } else { - expiry = directive.getExpiration().toString(); + try { + DistributedFileSystem dfs = getDFS(conf); + RemoteIterator iter = + dfs.listCacheDirectives(builder.build()); + int numEntries = 0; + while (iter.hasNext()) { + CacheDirectiveEntry entry = iter.next(); + CacheDirectiveInfo directive = entry.getInfo(); + CacheDirectiveStats stats = entry.getStats(); + List row = new LinkedList(); + row.add("" + directive.getId()); + row.add(directive.getPool()); + row.add("" + directive.getReplication()); + String expiry; + // This is effectively never, round for nice printing + if (directive.getExpiration().getMillis() > + Expiration.MAX_RELATIVE_EXPIRY_MS / 2) { + expiry = "never"; + } else { + expiry = directive.getExpiration().toString(); + } + row.add(expiry); + row.add(directive.getPath().toUri().getPath()); + if (printStats) { + row.add("" + stats.getBytesNeeded()); + row.add("" + stats.getBytesCached()); + row.add("" + stats.getFilesNeeded()); + row.add("" + stats.getFilesCached()); + } + tableListing.addRow(row.toArray(new String[0])); + numEntries++; } - row.add(expiry); - row.add(directive.getPath().toUri().getPath()); - if (printStats) { - row.add("" + stats.getBytesNeeded()); - row.add("" + stats.getBytesCached()); - row.add("" + stats.getFilesNeeded()); - row.add("" + stats.getFilesCached()); + System.out.print(String.format("Found %d entr%s\n", + numEntries, numEntries == 1 ? "y" : "ies")); + if (numEntries > 0) { + System.out.print(tableListing); } - tableListing.addRow(row.toArray(new String[0])); - numEntries++; - } - System.out.print(String.format("Found %d entr%s\n", - numEntries, numEntries == 1 ? "y" : "ies")); - if (numEntries > 0) { - System.out.print(tableListing); + } catch (IOException e) { + System.err.println(prettifyException(e)); + return 2; } return 0; } @@ -552,7 +577,8 @@ public class CacheAdmin extends Configured implements Tool { @Override public String getShortUsage() { return "[" + NAME + " [-owner ] " + - "[-group ] [-mode ] [-limit ]]\n"; + "[-group ] [-mode ] [-limit ] " + + "[-maxttl ]\n"; } @Override @@ -571,7 +597,11 @@ public class CacheAdmin extends Configured implements Tool { listing.addRow("", "The maximum number of bytes that can be " + "cached by directives in this pool, in aggregate. By default, " + "no limit is set."); - + listing.addRow("", "The maximum allowed time-to-live for " + + "directives being added to the pool. This can be specified in " + + "seconds, minutes, hours, and days, e.g. 120s, 30m, 4h, 2d. " + + "Valid units are [smhd]. By default, no maximum is set. " + + "This can also be manually specified by \"never\"."); return getShortUsage() + "\n" + "Add a new cache pool.\n\n" + listing.toString(); @@ -605,6 +635,18 @@ public class CacheAdmin extends Configured implements Tool { long limit = Long.parseLong(limitString); info.setLimit(limit); } + String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); + try { + Long maxTtl = parseTtlString(maxTtlString); + if (maxTtl != null) { + info.setMaxRelativeExpiryMs(maxTtl); + } + } catch (IOException e) { + System.err.println( + "Error while parsing maxTtl value: " + e.getMessage()); + return 1; + } + if (!args.isEmpty()) { System.err.print("Can't understand arguments: " + Joiner.on(" ").join(args) + "\n"); @@ -615,7 +657,8 @@ public class CacheAdmin extends Configured implements Tool { try { dfs.addCachePool(info); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.println("Successfully added cache pool " + name + "."); return 0; @@ -632,7 +675,8 @@ public class CacheAdmin extends Configured implements Tool { @Override public String getShortUsage() { return "[" + getName() + " [-owner ] " + - "[-group ] [-mode ] [-limit ]]\n"; + "[-group ] [-mode ] [-limit ] " + + "[-maxTtl ]]\n"; } @Override @@ -645,6 +689,8 @@ public class CacheAdmin extends Configured implements Tool { listing.addRow("", "Unix-style permissions of the pool in octal."); listing.addRow("", "Maximum number of bytes that can be cached " + "by this pool."); + listing.addRow("", "The maximum allowed time-to-live for " + + "directives being added to the pool."); return getShortUsage() + "\n" + WordUtils.wrap("Modifies the metadata of an existing cache pool. " + @@ -663,6 +709,15 @@ public class CacheAdmin extends Configured implements Tool { String limitString = StringUtils.popOptionWithArgument("-limit", args); Long limit = (limitString == null) ? null : Long.parseLong(limitString); + String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); + Long maxTtl = null; + try { + maxTtl = parseTtlString(maxTtlString); + } catch (IOException e) { + System.err.println( + "Error while parsing maxTtl value: " + e.getMessage()); + return 1; + } String name = StringUtils.popFirstNonOption(args); if (name == null) { System.err.println("You must specify a name when creating a " + @@ -693,6 +748,10 @@ public class CacheAdmin extends Configured implements Tool { info.setLimit(limit); changed = true; } + if (maxTtl != null) { + info.setMaxRelativeExpiryMs(maxTtl); + changed = true; + } if (!changed) { System.err.println("You must specify at least one attribute to " + "change in the cache pool."); @@ -702,7 +761,8 @@ public class CacheAdmin extends Configured implements Tool { try { dfs.modifyCachePool(info); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.print("Successfully modified cache pool " + name); String prefix = " to have "; @@ -722,6 +782,9 @@ public class CacheAdmin extends Configured implements Tool { System.out.print(prefix + "limit " + limit); prefix = " and "; } + if (maxTtl != null) { + System.out.print(prefix + "max time-to-live " + maxTtlString); + } System.out.print("\n"); return 0; } @@ -765,7 +828,8 @@ public class CacheAdmin extends Configured implements Tool { try { dfs.removeCachePool(name); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.println("Successfully removed cache pool " + name + "."); return 0; @@ -813,7 +877,8 @@ public class CacheAdmin extends Configured implements Tool { addField("OWNER", Justification.LEFT). addField("GROUP", Justification.LEFT). addField("MODE", Justification.LEFT). - addField("LIMIT", Justification.RIGHT); + addField("LIMIT", Justification.RIGHT). + addField("MAXTTL", Justification.RIGHT); if (printStats) { builder. addField("BYTES_NEEDED", Justification.RIGHT). @@ -837,12 +902,23 @@ public class CacheAdmin extends Configured implements Tool { row.add(info.getMode() != null ? info.getMode().toString() : null); Long limit = info.getLimit(); String limitString; - if (limit != null && limit.equals(CachePool.DEFAULT_LIMIT)) { + if (limit != null && limit.equals(CachePoolInfo.LIMIT_UNLIMITED)) { limitString = "unlimited"; } else { limitString = "" + limit; } row.add(limitString); + Long maxTtl = info.getMaxRelativeExpiryMs(); + String maxTtlString = null; + + if (maxTtl != null) { + if (maxTtl.longValue() == CachePoolInfo.RELATIVE_EXPIRY_NEVER) { + maxTtlString = "never"; + } else { + maxTtlString = DFSUtil.durationToString(maxTtl); + } + } + row.add(maxTtlString); if (printStats) { CachePoolStats stats = entry.getStats(); row.add(Long.toString(stats.getBytesNeeded())); @@ -859,7 +935,8 @@ public class CacheAdmin extends Configured implements Tool { } } } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.print(String.format("Found %d result%s.\n", numResults, (numResults == 1 ? "" : "s"))); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index ee1d10415b9..c7a6465f11f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -434,6 +434,7 @@ message CachePoolInfoProto { optional string groupName = 3; optional int32 mode = 4; optional int64 limit = 5; + optional int64 maxRelativeExpiry = 6; } message CachePoolStatsProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 90daa623bf7..3a0134ed393 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -62,7 +62,6 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.Assume; import org.junit.Before; @@ -730,16 +729,15 @@ public class TestDFSUtil { @Test(timeout=1000) public void testDurationToString() throws Exception { - assertEquals("000:00:00:00", DFSUtil.durationToString(0)); - try { - DFSUtil.durationToString(-199); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains("Invalid negative duration", e); - } - assertEquals("001:01:01:01", + assertEquals("000:00:00:00.000", DFSUtil.durationToString(0)); + assertEquals("001:01:01:01.000", DFSUtil.durationToString(((24*60*60)+(60*60)+(60)+1)*1000)); - assertEquals("000:23:59:59", - DFSUtil.durationToString(((23*60*60)+(59*60)+(59))*1000)); + assertEquals("000:23:59:59.999", + DFSUtil.durationToString(((23*60*60)+(59*60)+(59))*1000+999)); + assertEquals("-001:01:01:01.000", + DFSUtil.durationToString(-((24*60*60)+(60*60)+(60)+1)*1000)); + assertEquals("-000:23:59:59.574", + DFSUtil.durationToString(-(((23*60*60)+(59*60)+(59))*1000+574))); } @Test(timeout=5000) @@ -763,7 +761,7 @@ public class TestDFSUtil { assertEquals(61*60*1000, DFSUtil.parseRelativeTime("61m")); assertEquals(0, DFSUtil.parseRelativeTime("0s")); assertEquals(25*60*60*1000, DFSUtil.parseRelativeTime("25h")); - assertEquals(4*24*60*60*1000, DFSUtil.parseRelativeTime("4d")); - assertEquals(999*24*60*60*1000, DFSUtil.parseRelativeTime("999d")); + assertEquals(4*24*60*60*1000l, DFSUtil.parseRelativeTime("4d")); + assertEquals(999*24*60*60*1000l, DFSUtil.parseRelativeTime("999d")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 03bfc30792e..916e1fa9829 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -23,6 +23,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMOR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS; +import static org.apache.hadoop.hdfs.protocol.CachePoolInfo.RELATIVE_EXPIRY_NEVER; +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -137,6 +139,8 @@ public class TestCacheDirectives { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); LogManager.getLogger(CacheReplicationMonitor.class.getName()).setLevel( Level.TRACE); + LogManager.getLogger(CacheManager.class.getName()).setLevel( + Level.TRACE); } @After @@ -1189,4 +1193,185 @@ public class TestCacheDirectives { new CacheDirectiveInfo.Builder().setPool(inadequate.getPoolName()) .setPath(path1).build(), EnumSet.of(CacheFlag.FORCE)); } + + @Test(timeout=30000) + public void testMaxRelativeExpiry() throws Exception { + // Test that negative and really big max expirations can't be set during add + try { + dfs.addCachePool(new CachePoolInfo("failpool").setMaxRelativeExpiryMs(-1l)); + fail("Added a pool with a negative max expiry."); + } catch (InvalidRequestException e) { + GenericTestUtils.assertExceptionContains("negative", e); + } + try { + dfs.addCachePool(new CachePoolInfo("failpool") + .setMaxRelativeExpiryMs(Long.MAX_VALUE - 1)); + fail("Added a pool with too big of a max expiry."); + } catch (InvalidRequestException e) { + GenericTestUtils.assertExceptionContains("too big", e); + } + // Test that setting a max relative expiry on a pool works + CachePoolInfo coolPool = new CachePoolInfo("coolPool"); + final long poolExpiration = 1000 * 60 * 10l; + dfs.addCachePool(coolPool.setMaxRelativeExpiryMs(poolExpiration)); + RemoteIterator poolIt = dfs.listCachePools(); + CachePoolInfo listPool = poolIt.next().getInfo(); + assertFalse("Should only be one pool", poolIt.hasNext()); + assertEquals("Expected max relative expiry to match set value", + poolExpiration, listPool.getMaxRelativeExpiryMs().longValue()); + // Test that negative and really big max expirations can't be modified + try { + dfs.addCachePool(coolPool.setMaxRelativeExpiryMs(-1l)); + fail("Added a pool with a negative max expiry."); + } catch (InvalidRequestException e) { + assertExceptionContains("negative", e); + } + try { + dfs.modifyCachePool(coolPool + .setMaxRelativeExpiryMs(CachePoolInfo.RELATIVE_EXPIRY_NEVER+1)); + fail("Added a pool with too big of a max expiry."); + } catch (InvalidRequestException e) { + assertExceptionContains("too big", e); + } + // Test that adding a directives without an expiration uses the pool's max + CacheDirectiveInfo defaultExpiry = new CacheDirectiveInfo.Builder() + .setPath(new Path("/blah")) + .setPool(coolPool.getPoolName()) + .build(); + dfs.addCacheDirective(defaultExpiry); + RemoteIterator dirIt = + dfs.listCacheDirectives(defaultExpiry); + CacheDirectiveInfo listInfo = dirIt.next().getInfo(); + assertFalse("Should only have one entry in listing", dirIt.hasNext()); + long listExpiration = listInfo.getExpiration().getAbsoluteMillis() + - new Date().getTime(); + assertTrue("Directive expiry should be approximately the pool's max expiry", + Math.abs(listExpiration - poolExpiration) < 10*1000); + // Test that the max is enforced on add for relative and absolute + CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder() + .setPath(new Path("/lolcat")) + .setPool(coolPool.getPoolName()); + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newRelative(poolExpiration+1)) + .build()); + fail("Added a directive that exceeds pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newAbsolute( + new Date().getTime() + poolExpiration + (10*1000))) + .build()); + fail("Added a directive that exceeds pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + // Test that max is enforced on modify for relative and absolute Expirations + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(poolExpiration+1)) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newAbsolute( + new Date().getTime() + poolExpiration + (10*1000))) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + // Test some giant limit values with add + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newRelative( + Long.MAX_VALUE)) + .build()); + fail("Added a directive with a gigantic max value"); + } catch (IllegalArgumentException e) { + assertExceptionContains("is too far in the future", e); + } + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newAbsolute( + Long.MAX_VALUE)) + .build()); + fail("Added a directive with a gigantic max value"); + } catch (InvalidRequestException e) { + assertExceptionContains("is too far in the future", e); + } + // Test some giant limit values with modify + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.NEVER) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newAbsolute( + Long.MAX_VALUE)) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("is too far in the future", e); + } + // Test that the max is enforced on modify correctly when changing pools + CachePoolInfo destPool = new CachePoolInfo("destPool"); + dfs.addCachePool(destPool.setMaxRelativeExpiryMs(poolExpiration / 2)); + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setPool(destPool.getPoolName()) + .build()); + fail("Modified a directive to a pool with a lower max expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setPool(destPool.getPoolName()) + .setExpiration(Expiration.newRelative(poolExpiration / 2)) + .build()); + dirIt = dfs.listCacheDirectives(new CacheDirectiveInfo.Builder() + .setPool(destPool.getPoolName()) + .build()); + listInfo = dirIt.next().getInfo(); + listExpiration = listInfo.getExpiration().getAbsoluteMillis() + - new Date().getTime(); + assertTrue("Unexpected relative expiry " + listExpiration + + " expected approximately " + poolExpiration/2, + Math.abs(poolExpiration/2 - listExpiration) < 10*1000); + // Test that cache pool and directive expiry can be modified back to never + dfs.modifyCachePool(destPool + .setMaxRelativeExpiryMs(CachePoolInfo.RELATIVE_EXPIRY_NEVER)); + poolIt = dfs.listCachePools(); + listPool = poolIt.next().getInfo(); + while (!listPool.getPoolName().equals(destPool.getPoolName())) { + listPool = poolIt.next().getInfo(); + } + assertEquals("Expected max relative expiry to match set value", + CachePoolInfo.RELATIVE_EXPIRY_NEVER, + listPool.getMaxRelativeExpiryMs().longValue()); + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(RELATIVE_EXPIRY_NEVER)) + .build()); + // Test modifying close to the limit + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(RELATIVE_EXPIRY_NEVER - 1)) + .build()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index bd1224d3b3e..03d3bd15381 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -13,8 +13,8 @@ 2 1 - 1387701670577 - 7bb5467995769b59 + 1388171826188 + c7d869c22c8afce1 @@ -24,8 +24,8 @@ 3 2 - 1387701670580 - a5a3a2755e36827b + 1388171826191 + a3c41446507dfca9 @@ -37,17 +37,17 @@ 16386 /file_create_u\0001;F431 1 - 1387010471220 - 1387010471220 + 1387480626844 + 1387480626844 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 7 @@ -59,8 +59,8 @@ 0 /file_create_u\0001;F431 1 - 1387010471276 - 1387010471220 + 1387480626885 + 1387480626844 512 @@ -78,8 +78,8 @@ 0 /file_create_u\0001;F431 /file_moved - 1387010471286 - 508263bb-692e-4439-8738-ff89b8b03923 + 1387480626894 + a90261a0-3759-4480-ba80-e10c9ae331e6 9 @@ -89,8 +89,8 @@ 7 0 /file_moved - 1387010471299 - 508263bb-692e-4439-8738-ff89b8b03923 + 1387480626905 + a90261a0-3759-4480-ba80-e10c9ae331e6 10 @@ -101,7 +101,7 @@ 0 16387 /directory_mkdir - 1387010471312 + 1387480626917 andrew supergroup @@ -136,7 +136,7 @@ 12 /directory_mkdir snapshot1 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 15 @@ -147,7 +147,7 @@ /directory_mkdir snapshot1 snapshot2 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 16 @@ -157,7 +157,7 @@ 14 /directory_mkdir snapshot2 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 17 @@ -169,17 +169,17 @@ 16388 /file_create_u\0001;F431 1 - 1387010471373 - 1387010471373 + 1387480626978 + 1387480626978 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 18 @@ -191,8 +191,8 @@ 0 /file_create_u\0001;F431 1 - 1387010471380 - 1387010471373 + 1387480626985 + 1387480626978 512 @@ -253,9 +253,9 @@ 0 /file_create_u\0001;F431 /file_moved - 1387010471428 + 1387480627035 NONE - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 25 @@ -267,17 +267,17 @@ 16389 /file_concat_target 1 - 1387010471438 - 1387010471438 + 1387480627043 + 1387480627043 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 27 @@ -388,8 +388,8 @@ 0 /file_concat_target 1 - 1387010471540 - 1387010471438 + 1387480627148 + 1387480627043 512 @@ -423,17 +423,17 @@ 16390 /file_concat_0 1 - 1387010471547 - 1387010471547 + 1387480627155 + 1387480627155 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 40 @@ -544,8 +544,8 @@ 0 /file_concat_0 1 - 1387010471588 - 1387010471547 + 1387480627193 + 1387480627155 512 @@ -579,17 +579,17 @@ 16391 /file_concat_1 1 - 1387010471595 - 1387010471595 + 1387480627200 + 1387480627200 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 52 @@ -700,8 +700,8 @@ 0 /file_concat_1 1 - 1387010471651 - 1387010471595 + 1387480627238 + 1387480627200 512 @@ -733,12 +733,12 @@ 56 0 /file_concat_target - 1387010471663 + 1387480627246 /file_concat_0 /file_concat_1 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 63 @@ -750,14 +750,14 @@ 16392 /file_symlink /file_concat_target - 1387010471674 - 1387010471674 + 1387480627255 + 1387480627255 andrew supergroup 511 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 64 @@ -771,11 +771,11 @@ andrew JobTracker - 1387010471682 - 1387615271682 + 1387480627262 + 1388085427262 2 - 1387096871682 + 1387567027262 @@ -788,11 +788,11 @@ andrew JobTracker - 1387010471682 - 1387615271682 + 1387480627262 + 1388085427262 2 - 1387096871717 + 1387567027281 @@ -805,8 +805,8 @@ andrew JobTracker - 1387010471682 - 1387615271682 + 1387480627262 + 1388085427262 2 @@ -820,7 +820,8 @@ andrew 493 9223372036854775807 - 508263bb-692e-4439-8738-ff89b8b03923 + 2305843009213693951 + a90261a0-3759-4480-ba80-e10c9ae331e6 68 @@ -833,7 +834,7 @@ party 448 1989 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 69 @@ -845,8 +846,8 @@ /bar 1 poolparty - -1 - 508263bb-692e-4439-8738-ff89b8b03923 + 2305844396694321272 + a90261a0-3759-4480-ba80-e10c9ae331e6 70 @@ -856,7 +857,7 @@ 64 1 /bar2 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 71 @@ -865,7 +866,7 @@ 65 1 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 72 @@ -874,7 +875,7 @@ 66 poolparty - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 73 @@ -886,17 +887,17 @@ 16393 /hard-lease-recovery-test 1 - 1387010471802 - 1387010471802 + 1387480627356 + 1387480627356 512 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 127.0.0.1 andrew supergroup 420 - 508263bb-692e-4439-8738-ff89b8b03923 + a90261a0-3759-4480-ba80-e10c9ae331e6 74 @@ -953,7 +954,7 @@ OP_REASSIGN_LEASE 73 - DFSClient_NONMAPREDUCE_-52011019_1 + DFSClient_NONMAPREDUCE_1147796111_1 /hard-lease-recovery-test HDFS_NameNode @@ -966,8 +967,8 @@ 0 /hard-lease-recovery-test 1 - 1387010474126 - 1387010471802 + 1387480629729 + 1387480627356 512 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml index 77f8671748f..64de6bf87c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml @@ -417,11 +417,11 @@ SubstringComparator - bar alice alicegroup rwxr-xr-x unlimited 0 0 0 0 0 + bar alice alicegroup rwxr-xr-x unlimited never 0 0 0 0 0 SubstringComparator - foo bob bob rw-rw-r-- unlimited 0 0 0 0 0 + foo bob bob rw-rw-r-- unlimited never 0 0 0 0 0 @@ -457,5 +457,37 @@ + + + Testing pool max ttl settings + + -addPool pool1 -owner andrew -group andrew + -addPool pool2 -owner andrew -group andrew -maxTtl 999d + -modifyPool pool2 -maxTtl never + -addPool pool3 -owner andrew -group andrew -maxTtl 4h + -listPools + + + -removePool pool1 + + + + SubstringComparator + Found 3 results + + + SubstringComparator + pool1 andrew andrew rwxr-xr-x unlimited never + + + SubstringComparator + pool2 andrew andrew rwxr-xr-x unlimited never + + + SubstringComparator + pool3 andrew andrew rwxr-xr-x unlimited 000:04:00:00.000 + + + From 624703ed7b6d80855d3e0567bee5c1b7659bc635 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 21 Dec 2013 16:29:10 +0000 Subject: [PATCH 09/11] YARN-1028. Addendum patch. Added FailoverProxyProvider capability to ResourceManager to help with RM failover. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1552920 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/io/retry/RetryPolicies.java | 15 --------------- .../apache/hadoop/yarn/client/TestRMFailover.java | 2 -- .../org/apache/hadoop/yarn/client/RMProxy.java | 12 +++++------- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index e2770030b11..3d5992716f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -69,13 +69,6 @@ public class RetryPolicies { */ public static final RetryPolicy RETRY_FOREVER = new RetryForever(); - /** - *

- * Keep failing over forever - *

- */ - public static final RetryPolicy FAILOVER_FOREVER = new FailoverForever(); - /** *

* Keep trying a limited number of times, waiting a fixed time between attempts, @@ -173,14 +166,6 @@ public class RetryPolicies { return RetryAction.RETRY; } } - - static class FailoverForever implements RetryPolicy { - @Override - public RetryAction shouldRetry(Exception e, int retries, int failovers, - boolean isIdempotentOrAtMostOnce) throws Exception { - return RetryAction.FAILOVER_AND_RETRY; - } - } /** * Retry up to maxRetries. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index 8545a1a2839..fed26d7f5aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -86,9 +86,7 @@ public class TestRMFailover { setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE); setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE); - conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 100); conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L); - conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS, 1000L); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index 1651c13100c..913eb04613c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -225,19 +225,17 @@ public class RMProxy { int maxFailoverAttempts = conf.getInt( YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1); - RetryPolicy basePolicy = RetryPolicies.TRY_ONCE_THEN_FAIL; if (maxFailoverAttempts == -1) { if (waitForEver) { - basePolicy = RetryPolicies.FAILOVER_FOREVER; + maxFailoverAttempts = Integer.MAX_VALUE; } else { - basePolicy = new FailoverUptoMaximumTimePolicy( - System.currentTimeMillis() + rmConnectWaitMS); + maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs); } - maxFailoverAttempts = 0; } - return RetryPolicies.failoverOnNetworkException(basePolicy, - maxFailoverAttempts, failoverSleepBaseMs, failoverSleepMaxMs); + return RetryPolicies.failoverOnNetworkException( + RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, + failoverSleepBaseMs, failoverSleepMaxMs); } if (waitForEver) { From e8de319789def7ccf2a7e9e14b1011ee825ff37b Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Mon, 23 Dec 2013 18:23:43 +0000 Subject: [PATCH 10/11] HDFS-5690. DataNode fails to start in secure mode when dfs.http.policy equals to HTTP_ONLY. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1553167 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/http/HttpServer.java | 5 +++-- .../java/org/apache/hadoop/http/TestHttpServer.java | 13 +++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/datanode/SecureDataNodeStarter.java | 12 +++++++++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java index 50a8d47260d..fa31331d72f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java @@ -305,12 +305,13 @@ public class HttpServer implements FilterContainer { } } - if (endpoints.size() == 0) { + if (endpoints.size() == 0 && connector == null) { throw new HadoopIllegalArgumentException("No endpoints specified"); } if (hostName == null) { - hostName = endpoints.get(0).getHost(); + hostName = endpoints.size() == 0 ? connector.getHost() : endpoints.get( + 0).getHost(); } if (this.conf == null) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index 13627988b15..6e5ca47824f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -66,6 +66,8 @@ import org.mockito.internal.util.reflection.Whitebox; import org.mortbay.jetty.Connector; import org.mortbay.util.ajax.JSON; +import static org.mockito.Mockito.*; + public class TestHttpServer extends HttpServerFunctionalTest { static final Log LOG = LogFactory.getLog(TestHttpServer.class); private static HttpServer server; @@ -588,4 +590,15 @@ public class TestHttpServer extends HttpServerFunctionalTest { assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date")); } + /** + * HTTPServer.Builder should proceed if a external connector is available. + */ + @Test + public void testHttpServerBuilderWithExternalConnector() throws Exception { + Connector c = mock(Connector.class); + doReturn("localhost").when(c).getHost(); + HttpServer s = new HttpServer.Builder().setName("test").setConnector(c) + .build(); + s.stop(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 34678de50bc..44956404f1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -830,6 +830,9 @@ Release 2.4.0 - UNRELEASED HDFS-5691. Fix typo in ShortCircuitLocalRead document. (Akira Ajisaka via suresh) + HDFS-5690. DataNode fails to start in secure mode when dfs.http.policy equals to + HTTP_ONLY. (Haohui Mai via jing9) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java index 23de29027e8..b6dc7fedbe2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java @@ -87,6 +87,7 @@ public class SecureDataNodeStarter implements Daemon { public static SecureResources getSecureResources(Configuration conf) throws Exception { HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); + boolean isSecure = UserGroupInformation.isSecurityEnabled(); // Obtain secure port for data streaming to datanode InetSocketAddress streamingAddr = DataNode.getStreamingAddr(conf); @@ -106,6 +107,11 @@ public class SecureDataNodeStarter implements Daemon { + ss.getLocalPort()); } + if (ss.getLocalPort() > 1023 && isSecure) { + throw new RuntimeException( + "Cannot start secure datanode with unprivileged RPC ports"); + } + System.err.println("Opened streaming server at " + streamingAddr); // Bind a port for the web server. The code intends to bind HTTP server to @@ -126,9 +132,9 @@ public class SecureDataNodeStarter implements Daemon { System.err.println("Successfully obtained privileged resources (streaming port = " + ss + " ) (http listener port = " + listener.getConnection() +")"); - if ((ss.getLocalPort() > 1023 || listener.getPort() > 1023) && - UserGroupInformation.isSecurityEnabled()) { - throw new RuntimeException("Cannot start secure datanode with unprivileged ports"); + if (listener.getPort() > 1023 && isSecure) { + throw new RuntimeException( + "Cannot start secure datanode with unprivileged HTTP ports"); } System.err.println("Opened info server at " + infoSocAddr); } From 150440c60738fc11f5eeb9feb3701222704e4dac Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Mon, 23 Dec 2013 18:33:19 +0000 Subject: [PATCH 11/11] HADOOP-10175. Har files system authority should preserve userinfo. Contributed by Chuan Liu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1553169 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/fs/HarFileSystem.java | 4 ++++ .../hadoop/fs/TestHarFileSystemBasics.java | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4ffa26b7a84..dcc8f5bde61 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -565,6 +565,9 @@ Release 2.3.0 - UNRELEASED HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe) + HADOOP-10175. Har files system authority should preserve userinfo. + (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 091b35a846a..88208da651a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -294,6 +294,10 @@ public class HarFileSystem extends FileSystem { private String getHarAuth(URI underLyingUri) { String auth = underLyingUri.getScheme() + "-"; if (underLyingUri.getHost() != null) { + if (underLyingUri.getUserInfo() != null) { + auth += underLyingUri.getUserInfo(); + auth += "@"; + } auth += underLyingUri.getHost(); if (underLyingUri.getPort() != -1) { auth += ":"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java index 237d7161f72..577abfd090e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java @@ -258,6 +258,22 @@ public class TestHarFileSystemBasics { 0, expectedFileNames.size()); } + @Test + public void testMakeQualifiedPath() throws Exception { + // Construct a valid har file system path with authority that + // contains userinfo and port. The userinfo and port are useless + // in local fs uri. They are only used to verify har file system + // can correctly preserve the information for the underlying file system. + String harPathWithUserinfo = "har://file-user:passwd@localhost:80" + + harPath.toUri().getPath().toString(); + Path path = new Path(harPathWithUserinfo); + Path qualifiedPath = path.getFileSystem(conf).makeQualified(path); + assertTrue(String.format( + "The qualified path (%s) did not match the expected path (%s).", + qualifiedPath.toString(), harPathWithUserinfo), + qualifiedPath.toString().equals(harPathWithUserinfo)); + } + // ========== Negative: @Test