YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the scheduler page. (Ashwin Shankar and Wei Yan via kasha)
(cherry picked from commit 270a271f53)
Conflicts:
hadoop-yarn-project/CHANGES.txt
This commit is contained in:
Karthik Kambatla
parent
c6cf748985
commit
d070597a0e
|
|
@ -129,6 +129,9 @@ Release 2.7.0 - UNRELEASED
|
|||
YARN-2998. Abstract out scheduler independent PlanFollower components.
|
||||
(Anubhav Dhoot via kasha)
|
||||
|
||||
YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
|
||||
scheduler page. (Ashwin Shankar and Wei Yan via kasha)
|
||||
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
|
|
@ -522,6 +525,7 @@ Release 2.6.0 - 2014-11-18
|
|||
YARN-2406. Move RM recovery related proto to
|
||||
yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
|
||||
|
||||
<<<<<<< HEAD
|
||||
YARN-1506. Changed RMNode/SchedulerNode to update resource with event
|
||||
notification. (Junping Du via jianhe)
|
||||
|
||||
|
|
@ -657,6 +661,7 @@ Release 2.6.0 - 2014-11-18
|
|||
YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run
|
||||
with both CS and FS. (Wei Yan and kasha via kasha)
|
||||
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
|||
|
|
@ -44,10 +44,12 @@ public class FairSchedulerPage extends RmView {
|
|||
static final float Q_MAX_WIDTH = 0.8f;
|
||||
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
|
||||
static final String Q_END = "left:101%";
|
||||
static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)";
|
||||
static final String Q_GIVEN = "left:0%;background:none;border:1px solid rgba(0,0,0,1)";
|
||||
static final String Q_INSTANTANEOUS_FS = "left:0%;background:none;border:1px dashed rgba(0,0,0,1)";
|
||||
static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
|
||||
static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
|
||||
|
||||
static final String STEADY_FAIR_SHARE = "Steady Fair Share";
|
||||
static final String INSTANTANEOUS_FAIR_SHARE = "Instantaneous Fair Share";
|
||||
@RequestScoped
|
||||
static class FSQInfo {
|
||||
FairSchedulerQueueInfo qinfo;
|
||||
|
|
@ -73,8 +75,8 @@ public class FairSchedulerPage extends RmView {
|
|||
if (maxApps < Integer.MAX_VALUE) {
|
||||
ri._("Max Running Applications:", qinfo.getMaxApplications());
|
||||
}
|
||||
ri._("Fair Share:", qinfo.getFairShare().toString());
|
||||
|
||||
ri._(STEADY_FAIR_SHARE + ":", qinfo.getSteadyFairShare().toString());
|
||||
ri._(INSTANTANEOUS_FAIR_SHARE + ":", qinfo.getFairShare().toString());
|
||||
html._(InfoBlock.class);
|
||||
|
||||
// clear the info contents so this queue's info doesn't accumulate into another queue's info
|
||||
|
|
@ -95,16 +97,21 @@ public class FairSchedulerPage extends RmView {
|
|||
UL<Hamlet> ul = html.ul("#pq");
|
||||
for (FairSchedulerQueueInfo info : subQueues) {
|
||||
float capacity = info.getMaxResourcesFraction();
|
||||
float fairShare = info.getFairShareMemoryFraction();
|
||||
float steadyFairShare = info.getSteadyFairShareMemoryFraction();
|
||||
float instantaneousFairShare = info.getFairShareMemoryFraction();
|
||||
float used = info.getUsedMemoryFraction();
|
||||
LI<UL<Hamlet>> li = ul.
|
||||
li().
|
||||
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
|
||||
$title(join("Fair Share:", percent(fairShare))).
|
||||
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
|
||||
$title(join(join(STEADY_FAIR_SHARE + ":", percent(steadyFairShare)),
|
||||
join(" " + INSTANTANEOUS_FAIR_SHARE + ":", percent(instantaneousFairShare)))).
|
||||
span().$style(join(Q_GIVEN, ";font-size:1px;", width(steadyFairShare / capacity))).
|
||||
_('.')._().
|
||||
span().$style(join(Q_INSTANTANEOUS_FS, ";font-size:1px;",
|
||||
width(instantaneousFairShare/capacity))).
|
||||
_('.')._().
|
||||
span().$style(join(width(used/capacity),
|
||||
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
|
||||
";font-size:1px;left:0%;", used > instantaneousFairShare ? Q_OVER : Q_UNDER)).
|
||||
_('.')._().
|
||||
span(".q", info.getQueueName())._().
|
||||
span().$class("qstats").$style(left(Q_STATS_POS)).
|
||||
|
|
@ -156,7 +163,13 @@ public class FairSchedulerPage extends RmView {
|
|||
li().$style("margin-bottom: 1em").
|
||||
span().$style("font-weight: bold")._("Legend:")._().
|
||||
span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
|
||||
_("Fair Share")._().
|
||||
$title("The steady fair shares consider all queues, " +
|
||||
"both active (with running applications) and inactive.").
|
||||
_(STEADY_FAIR_SHARE)._().
|
||||
span().$class("qlegend ui-corner-all").$style(Q_INSTANTANEOUS_FS).
|
||||
$title("The instantaneous fair shares consider only active " +
|
||||
"queues (with running applications).").
|
||||
_(INSTANTANEOUS_FAIR_SHARE)._().
|
||||
span().$class("qlegend ui-corner-all").$style(Q_UNDER).
|
||||
_("Used")._().
|
||||
span().$class("qlegend ui-corner-all").$style(Q_OVER).
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlRootElement;
|
|||
import javax.xml.bind.annotation.XmlSeeAlso;
|
||||
import javax.xml.bind.annotation.XmlTransient;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
||||
|
|
@ -44,6 +43,8 @@ public class FairSchedulerQueueInfo {
|
|||
@XmlTransient
|
||||
private float fractionMemUsed;
|
||||
@XmlTransient
|
||||
private float fractionMemSteadyFairShare;
|
||||
@XmlTransient
|
||||
private float fractionMemFairShare;
|
||||
@XmlTransient
|
||||
private float fractionMemMinShare;
|
||||
|
|
@ -53,6 +54,7 @@ public class FairSchedulerQueueInfo {
|
|||
private ResourceInfo minResources;
|
||||
private ResourceInfo maxResources;
|
||||
private ResourceInfo usedResources;
|
||||
private ResourceInfo steadyFairResources;
|
||||
private ResourceInfo fairResources;
|
||||
private ResourceInfo clusterResources;
|
||||
|
||||
|
|
@ -75,15 +77,19 @@ public class FairSchedulerQueueInfo {
|
|||
usedResources = new ResourceInfo(queue.getResourceUsage());
|
||||
fractionMemUsed = (float)usedResources.getMemory() /
|
||||
clusterResources.getMemory();
|
||||
|
||||
|
||||
steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
|
||||
fairResources = new ResourceInfo(queue.getFairShare());
|
||||
minResources = new ResourceInfo(queue.getMinShare());
|
||||
maxResources = new ResourceInfo(queue.getMaxShare());
|
||||
maxResources = new ResourceInfo(
|
||||
Resources.componentwiseMin(queue.getMaxShare(),
|
||||
scheduler.getClusterResource()));
|
||||
|
||||
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
|
||||
|
||||
fractionMemSteadyFairShare =
|
||||
(float)steadyFairResources.getMemory() / clusterResources.getMemory();
|
||||
fractionMemFairShare = (float) fairResources.getMemory()
|
||||
/ clusterResources.getMemory();
|
||||
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
|
||||
fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
|
||||
|
||||
|
|
@ -105,20 +111,34 @@ public class FairSchedulerQueueInfo {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the steady fair share as a fraction of the entire cluster capacity.
|
||||
*/
|
||||
public float getSteadyFairShareMemoryFraction() {
|
||||
return fractionMemSteadyFairShare;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the fair share as a fraction of the entire cluster capacity.
|
||||
*/
|
||||
public float getFairShareMemoryFraction() {
|
||||
return fractionMemFairShare;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the fair share of this queue in megabytes.
|
||||
* Returns the steady fair share of this queue in megabytes.
|
||||
*/
|
||||
public ResourceInfo getSteadyFairShare() {
|
||||
return steadyFairResources;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the fair share of this queue in megabytes
|
||||
*/
|
||||
public ResourceInfo getFairShare() {
|
||||
return fairResources;
|
||||
}
|
||||
|
||||
|
||||
public ResourceInfo getMinResources() {
|
||||
return minResources;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -453,13 +453,19 @@ Monitoring through web UI
|
|||
|
||||
* Max Resources - The configured maximum resources that are allowed to the queue.
|
||||
|
||||
* Fair Share - The queue's fair share of resources. Queues may be allocated
|
||||
resources beyond their fair share when other queues aren't using them. A
|
||||
queue whose resource consumption lies at or below its fair share will never
|
||||
have its containers preempted.
|
||||
* Instantaneous Fair Share - The queue's instantaneous fair share of resources.
|
||||
These shares consider only actives queues (those with running applications),
|
||||
and are used for scheduling decisions. Queues may be allocated resources
|
||||
beyond their shares when other queues aren't using them. A queue whose
|
||||
resource consumption lies at or below its instantaneous fair share will never
|
||||
have its containers preempted.
|
||||
|
||||
In addition to the information that the ResourceManager normally displays
|
||||
about each application, the web interface includes the application's fair share.
|
||||
* Steady Fair Share - The queue's steady fair share of resources. These shares
|
||||
consider all the queues irrespective of whether they are active (have
|
||||
running applications) or not. These are computed less frequently and
|
||||
change only when the configuration or capacity changes.They are meant to
|
||||
provide visibility into resources the user can expect, and hence displayed
|
||||
in the Web UI.
|
||||
|
||||
Moving applications between queues
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue