YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the scheduler page. (Ashwin Shankar and Wei Yan via kasha)
This commit is contained in:
parent
9ad413b19d
commit
270a271f53
|
@ -163,6 +163,9 @@ Release 2.6.0 - UNRELEASED
|
||||||
YARN-2406. Move RM recovery related proto to
|
YARN-2406. Move RM recovery related proto to
|
||||||
yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
|
yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
|
||||||
|
|
||||||
|
YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
|
||||||
|
scheduler page. (Ashwin Shankar and Wei Yan via kasha)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
|
|
@ -44,10 +44,12 @@ public class FairSchedulerPage extends RmView {
|
||||||
static final float Q_MAX_WIDTH = 0.8f;
|
static final float Q_MAX_WIDTH = 0.8f;
|
||||||
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
|
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
|
||||||
static final String Q_END = "left:101%";
|
static final String Q_END = "left:101%";
|
||||||
static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)";
|
static final String Q_GIVEN = "left:0%;background:none;border:1px solid rgba(0,0,0,1)";
|
||||||
|
static final String Q_INSTANTANEOUS_FS = "left:0%;background:none;border:1px dashed rgba(0,0,0,1)";
|
||||||
static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
|
static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
|
||||||
static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
|
static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
|
||||||
|
static final String STEADY_FAIR_SHARE = "Steady Fair Share";
|
||||||
|
static final String INSTANTANEOUS_FAIR_SHARE = "Instantaneous Fair Share";
|
||||||
@RequestScoped
|
@RequestScoped
|
||||||
static class FSQInfo {
|
static class FSQInfo {
|
||||||
FairSchedulerQueueInfo qinfo;
|
FairSchedulerQueueInfo qinfo;
|
||||||
|
@ -73,8 +75,8 @@ public class FairSchedulerPage extends RmView {
|
||||||
if (maxApps < Integer.MAX_VALUE) {
|
if (maxApps < Integer.MAX_VALUE) {
|
||||||
ri._("Max Running Applications:", qinfo.getMaxApplications());
|
ri._("Max Running Applications:", qinfo.getMaxApplications());
|
||||||
}
|
}
|
||||||
ri._("Fair Share:", qinfo.getFairShare().toString());
|
ri._(STEADY_FAIR_SHARE + ":", qinfo.getSteadyFairShare().toString());
|
||||||
|
ri._(INSTANTANEOUS_FAIR_SHARE + ":", qinfo.getFairShare().toString());
|
||||||
html._(InfoBlock.class);
|
html._(InfoBlock.class);
|
||||||
|
|
||||||
// clear the info contents so this queue's info doesn't accumulate into another queue's info
|
// clear the info contents so this queue's info doesn't accumulate into another queue's info
|
||||||
|
@ -95,16 +97,21 @@ public class FairSchedulerPage extends RmView {
|
||||||
UL<Hamlet> ul = html.ul("#pq");
|
UL<Hamlet> ul = html.ul("#pq");
|
||||||
for (FairSchedulerQueueInfo info : subQueues) {
|
for (FairSchedulerQueueInfo info : subQueues) {
|
||||||
float capacity = info.getMaxResourcesFraction();
|
float capacity = info.getMaxResourcesFraction();
|
||||||
float fairShare = info.getFairShareMemoryFraction();
|
float steadyFairShare = info.getSteadyFairShareMemoryFraction();
|
||||||
|
float instantaneousFairShare = info.getFairShareMemoryFraction();
|
||||||
float used = info.getUsedMemoryFraction();
|
float used = info.getUsedMemoryFraction();
|
||||||
LI<UL<Hamlet>> li = ul.
|
LI<UL<Hamlet>> li = ul.
|
||||||
li().
|
li().
|
||||||
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
|
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
|
||||||
$title(join("Fair Share:", percent(fairShare))).
|
$title(join(join(STEADY_FAIR_SHARE + ":", percent(steadyFairShare)),
|
||||||
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
|
join(" " + INSTANTANEOUS_FAIR_SHARE + ":", percent(instantaneousFairShare)))).
|
||||||
|
span().$style(join(Q_GIVEN, ";font-size:1px;", width(steadyFairShare / capacity))).
|
||||||
|
_('.')._().
|
||||||
|
span().$style(join(Q_INSTANTANEOUS_FS, ";font-size:1px;",
|
||||||
|
width(instantaneousFairShare/capacity))).
|
||||||
_('.')._().
|
_('.')._().
|
||||||
span().$style(join(width(used/capacity),
|
span().$style(join(width(used/capacity),
|
||||||
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
|
";font-size:1px;left:0%;", used > instantaneousFairShare ? Q_OVER : Q_UNDER)).
|
||||||
_('.')._().
|
_('.')._().
|
||||||
span(".q", info.getQueueName())._().
|
span(".q", info.getQueueName())._().
|
||||||
span().$class("qstats").$style(left(Q_STATS_POS)).
|
span().$class("qstats").$style(left(Q_STATS_POS)).
|
||||||
|
@ -156,7 +163,13 @@ public class FairSchedulerPage extends RmView {
|
||||||
li().$style("margin-bottom: 1em").
|
li().$style("margin-bottom: 1em").
|
||||||
span().$style("font-weight: bold")._("Legend:")._().
|
span().$style("font-weight: bold")._("Legend:")._().
|
||||||
span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
|
span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
|
||||||
_("Fair Share")._().
|
$title("The steady fair shares consider all queues, " +
|
||||||
|
"both active (with running applications) and inactive.").
|
||||||
|
_(STEADY_FAIR_SHARE)._().
|
||||||
|
span().$class("qlegend ui-corner-all").$style(Q_INSTANTANEOUS_FS).
|
||||||
|
$title("The instantaneous fair shares consider only active " +
|
||||||
|
"queues (with running applications).").
|
||||||
|
_(INSTANTANEOUS_FAIR_SHARE)._().
|
||||||
span().$class("qlegend ui-corner-all").$style(Q_UNDER).
|
span().$class("qlegend ui-corner-all").$style(Q_UNDER).
|
||||||
_("Used")._().
|
_("Used")._().
|
||||||
span().$class("qlegend ui-corner-all").$style(Q_OVER).
|
span().$class("qlegend ui-corner-all").$style(Q_OVER).
|
||||||
|
|
|
@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlRootElement;
|
||||||
import javax.xml.bind.annotation.XmlSeeAlso;
|
import javax.xml.bind.annotation.XmlSeeAlso;
|
||||||
import javax.xml.bind.annotation.XmlTransient;
|
import javax.xml.bind.annotation.XmlTransient;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
||||||
|
@ -44,6 +43,8 @@ public class FairSchedulerQueueInfo {
|
||||||
@XmlTransient
|
@XmlTransient
|
||||||
private float fractionMemUsed;
|
private float fractionMemUsed;
|
||||||
@XmlTransient
|
@XmlTransient
|
||||||
|
private float fractionMemSteadyFairShare;
|
||||||
|
@XmlTransient
|
||||||
private float fractionMemFairShare;
|
private float fractionMemFairShare;
|
||||||
@XmlTransient
|
@XmlTransient
|
||||||
private float fractionMemMinShare;
|
private float fractionMemMinShare;
|
||||||
|
@ -53,6 +54,7 @@ public class FairSchedulerQueueInfo {
|
||||||
private ResourceInfo minResources;
|
private ResourceInfo minResources;
|
||||||
private ResourceInfo maxResources;
|
private ResourceInfo maxResources;
|
||||||
private ResourceInfo usedResources;
|
private ResourceInfo usedResources;
|
||||||
|
private ResourceInfo steadyFairResources;
|
||||||
private ResourceInfo fairResources;
|
private ResourceInfo fairResources;
|
||||||
private ResourceInfo clusterResources;
|
private ResourceInfo clusterResources;
|
||||||
|
|
||||||
|
@ -76,6 +78,7 @@ public class FairSchedulerQueueInfo {
|
||||||
fractionMemUsed = (float)usedResources.getMemory() /
|
fractionMemUsed = (float)usedResources.getMemory() /
|
||||||
clusterResources.getMemory();
|
clusterResources.getMemory();
|
||||||
|
|
||||||
|
steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
|
||||||
fairResources = new ResourceInfo(queue.getFairShare());
|
fairResources = new ResourceInfo(queue.getFairShare());
|
||||||
minResources = new ResourceInfo(queue.getMinShare());
|
minResources = new ResourceInfo(queue.getMinShare());
|
||||||
maxResources = new ResourceInfo(queue.getMaxShare());
|
maxResources = new ResourceInfo(queue.getMaxShare());
|
||||||
|
@ -83,7 +86,10 @@ public class FairSchedulerQueueInfo {
|
||||||
Resources.componentwiseMin(queue.getMaxShare(),
|
Resources.componentwiseMin(queue.getMaxShare(),
|
||||||
scheduler.getClusterResource()));
|
scheduler.getClusterResource()));
|
||||||
|
|
||||||
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
|
fractionMemSteadyFairShare =
|
||||||
|
(float)steadyFairResources.getMemory() / clusterResources.getMemory();
|
||||||
|
fractionMemFairShare = (float) fairResources.getMemory()
|
||||||
|
/ clusterResources.getMemory();
|
||||||
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
|
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
|
||||||
fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
|
fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
|
||||||
|
|
||||||
|
@ -100,6 +106,13 @@ public class FairSchedulerQueueInfo {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the steady fair share as a fraction of the entire cluster capacity.
|
||||||
|
*/
|
||||||
|
public float getSteadyFairShareMemoryFraction() {
|
||||||
|
return fractionMemSteadyFairShare;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the fair share as a fraction of the entire cluster capacity.
|
* Returns the fair share as a fraction of the entire cluster capacity.
|
||||||
*/
|
*/
|
||||||
|
@ -108,7 +121,14 @@ public class FairSchedulerQueueInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the fair share of this queue in megabytes.
|
* Returns the steady fair share of this queue in megabytes.
|
||||||
|
*/
|
||||||
|
public ResourceInfo getSteadyFairShare() {
|
||||||
|
return steadyFairResources;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the fair share of this queue in megabytes
|
||||||
*/
|
*/
|
||||||
public ResourceInfo getFairShare() {
|
public ResourceInfo getFairShare() {
|
||||||
return fairResources;
|
return fairResources;
|
||||||
|
|
|
@ -429,13 +429,19 @@ Monitoring through web UI
|
||||||
|
|
||||||
* Max Resources - The configured maximum resources that are allowed to the queue.
|
* Max Resources - The configured maximum resources that are allowed to the queue.
|
||||||
|
|
||||||
* Fair Share - The queue's fair share of resources. Queues may be allocated
|
* Instantaneous Fair Share - The queue's instantaneous fair share of resources.
|
||||||
resources beyond their fair share when other queues aren't using them. A
|
These shares consider only actives queues (those with running applications),
|
||||||
queue whose resource consumption lies at or below its fair share will never
|
and are used for scheduling decisions. Queues may be allocated resources
|
||||||
|
beyond their shares when other queues aren't using them. A queue whose
|
||||||
|
resource consumption lies at or below its instantaneous fair share will never
|
||||||
have its containers preempted.
|
have its containers preempted.
|
||||||
|
|
||||||
In addition to the information that the ResourceManager normally displays
|
* Steady Fair Share - The queue's steady fair share of resources. These shares
|
||||||
about each application, the web interface includes the application's fair share.
|
consider all the queues irrespective of whether they are active (have
|
||||||
|
running applications) or not. These are computed less frequently and
|
||||||
|
change only when the configuration or capacity changes.They are meant to
|
||||||
|
provide visibility into resources the user can expect, and hence displayed
|
||||||
|
in the Web UI.
|
||||||
|
|
||||||
Moving applications between queues
|
Moving applications between queues
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue