From 21a07769d425e9052d4a73718105eb4658b7c2f3 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 8 May 2015 16:31:05 -0700 Subject: [PATCH] YARN-1050. Document the Fair Scheduler REST API. (Kenji Kikushima and Roman Shaposhnik via kasha) (cherry picked from commit 96473bdc2b090c13708fd467fd626621ef1d47eb) --- hadoop-yarn-project/CHANGES.txt | 2 + .../src/site/markdown/ResourceManagerRest.md | 306 +++++++++++++++++- 2 files changed, 307 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index aef444602a9..6e555ca342c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -164,6 +164,8 @@ Release 2.8.0 - UNRELEASED YARN-2331. Distinguish shutdown during supervision vs. shutdown for rolling upgrade. (Jason Lowe via xgong) + YARN-1050. Document the Fair Scheduler REST API. (Kenji Kikushima and Roman Shaposhnik via kasha) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index b1591bb5ea6..d5df8ba21d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -278,7 +278,7 @@ Response Body: Cluster Scheduler API --------------------- -A scheduler resource contains information about the current scheduler configured in a cluster. It currently supports both the Fifo and Capacity Scheduler. You will get different information depending on which scheduler is configured so be sure to look at the type information. +A scheduler resource contains information about the current scheduler configured in a cluster. It currently supports the Fifo, Capacity and Fair Scheduler. You will get different information depending on which scheduler is configured so be sure to look at the type information. ### URI @@ -984,6 +984,310 @@ Response Body: ``` +### Fair Scheduler API + +### Elements of the *schedulerInfo* object + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| type | string | Scheduler type - fairScheduler | +| rootQueue | The root queue object | A collection of root queue resources | + +### Elements of the root queue object + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| maxApps | int | The maximum number of applications the queue can have | +| minResources | A single resource object | The configured minimum resources that are guaranteed to the queue | +| maxResources | A single resource object | The configured maximum resources that are allowed to the queue | +| usedResources | A single resource object | The sum of resources allocated to containers within the queue | +| fairResources | A single resource object | The queue's fair share of resources | +| clusterResources | A single resource object | The capacity of the cluster | +| queueName | string | The name of the queue | +| schedulingPolicy | string | The name of the scheduling policy used by the queue | +| childQueues | array of queues(JSON)/queue objects(XML) | A collection of sub-queue information | + +### Elements of the queues object for a Leaf queue - contains all elements in parent plus the following + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| type | string | type of the queue - fairSchedulerLeafQueueInfo | +| numActiveApps | int | The number of active applications in this queue | +| numPendingApps | int | The number of pending applications in this queue | + +### Elements of the resource object for resourcesUsed in queues + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| memory | int | The amount of memory used (in MB) | +| vCores | int | The number of virtual cores | + +#### Response Examples + +**JSON response** + +HTTP Request: + + GET http:///ws/v1/cluster/scheduler + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) + +Response Body: + +```json +{ + "scheduler": { + "schedulerInfo": { + "rootQueue": { + "childQueues": [ + { + "clusterResources": { + "memory": 8192, + "vCores": 8 + }, + "fairResources": { + "memory": 0, + "vCores": 0 + }, + "maxApps": 2147483647, + "maxResources": { + "memory": 8192, + "vCores": 8 + }, + "minResources": { + "memory": 0, + "vCores": 0 + }, + "numActiveApps": 0, + "numPendingApps": 0, + "queueName": "root.default", + "schedulingPolicy": "fair", + "type": "fairSchedulerLeafQueueInfo", + "usedResources": { + "memory": 0, + "vCores": 0 + } + }, + { + "childQueues": { + "clusterResources": { + "memory": 8192, + "vCores": 8 + }, + "fairResources": { + "memory": 10000, + "vCores": 0 + }, + "maxApps": 2147483647, + "maxResources": { + "memory": 8192, + "vCores": 8 + }, + "minResources": { + "memory": 5000, + "vCores": 0 + }, + "numActiveApps": 0, + "numPendingApps": 0, + "queueName": "root.sample_queue.sample_sub_queue", + "schedulingPolicy": "fair", + "type": [ + "fairSchedulerLeafQueueInfo" + ], + "usedResources": { + "memory": 0, + "vCores": 0 + } + }, + "clusterResources": { + "memory": 8192, + "vCores": 8 + }, + "fairResources": { + "memory": 10000, + "vCores": 0 + }, + "maxApps": 50, + "maxResources": { + "memory": 8192, + "vCores": 0 + }, + "minResources": { + "memory": 10000, + "vCores": 0 + }, + "queueName": "root.sample_queue", + "schedulingPolicy": "fair", + "usedResources": { + "memory": 0, + "vCores": 0 + } + } + ], + "clusterResources": { + "memory": 8192, + "vCores": 8 + }, + "fairResources": { + "memory": 8192, + "vCores": 8 + }, + "maxApps": 2147483647, + "maxResources": { + "memory": 8192, + "vCores": 8 + }, + "minResources": { + "memory": 0, + "vCores": 0 + }, + "queueName": "root", + "schedulingPolicy": "fair", + "usedResources": { + "memory": 0, + "vCores": 0 + } + }, + "type": "fairScheduler" + } + } +} +``` + +**XML response** + +HTTP Request: + + GET http:///ws/v1/cluster/scheduler + Accept: application/xml + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2321 + Server: Jetty(6.1.26) + +Response Body: + +```xml + + + + + 2147483647 + + 0 + 0 + + + 8192 + 8 + + + 0 + 0 + + + 8192 + 8 + + + 8192 + 8 + + root + fair + + 2147483647 + + 0 + 0 + + + 8192 + 8 + + + 0 + 0 + + + 0 + 0 + + + 8192 + 8 + + root.default + fair + 0 + 0 + + + 50 + + 10000 + 0 + + + 8192 + 0 + + + 0 + 0 + + + 10000 + 0 + + + 8192 + 8 + + root.sample_queue + fair + + 2147483647 + + 5000 + 0 + + + 8192 + 8 + + + 0 + 0 + + + 10000 + 0 + + + 8192 + 8 + + root.sample_queue.sample_sub_queue + fair + 0 + 0 + + + + + +``` + + Cluster Applications API ------------------------