diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 618ad4cb44d..ae3aef579f9 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -145,7 +145,7 @@ - + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 9c9b03e422b..737bdc20d7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -15,19 +15,7 @@ Hadoop: Capacity Scheduler ========================== -* [Purpose](#Purpose) -* [Overview](#Overview) -* [Features](#Features) -* [Configuration](#Configuration) - * [Setting up `ResourceManager` to use `CapacityScheduler`](#Setting_up_ResourceManager_to_use_CapacityScheduler`) - * [Setting up queues](#Setting_up_queues) - * [Queue Properties](#Queue_Properties) - * [Setup for application priority](#Setup_for_application_priority.) - * [Capacity Scheduler container preemption](#Capacity_Scheduler_container_preemption) - * [Configuring `ReservationSystem` with `CapacityScheduler`](#Configuring_ReservationSystem_with_CapacityScheduler) - * [Other Properties](#Other_Properties) - * [Reviewing the configuration of the CapacityScheduler](#Reviewing_the_configuration_of_the_CapacityScheduler) -* [Changing Queue Configuration](#Changing_Queue_Configuration) + Purpose ------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index b74fa7b47d7..e66d079363f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -15,13 +15,7 @@ Launching Applications Using Docker Containers ============================================== -* [Overview](#Overview) -* [Cluster Configuration](#Cluster_Configuration) -* [Docker Image Requirements](#Docker_Image_Requirements) -* [Application Submission](#Application_Submission) -* [Connecting to a Secure Docker Repository](#Connecting_to_a_Secure_Docker_Repository) -* [Example: MapReduce](#Example:_MapReduce) -* [Example: Spark](#Example:_Spark) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index 5d7b3dccb42..2f0164d29c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -15,19 +15,7 @@ Hadoop: Fair Scheduler ====================== -* [Purpose](#Purpose) -* [Introduction](#Introduction) -* [Hierarchical queues with pluggable policies](#Hierarchical_queues_with_pluggable_policies) -* [Automatically placing applications in queues](#Automatically_placing_applications_in_queues) -* [Installation](#Installation) -* [Configuration](#Configuration) - * [Properties that can be placed in yarn-site.xml](#Properties_that_can_be_placed_in_yarn-site.xml) - * [Allocation file format](#Allocation_file_format) - * [Queue Access Control Lists](#Queue_Access_Control_Lists) -* [Administration](#Administration) - * [Modifying configuration at runtime](#Modifying_configuration_at_runtime) - * [Monitoring through web UI](#Monitoring_through_web_UI) - * [Moving applications between queues](#Moving_applications_between_queues) + ##Purpose diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md index 73c9f41ac9e..39d8311fa08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md @@ -15,19 +15,7 @@ YARN Node Labels =============== -* [Overview](#Overview) -* [Features](#Features) -* [Configuration](#Configuration) - * [Setting up ResourceManager to enable Node Labels](#Setting_up_ResourceManager_to_enable_Node_Labels) - * [Add/modify node labels list to YARN](#Add/modify_node_labels_list_to_YARN) - * [Remove node labels from YARN](#Remove_node_labels_from_YARN) - * [Add/modify node-to-labels mapping to YARN](#Add/modify_node-to-labels_mapping_to_YARN) - * [Configuration of Schedulers for node labels](#Configuration_of_Schedulers_for_node_labels) -* [Specifying node label for application](#Specifying_node_label_for_application) -* [Monitoring](#Monitoring) - * [Monitoring through web UI](#Monitoring_through_web_UI) - * [Monitoring through commandline](#Monitoring_through_commandline) -* [Useful links](#Useful_links) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md index b3a92dc2a87..3261cd7489b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md @@ -15,13 +15,7 @@ NodeManager =========== -* [Overview](#Overview) -* [Health Checker Service](#Health_checker_service) - * [Disk Checker](#Disk_Checker) - * [External Health Script](#External_Health_Script) -* [NodeManager Restart](#NodeManager_Restart) - * [Introduction](#Introduction) - * [Enabling NM Restart](#Enabling_NM_Restart) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md index 50f2faf2c2c..2704f10a1c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md @@ -15,8 +15,7 @@ Using CGroups with YARN ======================= -* [CGroups Configuration](#CGroups_configuration) -* [CGroups and Security](#CGroups_and_security) + CGroups is a mechanism for aggregating/partitioning sets of tasks, and all their future children, into hierarchical groups with specialized behaviour. CGroups is a Linux kernel feature and was merged into kernel version 2.6.24. From a YARN perspective, this allows containers to be limited in their resource usage. A good example of this is CPU usage. Without CGroups, it becomes hard to limit container CPU usage. Currently, CGroups is only used for limiting CPU usage. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md index 26b21bec797..98d570ca29a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md @@ -15,13 +15,7 @@ NodeManager REST API's ======================= -* [Overview](#Overview) -* [Enabling CORS support](#Enabling_CORS_support) -* [NodeManager Information API](#NodeManager_Information_API) -* [Applications API](#Applications_API) -* [Application API](#Application_API) -* [Containers API](#Containers_API) -* [Container API](#Container_API) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md index 496777abff7..83beb0742af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md @@ -15,22 +15,7 @@ Opportunistic Containers ======================== -* [Purpose](#Purpose) -* [Quick Guide](#Quick_Guide) - * [Main Goal](#Main_Goal) - * [Enabling Opportunistic Containers](#Enabling_Opportunistic_Containers) - * [Running a Sample Job](#Running_a_Sample_Job) - * [Opportunistic Containers in Web UI](#Opportunistic_Containers_in_Web_UI) -* [Overview](#Overview) -* [Container Execution Types](#Container_Execution_Types) -* [Execution of Opportunistic Containers](#Execution_of_Opportunistic_Containers) -* [Allocation of Opportunistic Containers](#Allocation_of_Opportunistic_Containers) - * [Centralized Allocation](#Centralized_Allocation) - * [Distributed Allocation](#Distributed_Allocation) - * [Determining Nodes for Allocation](#Determining_Nodes_for_Allocation) - * [Rebalancing Node Load](#Rebalancing_Node_Load) -* [Advanced Configuration](#Advanced_Configuration) -* [Items for Future Work](#Items_for_Future_Work) + Purpose diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md index 7d1309ffde1..cd4ae54e01c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md @@ -15,11 +15,7 @@ Reservation System ================== - -* [Purpose](#Purpose) -* [Overview](#Overview) -* [Flow of a Reservation](#Flow_of_a_Reservation) -* [Configuring the Reservation System](#Configuring_the_Reservation_System) + Purpose ------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md index 49669a14c76..984959f8f76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md @@ -15,15 +15,7 @@ ResourceManager High Availability ================================= -* [Introduction](#Introduction) -* [Architecture](#Architecture) - * [RM Failover](#RM_Failover) - * [Recovering prevous active-RM's state](#Recovering_prevous_active-RMs_state) -* [Deployment](#Deployment) - * [Configurations](#Configurations) - * [Admin commands](#Admin_commands) - * [ResourceManager Web UI services](#ResourceManager_Web_UI_services) - * [Web Services](#Web_Services) + Introduction ------------ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 7881b5db0be..48dea5ac835 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -15,32 +15,7 @@ ResourceManager REST API's. =========================== -* [Overview](#Overview) -* [Enabling CORS support](#Enabling_CORS_support) -* [Cluster Information API](#Cluster_Information_API) -* [Cluster Metrics API](#Cluster_Metrics_API) -* [Cluster Scheduler API](#Cluster_Scheduler_API) -* [Cluster Applications API](#Cluster_Applications_API) -* [Cluster Application Statistics API](#Cluster_Application_Statistics_API) -* [Cluster Application API](#Cluster_Application_API) -* [Cluster Application Attempts API](#Cluster_Application_Attempts_API) -* [Cluster Nodes API](#Cluster_Nodes_API) -* [Cluster Node API](#Cluster_Node_API) -* [Cluster Writeable APIs](#Cluster_Writeable_APIs) -* [Cluster New Application API](#Cluster_New_Application_API) -* [Cluster Applications API(Submit Application)](#Cluster_Applications_APISubmit_Application) -* [Cluster Application State API](#Cluster_Application_State_API) -* [Cluster Application Queue API](#Cluster_Application_Queue_API) -* [Cluster Application Priority API](#Cluster_Application_Priority_API) -* [Cluster Delegation Tokens API](#Cluster_Delegation_Tokens_API) -* [Cluster Reservation API List](#Cluster_Reservation_API_List) -* [Cluster Reservation API Create](#Cluster_Reservation_API_Create) -* [Cluster Reservation API Submit](#Cluster_Reservation_API_Submit) -* [Cluster Reservation API Update](#Cluster_Reservation_API_Update) -* [Cluster Reservation API Delete](#Cluster_Reservation_API_Delete) -* [Cluster Application Timeouts API](#Cluster_Application_Timeouts_API) -* [Cluster Application Timeout API](#Cluster_Application_Timeout_API) -* [Cluster Application Timeout Update API](#Cluster_Application_Timeout_Update_API) + Overview -------- @@ -4280,4 +4255,4 @@ Response Body: 2016-11-27T09:36:16.678+05:30 90 -``` \ No newline at end of file +``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md index ee222c735d8..e18945d74b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRestart.md @@ -15,18 +15,7 @@ ResourceManger Restart ====================== -* [Overview](#Overview) -* [Feature](#Feature) -* [Configurations](#Configurations) - * [Enable RM Restart](#Enable_RM_Restart) - * [Configure the state-store for persisting the RM state](#Configure_the_state-store_for_persisting_the_RM_state) - * [How to choose the state-store implementation](#How_to_choose_the_state-store_implementation) - * [Configurations for Hadoop FileSystem based state-store implementation](#Configurations_for_Hadoop_FileSystem_based_state-store_implementation) - * [Configurations for ZooKeeper based state-store implementation](#Configurations_for_ZooKeeper_based_state-store_implementation) - * [Configurations for LevelDB based state-store implementation](#Configurations_for_LevelDB_based_state-store_implementation) - * [Configurations for work-preserving RM recovery](#Configurations_for_work-preserving_RM_recovery) -* [Notes](#Notes) -* [Sample Configurations](#Sample_Configurations) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md index f7706c7426b..a66c0421370 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md @@ -15,7 +15,7 @@ YARN Secure Containers ====================== -* [Overview](#Overview) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index ae9faae825a..ac9b2ec9d10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -15,21 +15,7 @@ The YARN Timeline Server ======================== -* [Overview](#Overview) - * [Introduction](#Introduction) - * [Current Status](#Current_Status) - * [Timeline Structure](#Timeline_Structure) -* [Deployment](#Deployment) - * [Configurations](#Configurations) - * [Running the Timeline Server](#Running_Timeline_Server) - * [Accessing generic-data via command-line](#Accessing_generic-data_via_command-line) -* [Publishing of application specific data](#Publishing_of_application_specific_data) -* [Timeline Server REST API](#Timeline_Server_REST_API_v1) -* [Generic Data REST APIs](#GENERIC_DATA_REST_APIS) -* [Timelnine Server Performance Test Tool](#TIMELINE_SERVER_PERFORMANCE_TEST_TOOL) - * [Highlights](#HIGHLIGHTS) - * [Usage](#USAGE) - * [Sample Runs](#SAMPLE_RUNS) + Overview --------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index 61aa6eda008..90c7a89f52c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -15,25 +15,9 @@ The YARN Timeline Service v.2 ======================== -* [Overview](#Overview) - * [Introduction](#Introduction) - * [Architecture](#Architecture) - * [Current Status](#Current_Status) -* [Deployment](#Deployment) - * [Configurations](#Configurations) - * [Enabling Timeline Service v.2](#Enabling_Timeline_Service_v2) -* [Publishing of application specific data](#Publishing_of_application_specific_data) -* [Timeline Service v.2 REST API](#Timeline_Service_REST_API_v2) - * [Query Flows](#REST_API_LIST_FLOWS) - * [Query Flow Runs](#REST_API_LIST_FLOW_RUNS) - * [Query Flow Run](#REST_API_LIST_FLOW_RUN) - * [Query Apps for a Flow](#REST_API_LIST_FLOW_APPS) - * [Query Apps for a Flow Run](#REST_API_LIST_FLOWRUN_APPS) - * [Query App](#REST_API_LIST_APP) - * [Query Generic Entities](#REST_API_LIST_ENTITIES) - * [Query Generic Entity](#REST_API_LIST_ENTITY) + -#Overview +## Overview ### Introduction @@ -128,9 +112,9 @@ analysis * Migration and compatibility with v.1 -#Deployment +## Deployment -###Configurations +### Configurations New configuration parameters that are introduced with v.2 are marked bold. @@ -269,7 +253,7 @@ To write MapReduce framework data to Timeline Service v.2, enable the following ``` -### Publishing application specific data +### Publishing application specific data This section is for YARN application developers that want to integrate with Timeline Service v.2. @@ -402,7 +386,7 @@ You can provide the flow context via YARN application tags: appContext.setApplicationTags(tags); -# Timeline Service v.2 REST API +## Timeline Service v.2 REST API Querying Timeline Service v.2 is currently only supported via REST API; there is no API client implemented in the YARN libraries. @@ -430,14 +414,14 @@ Returns a JSON object describing the service instance and version information. The following shows the supported queries on the REST API. -## Query Flows +### Query Flows With Query Flows API, you can retrieve a list of active flows that had runs most recently. If the REST endpoint without the cluster name is used, the cluster specified by the configuration `yarn.resourcemanager.cluster-id` in `yarn-site.xml` is taken. If none of the flows match the predicates, an empty list will be returned. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/flows/ @@ -445,7 +429,7 @@ predicates, an empty list will be returned. GET /ws/v2/timeline/flows/ -### Query Parameters Supported: +#### Query Parameters Supported: 1. `limit` - If specified, defines the number of flows to return. The maximum possible value for limit is maximum value of Long. If it is not specified or has a value less than 0, then limit will be @@ -462,7 +446,7 @@ predicates, an empty list will be returned. "daterange=20150711-" returns flows active on and after 20150711.
"daterange=-20150711" returns flows active on and before 20150711.
-### Example JSON Response: +#### Example JSON Response: [ { @@ -515,14 +499,14 @@ predicates, an empty list will be returned. } ] -### Response Codes +#### Response Codes 1. If successful, a HTTP 200 (OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query Flow Runs +### Query Flow Runs With Query Flow Runs API, you can drill further down to get the runs (specific instances) of a given flow. This returns the most recent runs that belong to the given flow. If the REST @@ -530,7 +514,7 @@ endpoint without the cluster name is used, the cluster specified by the configur `yarn.resourcemanager.cluster-id` in `yarn-site.xml` is taken. If none of the flow runs match the predicates, an empty list will be returned. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/users/{user name}/flows/{flow name}/runs/ @@ -538,7 +522,7 @@ predicates, an empty list will be returned. GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs/ -### Query Parameters Supported: +#### Query Parameters Supported: 1. `limit` - If specified, defines the number of flows to return. The maximum possible value for limit is maximum value of Long. If it is not specified or has a value less than 0, then limit will be @@ -558,7 +542,7 @@ predicates, an empty list will be returned. Other fields will lead to HTTP 400 (Bad Request) response. If not specified, in response, id, type, createdtime and info fields will be returned. -### Example JSON Response: +#### Example JSON Response: [ { @@ -595,21 +579,21 @@ predicates, an empty list will be returned. } ] -### Response Codes +#### Response Codes 1. If successful, a HTTP 200 (OK) response is returned. 1. If any problem occurs in parsing request or if an invalid field is specified in fields query param, HTTP 400 (Bad Request) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query Flow Run +### Query Flow Run With this API, you can query a specific flow run identified by cluster, user, flow name and run id. If the REST endpoint without the cluster name is used, the cluster specified by the configuration `yarn.resourcemanager.cluster-id` in `yarn-site.xml` is taken. Metrics are returned by default while querying individual flow runs. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/users/{user name}/flows/{flow name}/runs/{run id} @@ -617,7 +601,7 @@ while querying individual flow runs. GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs/{run id} -### Query Parameters Supported: +#### Query Parameters Supported: 1. `metricstoretrieve` - If specified, defines which metrics to retrieve or which ones not to retrieve and send back in response. metricstoretrieve can be an expression of the form :
@@ -628,7 +612,7 @@ while querying individual flow runs. This specifies a comma separated list of metric id prefixes. Only metrics not matching any of the prefixes will be retrieved.
Please note that URL unsafe characters such as spaces will have to be suitably encoded. -### Example JSON Response: +#### Example JSON Response: { "metrics": [ @@ -664,14 +648,14 @@ while querying individual flow runs. "relatesto": {} } -### Response Codes +#### Response Codes 1. If successful, a HTTP 200(OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. If flow run for the given flow run id cannot be found, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query Apps for a flow +### Query Apps for a flow With this API, you can query all the YARN applications that are part of a specific flow. If the REST endpoint without the cluster name is used, the cluster specified by the configuration @@ -679,7 +663,7 @@ REST endpoint without the cluster name is used, the cluster specified by the con applications are more than the limit, the most recent apps up to the limit will be returned. If none of the apps match the predicates, an empty list will be returned. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/users/{user name}/flows/{flow name}/apps @@ -687,7 +671,7 @@ none of the apps match the predicates, an empty list will be returned. GET /ws/v2/timeline/users/{user name}/flows/{flow name}/apps -### Query Parameters Supported: +#### Query Parameters Supported: 1. `limit` - If specified, defines the number of applications to return. The maximum possible value for limit is maximum value of Long. If it is not specified or has a value less than 0, then limit will be @@ -761,7 +745,7 @@ none of the apps match the predicates, an empty list will be returned. Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. -### Example JSON Response: +#### Example JSON Response: [ { @@ -792,13 +776,13 @@ none of the apps match the predicates, an empty list will be returned. } ] -### Response Codes +#### Response Codes 1. If successful, a HTTP 200 (OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query Apps for a flow run +### Query Apps for a flow run With this API, you can query all the YARN applications that are part of a specific flow run. If the REST endpoint without the cluster name is used, the cluster specified by the configuration @@ -806,7 +790,7 @@ REST endpoint without the cluster name is used, the cluster specified by the con are more than the limit, the most recent apps up to the limit will be returned. If none of the apps match the predicates, an empty list will be returned. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/users/{user name}/flows/{flow name}/runs/{run id}/apps @@ -814,7 +798,7 @@ match the predicates, an empty list will be returned. GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs/{run id}/apps/ -### Query Parameters Supported: +#### Query Parameters Supported: 1. `limit` - If specified, defines the number of applications to return. The maximum possible value for limit is maximum value of Long. If it is not specified or has a value less than 0, then limit will be @@ -888,7 +872,7 @@ match the predicates, an empty list will be returned. Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. -### Example JSON Response: +#### Example JSON Response: [ { @@ -906,14 +890,14 @@ match the predicates, an empty list will be returned. } ] -### Response Codes +#### Response Codes 1. If successful, a HTTP 200 (OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query app +### Query app With this API, you can query a single YARN application identified by the cluster and the application ID. If the REST endpoint without the cluster name is used, the cluster specified by the @@ -922,7 +906,7 @@ information i.e. user, flow name and run id are not mandatory but if specified i preclude the need for an additional operation to fetch flow context information based on cluster and app id. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/apps/{app id} @@ -930,7 +914,7 @@ and app id. GET /ws/v2/timeline/apps/{app id} -### Query Parameters Supported: +#### Query Parameters Supported: 1. `userid` - If specified, only applications belonging to this user will be returned. This query param must be specified along with flowname and flowrunid query params, otherwise it will be ignored. If userid, flowname and flowrunid are not specified, we would have to fetch flow context information based on cluster and appid while executing the query. @@ -964,7 +948,7 @@ and app id. Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. -### Example JSON Response: +#### Example JSON Response: { "metrics": [], @@ -980,14 +964,14 @@ and app id. "relatesto": {} } -### Response Codes +#### Response Codes 1. If successful, a HTTP 200(OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. If flow context information cannot be retrieved or application for the given app id cannot be found, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query generic entities +### Query generic entities With this API, you can query generic entities identified by cluster ID, application ID and per-framework entity type. If the REST endpoint without the cluster name is used, the cluster @@ -1001,7 +985,7 @@ For instance, we can query containers by specifying entity type as `YARN_CONTAIN attempts by specifying entity type as `YARN_APPLICATION_ATTEMPT`. If none of the entities match the predicates, an empty list will be returned. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/apps/{app id}/entities/{entity type} @@ -1009,7 +993,7 @@ If none of the entities match the predicates, an empty list will be returned. GET /ws/v2/timeline/apps/{app id}/entities/{entity type} -### Query Parameters Supported: +#### Query Parameters Supported: 1. `userid` - If specified, only entities belonging to this user will be returned. This query param must be specified along with flowname and flowrunid query params, otherwise it will be ignored. If userid, flowname and flowrunid are not specified, we would have to fetch flow context information based on cluster and appid while executing the query. @@ -1088,7 +1072,7 @@ If none of the entities match the predicates, an empty list will be returned. Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. -### Example JSON Response: +#### Example JSON Response: [ { @@ -1119,14 +1103,14 @@ If none of the entities match the predicates, an empty list will be returned. } ] -### Response Codes +#### Response Codes 1. If successful, a HTTP 200(OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. If flow context information cannot be retrieved, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -## Query generic entity +### Query generic entity With this API, you can query a specific generic entity identified by cluster ID, application ID, per-framework entity type and entity ID. If the REST endpoint without the cluster name is used, the @@ -1139,7 +1123,7 @@ a specific YARN container by specifying entity type as `YARN_CONTAINER` and givi container ID. Similarly, application attempt can be queried by specifying entity type as `YARN_APPLICATION_ATTEMPT` and entity ID being the application attempt ID. -### HTTP request: +#### HTTP request: GET /ws/v2/timeline/clusters/{cluster name}/apps/{app id}/entities/{entity type}/{entity id} @@ -1147,7 +1131,7 @@ container ID. Similarly, application attempt can be queried by specifying entity GET /ws/v2/timeline/apps/{app id}/entities/{entity type}/{entity id} -### Query Parameters Supported: +#### Query Parameters Supported: 1. `userid` - If specified, entity must belong to this user. This query param must be specified along with flowname and flowrunid query params, otherwise it will be ignored. If userid, flowname and flowrunid are not specified, we would have to fetch flow context information based on cluster and appid while executing the query. @@ -1181,7 +1165,7 @@ container ID. Similarly, application attempt can be queried by specifying entity Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. -### Example JSON Response: +#### Example JSON Response: { "metrics": [ ], @@ -1197,7 +1181,7 @@ container ID. Similarly, application attempt can be queried by specifying entity "relatesto": { } } -### Response Codes +#### Response Codes 1. If successful, a HTTP 200 (OK) response is returned. 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebApplicationProxy.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebApplicationProxy.md index 5fa9a9de02a..c3677cf92fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebApplicationProxy.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebApplicationProxy.md @@ -15,12 +15,7 @@ Web Application Proxy ===================== -* [Overview](#Overview) - * [Introduction](#Introduction) - * [Current Status](#Current_Status) -* [Deployment](#Deployment) - * [Configurations](#Configurations) - * [Running Web Application Proxy](#Running_Web_Proxy) + Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md index 0a396d32d5f..11452f65785 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md @@ -15,18 +15,7 @@ Hadoop YARN - Introduction to the web services REST API's ========================================================== -* [Overview](#Overview) -* [URI's](#URIs) -* [HTTP Requests](#HTTP_Requests) - * [Summary of HTTP operations](#Summary_of_HTTP_operations) - * [Security](#Security) - * [Headers Supported](#Headers_Supported) -* [HTTP Responses](#HTTP_Responses) - * [Compression](#Compression) - * [Response Formats](#Response_Formats) - * [Response Errors](#Response_Errors) - * [Response Examples](#Response_Examples) -* [Sample Usage](#Sample_Usage) + Overview -------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md index cdad3f92231..07c37655d8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md @@ -15,19 +15,7 @@ Hadoop: Writing YARN Applications ================================= -* [Purpose](#Purpose) -* [Concepts and Flow](#Concepts_and_Flow) -* [Interfaces](#Interfaces) -* [Writing a Simple Yarn Application](#Writing_a_Simple_Yarn_Application) - * [Writing a simple Client](#Writing_a_simple_Client) - * [Writing an ApplicationMaster (AM)](#Writing_an_ApplicationMaster_AM) -* [FAQ](#FAQ) - * [How can I distribute my application's jars to all of the nodes in the YARN cluster that need it?](#How_can_I_distribute_my_applications_jars_to_all_of_the_nodes_in_the_YARN_cluster_that_need_it) - * [How do I get the ApplicationMaster's ApplicationAttemptId?](#How_do_I_get_the_ApplicationMasters_ApplicationAttemptId) - * [Why my container is killed by the NodeManager?](#Why_my_container_is_killed_by_the_NodeManager) - * [How do I include native libraries?](#How_do_I_include_native_libraries) -* [Useful Links](#Useful_Links) -* [Sample Code](#Sample_Code) + Purpose ------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md index 640139f9831..bab46b9384b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md @@ -14,6 +14,8 @@ # YARN Application Security + + Anyone writing a YARN application needs to understand the process, in order to write short-lived applications or long-lived services. They also need to start testing on secure clusters during early development stages, in order diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 708a8b4781b..478377fadb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -15,31 +15,7 @@ YARN Commands ============= -* [Overview](#Overview) -* [User Commands](#User_Commands) - * [application](#application) - * [applicationattempt](#applicationattempt) - * [classpath](#classpath) - * [container](#container) - * [jar](#jar) - * [logs](#logs) - * [node](#node) - * [queue](#queue) - * [version](#version) -* [Administration Commands](#Administration_Commands) - * [daemonlog](#daemonlog) - * [nodemanager](#nodemanager) - * [proxyserver](#proxyserver) - * [resourcemanager](#resourcemanager) - * [rmadmin](#rmadmin) - * [scmadmin](#scmadmin) - * [sharedcachemanager](#sharedcachemanager) - * [timelineserver](#timelineserver) -* [Files](#Files) - * [etc/hadoop/hadoop-env.sh](#etchadoophadoop-env.sh) - * [etc/hadoop/yarn-env.sh](#etchadoopyarn-env.sh) - * [etc/hadoop/hadoop-user-functions.sh](#etchadoophadoop-user-functions.sh) - * [~/.hadooprc](#a.hadooprc) + Overview --------