From 5fe1dbf1959976d0dc5a8e614dd74836cfbee04c Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 22 Nov 2018 10:25:22 +0530 Subject: [PATCH] YARN-8964. [UI2] YARN ui2 should use clusters/{cluster name} for all ATSv2 REST APIs. Contributed by Akhil PB. --- .../main/webapp/app/adapters/yarn-app-log.js | 11 ++-- .../webapp/app/adapters/yarn-app-timeline.js | 6 +- .../app/adapters/yarn-component-instance.js | 8 ++- .../webapp/app/adapters/yarn-flow-activity.js | 4 ++ .../src/main/webapp/app/adapters/yarn-log.js | 8 +-- .../app/adapters/yarn-service-component.js | 8 ++- .../webapp/app/adapters/yarn-service-info.js | 8 ++- .../app/adapters/yarn-timeline-appattempt.js | 8 +++ .../app/adapters/yarn-timeline-container.js | 4 ++ .../main/webapp/app/initializers/loader.js | 34 +++++++++- .../src/main/webapp/app/router.js | 6 +- .../webapp/app/routes/yarn-log-service.js | 66 +++++++++++++++++++ .../webapp/app/serializers/yarn-app-log.js | 4 +- .../webapp/app/templates/yarn-log-service.hbs | 37 +++++++++++ .../src/main/webapp/app/utils/converter.js | 2 +- .../src/main/webapp/config/default-config.js | 1 + 16 files changed, 194 insertions(+), 21 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-log-service.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-log-service.hbs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-log.js index 318b27319cf..847171e22bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-log.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-log.js @@ -16,7 +16,6 @@ * limitations under the License. */ -import DS from 'ember-data'; import Ember from 'ember'; import Converter from 'yarn-ui/utils/converter'; import RESTAbstractAdapter from './restabstract'; @@ -26,8 +25,8 @@ import RESTAbstractAdapter from './restabstract'; * in plain text format and not JSON. */ export default RESTAbstractAdapter.extend({ - address: "timelineV1WebAddress", - restNameSpace: "timeline", + address: "timelineWebAddress", + restNameSpace: "timelineV2Log", serverName: "ATS", headers: { @@ -36,10 +35,14 @@ export default RESTAbstractAdapter.extend({ urlForFindRecord(id/*, modelName, snapshot*/) { var splits = Converter.splitForAppLogs(id); + var clusterId = this.get("env.app.clusterId"); var containerId = splits[0]; var logFile = splits[1]; + if (splits[2]) { + clusterId = splits[2]; + } var url = this._buildURL(); - url = url + '/containers/' + containerId + '/logs/' + logFile; + url = url + '/containers/' + containerId + '/logs/' + logFile + '?clusterid=' + clusterId; console.log('log url' + url); return url; }, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-timeline.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-timeline.js index 831649ce060..f2bc30db209 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-timeline.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-app-timeline.js @@ -25,7 +25,11 @@ export default AbstractAdapter.extend({ urlForFindRecord(id/*, modelName, snapshot*/) { var url = this._buildURL(); - url = url + '/apps/' + id + '?fields=ALL'; + var clusterId = this.get("env.app.clusterId"); + if (clusterId) { + url += `/clusters/${clusterId}`; + } + url += '/apps/' + id + '?fields=ALL'; return url; }, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-component-instance.js index 062a0062fa6..86e08fe3786 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-component-instance.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-component-instance.js @@ -20,12 +20,16 @@ import AbstractAdapter from './abstract'; export default AbstractAdapter.extend({ address: "timelineWebAddress", - restNameSpace: "timelineService", + restNameSpace: "timelineV2", serverName: "ATS", urlForQuery(query/*, modelName*/) { var url = this.buildURL(); - url += '/' + query.appId + '/entities/COMPONENT_INSTANCE?fields=ALL'; + var clusterId = this.get("env.app.clusterId"); + if (clusterId) { + url += `/clusters/${clusterId}`; + } + url += '/apps/' + query.appId + '/entities/COMPONENT_INSTANCE?fields=ALL'; delete query.appId; return url; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-flow-activity.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-flow-activity.js index a8c436aa565..cb8dfee20fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-flow-activity.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-flow-activity.js @@ -25,6 +25,10 @@ export default AbstractAdapter.extend({ serverName: "ATS", pathForType(/*modelName*/) { + var clusterId = this.get("env.app.clusterId"); + if (clusterId) { + return `clusters/${clusterId}/flows`; + } return 'flows'; // move to some common place, return path by modelname. }, }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js index 58cbea2a8bf..66f34061546 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js @@ -19,15 +19,15 @@ import AbstractAdapter from './abstract'; export default AbstractAdapter.extend({ - address: "timelineV1WebAddress", - // restNameSpace: "timelineV2", // Use ATSv2 when it supports log APIs. - restNameSpace: "timeline", //Using ATSv1.5 now, would be supported by ATSv2 very soon. + address: "timelineWebAddress", + restNameSpace: "timelineV2Log", serverName: "ATS", urlForQuery(query/*, modelName*/) { var url = this._buildURL(); var containerId = query['containerId']; + var clusterId = this.get("env.app.clusterId"); delete query.containerId; - return url + '/containers/' + containerId + '/logs'; + return url + '/containers/' + containerId + '/logs' + '?clusterid=' + clusterId; } }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-component.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-component.js index c3561920a65..4fda603e11a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-component.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-component.js @@ -20,12 +20,16 @@ import AbstractAdapter from './abstract'; export default AbstractAdapter.extend({ address: "timelineWebAddress", - restNameSpace: "timelineService", + restNameSpace: "timelineV2", serverName: "ATS", urlForQuery(query/*, modelName*/) { var url = this.buildURL(); - url += '/' + query.appId + '/entities/COMPONENT?fields=ALL'; + var clusterId = this.get("env.app.clusterId"); + if (clusterId) { + url += `/clusters/${clusterId}`; + } + url += '/apps/' + query.appId + '/entities/COMPONENT?fields=ALL'; delete query.appId; return url; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-info.js index dec3e50fc6e..9e3775489a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-info.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-service-info.js @@ -20,12 +20,16 @@ import AbstractAdapter from './abstract'; export default AbstractAdapter.extend({ address: "timelineWebAddress", - restNameSpace: "timelineService", + restNameSpace: "timelineV2", serverName: "ATS", urlForQueryRecord(query/*, modelName*/) { var url = this.buildURL(); - url += '/' + query.appId + '/entities/SERVICE_ATTEMPT?fields=ALL'; + var clusterId = this.get("env.app.clusterId"); + if (clusterId) { + url += `/clusters/${clusterId}`; + } + url += '/apps/' + query.appId + '/entities/SERVICE_ATTEMPT?fields=ALL'; delete query.appId; return url; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-appattempt.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-appattempt.js index 2557a8642ca..cbc10bd6890 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-appattempt.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-appattempt.js @@ -26,6 +26,10 @@ export default AbstractAdapter.extend({ urlForQuery(query/*, modelName*/) { var url = this._buildURL(); + var clusterId = this.get("env.app.clusterId") + if (clusterId) { + url += `/clusters/${clusterId}`; + } var appId = query.appId; query.fields = 'ALL'; delete query.appId; @@ -34,6 +38,10 @@ export default AbstractAdapter.extend({ urlForFindRecord(id/*, modelName, snapshot*/) { var url = this._buildURL(); + var clusterId = this.get("env.app.clusterId") + if (clusterId) { + url += `/clusters/${clusterId}`; + } return url + '/apps/' + Converter.attemptIdToAppId(id) + "/entities/YARN_APPLICATION_ATTEMPT/" + id + "?fields=ALL"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-container.js index f47335a3411..546b03f4627 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-container.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-timeline-container.js @@ -26,6 +26,10 @@ export default AbstractAdapter.extend({ urlForQuery(query/*, modelName*/){ var url = this._buildURL(); + var clusterId = this.get("env.app.clusterId") + if (clusterId) { + url += `/clusters/${clusterId}`; + } var app_attempt_id = query.app_attempt_id; query.fields = 'ALL'; delete query.app_attempt_id; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js index 98d1348c84f..befff8fdcca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js @@ -84,9 +84,36 @@ function getSecurityURL(rmhost) { return url; } +function getClusterIdFromYARN(rmhost, application) { + var httpUrl = window.location.protocol + '//' + + (ENV.hosts.localBaseAddress? ENV.hosts.localBaseAddress + '/' : '') + rmhost; + + httpUrl += '/conf?name=yarn.resourcemanager.cluster-id'; + Ember.Logger.log("Get cluster-id URL is: " + httpUrl); + + var clusterId = ""; + $.ajax({ + type: 'GET', + dataType: 'json', + async: false, + context: this, + url: httpUrl, + success: function(data) { + clusterId = data.property.value; + Ember.Logger.log("Cluster Id from RM: " + clusterId); + application.advanceReadiness(); + }, + error: function() { + application.advanceReadiness(); + } + }); + return clusterId; +} + function updateConfigs(application) { var hostname = window.location.hostname; - var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') + skipTrailingSlash(window.location.pathname); + var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') + + skipTrailingSlash(window.location.pathname); window.ENV = window.ENV || {}; window.ENV.hosts = window.ENV.hosts || {}; @@ -103,6 +130,10 @@ function updateConfigs(application) { var protocolSchemeFromRM = getYarnHttpProtocolScheme(rmhost, application); Ember.Logger.log("Is protocol scheme https? " + (protocolSchemeFromRM == "HTTPS_ONLY")); var isHttpsSchemeEnabled = (protocolSchemeFromRM == "HTTPS_ONLY"); + + var clusterIdFromYARN = getClusterIdFromYARN(rmhost, application); + ENV.clusterId = clusterIdFromYARN; + if(!ENV.hosts.timelineWebAddress) { var timelinehost = ""; $.ajax({ @@ -207,6 +238,7 @@ export default { }; const skipTrailingSlash = function(path) { + path = path.replace('index.html', ''); path = path.replace('ui2/', ''); path = path.replace(/\/$/, ''); console.log('base url:' + path) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js index cd12fd2a449..aa0e983a760 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js @@ -49,8 +49,10 @@ Router.map(function() { { path: '/yarn-node-containers/:node_id/:node_addr' }); this.route('yarn-node-container', { path: '/yarn-node-container/:node_id/:node_addr/:container_id' }); - this.route('yarn-container-log', { path: - '/yarn-container-log/:node_id/:node_addr/:container_id/:filename' }); + this.route('yarn-container-log', + { path: '/yarn-container-log/:node_id/:node_addr/:container_id/:filename' }); + this.route('yarn-log-service', + { path: '/yarn-log-service/:cluster_id/:container_id/:filename' }); this.route('yarn-deploy-service'); this.route('cluster-overview'); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-log-service.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-log-service.js new file mode 100644 index 00000000000..e1c45406d20 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-log-service.js @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; +import Constants from 'yarn-ui/constants'; + +import AbstractRoute from './abstract'; + +export default AbstractRoute.extend({ + model(param) { + var id = param.container_id + Constants.PARAM_SEPARATOR + param.filename + + Constants.PARAM_SEPARATOR + param.cluster_id; + return Ember.RSVP.hash({ + containerLog: this.store.findRecord('yarn-app-log', id), + containerInfo: { id: param.container_id } + }).then(function (hash) { + // Just return as its success. + return hash; + }, function (reason) { + if (reason.errors && reason.errors[0]) { + // This means HTTP error response was sent by adapter. + return reason; + } else { + // Assume empty response received from server. + return { + containerInfo: { id: param.container_id }, + containerLog: { + logs: "", + containerID: param.container_id, + logFileName: param.filename + } + }; + } + }); + }, + + afterModel(model) { + // Handle errors and redirect if promise is rejected. + if (model.errors && model.errors[0]) { + if (parseInt(model.errors[0].status) === 404) { + this.replaceWith('/notfound'); + } else { + this.replaceWith('/error'); + } + } + }, + + unloadAll() { + this.store.unloadAll('yarn-app-log'); + } +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app-log.js index 4bfc5b470b4..3cfabd38186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app-log.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app-log.js @@ -29,8 +29,8 @@ export default DS.JSONAPISerializer.extend({ type: primaryModelClass.modelName, attributes: { logs: payload, - containerID: splits[1], - logFileName: splits[2] + containerID: splits[0], + logFileName: splits[1] } }; return { data: convertedPayload }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-log-service.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-log-service.hbs new file mode 100644 index 00000000000..708c861d871 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-log-service.hbs @@ -0,0 +1,37 @@ +{{!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--}} + +
+
+
+
+
+ Log: {{model.containerLog.logFileName}} [ {{model.containerLog.containerID}} ] +
+
+
+ {{#if model.containerLog.logs}} + 
{{model.containerLog.logs}}
+ {{else}} +

No logs were written in {{model.containerLog.logFileName}}.

+ {{/if}} +
+
+
+
+{{outlet}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js index d5acaa22eb1..689aa5eeff7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js @@ -119,7 +119,7 @@ export default { if (splitLen < 2) { return null; } - return [splits[0], splits[1]]; + return splits[2] ? [splits[0], splits[1], splits[2]] : [splits[0], splits[1]]; } }, memoryToSimpliedUnit: function(mb) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js index 3d3779650a1..9716488bb9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js @@ -31,6 +31,7 @@ module.exports = { // YARN UI App configurations cluster: 'ws/v1/cluster', metrics: 'ws/v1/cluster/metrics', timelineV2: 'ws/v2/timeline', + timelineV2Log: 'ws/v2/applicationlog', dashService: 'app/v1/services', node: '{nodeAddress}/ws/v1/node' },