YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.
This commit is contained in:
parent
b9890d1f66
commit
0a01b1350d
|
@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
|
@ -538,4 +539,16 @@ public class ResourceMgrDelegate extends YarnClient {
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
return client.getResourceTypeInfo();
|
return client.getResourceTypeInfo();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<NodeAttribute> getClusterAttributes()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return client.getClusterAttributes();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||||
|
Set<NodeAttribute> attributes) throws YarnException, IOException {
|
||||||
|
return client.getAttributesToNodes(attributes);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -82,8 +82,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -521,6 +525,19 @@ public class TestClientRedirect {
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class HistoryService extends AMService implements HSClientProtocol {
|
class HistoryService extends AMService implements HSClientProtocol {
|
||||||
|
|
|
@ -27,8 +27,12 @@ import org.apache.hadoop.io.retry.Idempotent;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -642,4 +646,37 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
|
||||||
@Unstable
|
@Unstable
|
||||||
GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
GetAllResourceTypeInfoResponse getResourceTypeInfo(
|
||||||
GetAllResourceTypeInfoRequest request) throws YarnException, IOException;
|
GetAllResourceTypeInfoRequest request) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The interface used by client to get attributes to nodes mappings
|
||||||
|
* available in ResourceManager.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param request request to get details of attributes to nodes mapping.
|
||||||
|
* @return Response containing the details of attributes to nodes mappings.
|
||||||
|
* @throws YarnException if any error happens inside YARN
|
||||||
|
* @throws IOException incase of other errors
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The interface used by client to get node attributes available in
|
||||||
|
* ResourceManager.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param request request to get node attributes collection of this cluster.
|
||||||
|
* @return Response containing node attributes collection.
|
||||||
|
* @throws YarnException if any error happens inside YARN.
|
||||||
|
* @throws IOException incase of other errors.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,70 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The request from clients to get attribtues to nodes mapping
|
||||||
|
* in the cluster from the <code>ResourceManager</code>.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @see ApplicationClientProtocol#getAttributesToNodes
|
||||||
|
* (GetAttributesToNodesRequest)
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract class GetAttributesToNodesRequest {
|
||||||
|
|
||||||
|
public static GetAttributesToNodesRequest newInstance() {
|
||||||
|
return Records.newRecord(GetAttributesToNodesRequest.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static GetAttributesToNodesRequest newInstance(
|
||||||
|
Set<NodeAttribute> attributes) {
|
||||||
|
GetAttributesToNodesRequest request =
|
||||||
|
Records.newRecord(GetAttributesToNodesRequest.class);
|
||||||
|
request.setNodeAttributes(attributes);
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set node attributes for which the mapping is required.
|
||||||
|
*
|
||||||
|
* @param attributes Set<NodeAttribute> provided.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract void setNodeAttributes(Set<NodeAttribute> attributes);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get node attributes for which mapping mapping is required.
|
||||||
|
*
|
||||||
|
* @return Set<NodeAttribute>
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract Set<NodeAttribute> getNodeAttributes();
|
||||||
|
}
|
|
@ -0,0 +1,62 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The response sent by the <code>ResourceManager</code> to a client requesting
|
||||||
|
* attributes to hostname mapping.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @see ApplicationClientProtocol#getAttributesToNodes
|
||||||
|
* (GetAttributesToNodesRequest)
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract class GetAttributesToNodesResponse {
|
||||||
|
public static GetAttributesToNodesResponse newInstance(
|
||||||
|
Map<NodeAttribute, Set<String>> map) {
|
||||||
|
GetAttributesToNodesResponse response =
|
||||||
|
Records.newRecord(GetAttributesToNodesResponse.class);
|
||||||
|
response.setAttributeToNodes(map);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract void setAttributeToNodes(Map<NodeAttribute, Set<String>> map);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Get attributes to node hostname mapping.
|
||||||
|
*
|
||||||
|
* @return Map<NodeAttribute, Set<String>> node attributes to hostname
|
||||||
|
* mapping.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes();
|
||||||
|
}
|
|
@ -0,0 +1,47 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The request from clients to get node attributes in the cluster from the
|
||||||
|
* <code>ResourceManager</code>.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @see ApplicationClientProtocol#getClusterNodeAttributes
|
||||||
|
* (GetClusterNodeAttributesRequest)
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract class GetClusterNodeAttributesRequest {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create new instance of GetClusterNodeAttributesRequest.
|
||||||
|
*
|
||||||
|
* @return GetClusterNodeAttributesRequest is returned.
|
||||||
|
*/
|
||||||
|
public static GetClusterNodeAttributesRequest newInstance() {
|
||||||
|
return Records.newRecord(GetClusterNodeAttributesRequest.class);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,72 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The response sent by the <code>ResourceManager</code> to a client requesting
|
||||||
|
* a node attributes in cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @see ApplicationClientProtocol#getClusterNodeAttributes
|
||||||
|
* (GetClusterNodeAttributesRequest)
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Evolving
|
||||||
|
public abstract class GetClusterNodeAttributesResponse {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create instance of GetClusterNodeAttributesResponse.
|
||||||
|
*
|
||||||
|
* @param attributes
|
||||||
|
* @return GetClusterNodeAttributesResponse.
|
||||||
|
*/
|
||||||
|
public static GetClusterNodeAttributesResponse newInstance(
|
||||||
|
Set<NodeAttribute> attributes) {
|
||||||
|
GetClusterNodeAttributesResponse response =
|
||||||
|
Records.newRecord(GetClusterNodeAttributesResponse.class);
|
||||||
|
response.setNodeAttributes(attributes);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set node attributes to the response.
|
||||||
|
*
|
||||||
|
* @param attributes Node attributes
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract void setNodeAttributes(Set<NodeAttribute> attributes);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get node attributes of the response.
|
||||||
|
*
|
||||||
|
* @return Node attributes
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract Set<NodeAttribute> getNodeAttributes();
|
||||||
|
}
|
|
@ -3491,7 +3491,7 @@ public class YarnConfiguration extends Configuration {
|
||||||
public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS =
|
public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS =
|
||||||
NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class";
|
NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class";
|
||||||
/**
|
/**
|
||||||
* File system not attribute store directory.
|
* File system node attribute store directory.
|
||||||
*/
|
*/
|
||||||
public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR =
|
public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR =
|
||||||
NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir";
|
NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir";
|
||||||
|
|
|
@ -64,4 +64,6 @@ service ApplicationClientProtocolService {
|
||||||
rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto);
|
rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto);
|
||||||
rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto);
|
rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto);
|
||||||
rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto);
|
rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto);
|
||||||
|
rpc getClusterNodeAttributes (GetClusterNodeAttributesRequestProto) returns (GetClusterNodeAttributesResponseProto);
|
||||||
|
rpc getAttributesToNodes (GetAttributesToNodesRequestProto) returns (GetAttributesToNodesResponseProto);
|
||||||
}
|
}
|
||||||
|
|
|
@ -383,6 +383,10 @@ message NodeAttributeProto {
|
||||||
optional string attributeValue = 4 [default=""];
|
optional string attributeValue = 4 [default=""];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message AttributeToNodesProto {
|
||||||
|
required NodeAttributeProto nodeAttribute = 1;
|
||||||
|
repeated string hostnames = 2;
|
||||||
|
}
|
||||||
|
|
||||||
enum ContainerTypeProto {
|
enum ContainerTypeProto {
|
||||||
APPLICATION_MASTER = 1;
|
APPLICATION_MASTER = 1;
|
||||||
|
|
|
@ -260,6 +260,21 @@ message GetClusterNodeLabelsResponseProto {
|
||||||
repeated NodeLabelProto nodeLabels = 2;
|
repeated NodeLabelProto nodeLabels = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message GetClusterNodeAttributesRequestProto {
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetClusterNodeAttributesResponseProto {
|
||||||
|
repeated NodeAttributeProto nodeAttributes = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetAttributesToNodesRequestProto {
|
||||||
|
repeated NodeAttributeProto nodeAttributes = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetAttributesToNodesResponseProto {
|
||||||
|
repeated AttributeToNodesProto attributeToNodes = 1;
|
||||||
|
}
|
||||||
|
|
||||||
message UpdateApplicationPriorityRequestProto {
|
message UpdateApplicationPriorityRequestProto {
|
||||||
required ApplicationIdProto applicationId = 1;
|
required ApplicationIdProto applicationId = 1;
|
||||||
required PriorityProto applicationPriority = 2;
|
required PriorityProto applicationPriority = 2;
|
||||||
|
|
|
@ -33,7 +33,6 @@ import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.service.AbstractService;
|
import org.apache.hadoop.service.AbstractService;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
|
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
|
||||||
|
@ -52,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
|
@ -900,4 +900,38 @@ public abstract class YarnClient extends AbstractService {
|
||||||
@Unstable
|
@Unstable
|
||||||
public abstract List<ResourceTypeInfo> getResourceTypeInfo()
|
public abstract List<ResourceTypeInfo> getResourceTypeInfo()
|
||||||
throws YarnException, IOException;
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The interface used by client to get node attributes in the cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @return cluster node attributes collection
|
||||||
|
* @throws YarnException when there is a failure in
|
||||||
|
* {@link ApplicationClientProtocol}
|
||||||
|
* @throws IOException when there is a failure in
|
||||||
|
* {@link ApplicationClientProtocol}
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract Set<NodeAttribute> getClusterAttributes()
|
||||||
|
throws YarnException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* The interface used by client to get Attributes to nodes mapping
|
||||||
|
* for specified node attributes in existing cluster.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param attributes Attributes for which Attributes to nodes mapping has to
|
||||||
|
* be retrieved.If empty or null is set then will return
|
||||||
|
* all attributes to node mapping in cluster.
|
||||||
|
* @return Attributes to nodes mappings for specific Attributes.
|
||||||
|
* @throws YarnException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||||
|
Set<NodeAttribute> attributes) throws YarnException, IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,6 @@ import java.io.IOException;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -52,8 +51,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
||||||
|
@ -96,6 +97,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
import org.apache.hadoop.yarn.api.records.NodeLabel;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
|
@ -104,7 +106,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
|
||||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
|
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
|
||||||
import org.apache.hadoop.yarn.api.records.Token;
|
import org.apache.hadoop.yarn.api.records.Token;
|
||||||
|
@ -977,4 +978,20 @@ public class YarnClientImpl extends YarnClient {
|
||||||
GetAllResourceTypeInfoRequest.newInstance();
|
GetAllResourceTypeInfoRequest.newInstance();
|
||||||
return rmClient.getResourceTypeInfo(request).getResourceTypeInfo();
|
return rmClient.getResourceTypeInfo(request).getResourceTypeInfo();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<NodeAttribute> getClusterAttributes()
|
||||||
|
throws YarnException, IOException {
|
||||||
|
GetClusterNodeAttributesRequest request =
|
||||||
|
GetClusterNodeAttributesRequest.newInstance();
|
||||||
|
return rmClient.getClusterNodeAttributes(request).getNodeAttributes();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||||
|
Set<NodeAttribute> attributes) throws YarnException, IOException {
|
||||||
|
GetAttributesToNodesRequest request =
|
||||||
|
GetAttributesToNodesRequest.newInstance(attributes);
|
||||||
|
return rmClient.getAttributesToNodes(request).getAttributesToNodes();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,8 +43,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -107,8 +111,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||||
|
@ -673,4 +681,33 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
YarnServiceProtos.GetAttributesToNodesRequestProto requestProto =
|
||||||
|
((GetAttributesToNodesRequestPBImpl) request).getProto();
|
||||||
|
try {
|
||||||
|
return new GetAttributesToNodesResponsePBImpl(
|
||||||
|
proxy.getAttributesToNodes(null, requestProto));
|
||||||
|
} catch (ServiceException e) {
|
||||||
|
RPCUtil.unwrapAndThrowException(e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
YarnServiceProtos.GetClusterNodeAttributesRequestProto requestProto =
|
||||||
|
((GetClusterNodeAttributesRequestPBImpl) request).getProto();
|
||||||
|
try {
|
||||||
|
return new GetClusterNodeAttributesResponsePBImpl(
|
||||||
|
proxy.getClusterNodeAttributes(null, requestProto));
|
||||||
|
} catch (ServiceException e) {
|
||||||
|
RPCUtil.unwrapAndThrowException(e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRes
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
|
||||||
|
@ -73,8 +76,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||||
|
@ -184,6 +191,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResp
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto;
|
||||||
|
|
||||||
import com.google.protobuf.RpcController;
|
import com.google.protobuf.RpcController;
|
||||||
import com.google.protobuf.ServiceException;
|
import com.google.protobuf.ServiceException;
|
||||||
|
@ -694,4 +703,39 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient
|
||||||
throw new ServiceException(ie);
|
throw new ServiceException(ie);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponseProto getClusterNodeAttributes(
|
||||||
|
RpcController controller,
|
||||||
|
YarnServiceProtos.GetClusterNodeAttributesRequestProto proto)
|
||||||
|
throws ServiceException {
|
||||||
|
GetClusterNodeAttributesRequest req =
|
||||||
|
new GetClusterNodeAttributesRequestPBImpl(proto);
|
||||||
|
try {
|
||||||
|
GetClusterNodeAttributesResponse resp =
|
||||||
|
real.getClusterNodeAttributes(req);
|
||||||
|
return ((GetClusterNodeAttributesResponsePBImpl) resp).getProto();
|
||||||
|
} catch (YarnException ye) {
|
||||||
|
throw new ServiceException(ye);
|
||||||
|
} catch (IOException ie) {
|
||||||
|
throw new ServiceException(ie);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponseProto getAttributesToNodes(
|
||||||
|
RpcController controller,
|
||||||
|
YarnServiceProtos.GetAttributesToNodesRequestProto proto)
|
||||||
|
throws ServiceException {
|
||||||
|
GetAttributesToNodesRequestPBImpl req =
|
||||||
|
new GetAttributesToNodesRequestPBImpl(proto);
|
||||||
|
try {
|
||||||
|
GetAttributesToNodesResponse resp = real.getAttributesToNodes(req);
|
||||||
|
return ((GetAttributesToNodesResponsePBImpl) resp).getProto();
|
||||||
|
} catch (YarnException ye) {
|
||||||
|
throw new ServiceException(ye);
|
||||||
|
} catch (IOException ie) {
|
||||||
|
throw new ServiceException(ie);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,175 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||||
|
|
||||||
|
import com.google.protobuf.TextFormat;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto;
|
||||||
|
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attributes to nodes mapping request.
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class GetAttributesToNodesRequestPBImpl
|
||||||
|
extends GetAttributesToNodesRequest {
|
||||||
|
|
||||||
|
private Set<NodeAttribute> nodeAttributes = null;
|
||||||
|
|
||||||
|
private GetAttributesToNodesRequestProto proto =
|
||||||
|
GetAttributesToNodesRequestProto.getDefaultInstance();
|
||||||
|
private GetAttributesToNodesRequestProto.Builder builder = null;
|
||||||
|
private boolean viaProto = false;
|
||||||
|
|
||||||
|
public GetAttributesToNodesRequestPBImpl() {
|
||||||
|
builder = GetAttributesToNodesRequestProto.newBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetAttributesToNodesRequestPBImpl(
|
||||||
|
GetAttributesToNodesRequestProto proto) {
|
||||||
|
this.proto = proto;
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetAttributesToNodesRequestProto getProto() {
|
||||||
|
mergeLocalToProto();
|
||||||
|
proto = viaProto ? proto : builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
return proto;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToProto() {
|
||||||
|
if (viaProto) {
|
||||||
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
|
mergeLocalToBuilder();
|
||||||
|
proto = builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToBuilder() {
|
||||||
|
if (this.nodeAttributes != null) {
|
||||||
|
addLocalAttributesToProto();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addLocalAttributesToProto() {
|
||||||
|
maybeInitBuilder();
|
||||||
|
builder.clearNodeAttributes();
|
||||||
|
if (nodeAttributes == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Iterable<NodeAttributeProto> iterable =
|
||||||
|
() -> new Iterator<NodeAttributeProto>() {
|
||||||
|
private Iterator<NodeAttribute> iter = nodeAttributes.iterator();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return iter.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeAttributeProto next() {
|
||||||
|
return convertToProtoFormat(iter.next());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
builder.addAllNodeAttributes(iterable);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) {
|
||||||
|
return new NodeAttributePBImpl(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||||
|
return ((NodeAttributePBImpl) t).getProto();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void maybeInitBuilder() {
|
||||||
|
if (viaProto || builder == null) {
|
||||||
|
builder = GetAttributesToNodesRequestProto.newBuilder(proto);
|
||||||
|
}
|
||||||
|
viaProto = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initNodeAttributes() {
|
||||||
|
if (this.nodeAttributes != null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
YarnServiceProtos.GetAttributesToNodesRequestProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
|
List<NodeAttributeProto> nodeAttributesList = p.getNodeAttributesList();
|
||||||
|
this.nodeAttributes = new HashSet<>();
|
||||||
|
nodeAttributesList
|
||||||
|
.forEach((v) -> nodeAttributes.add(convertFromProtoFormat(v)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return getProto().hashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return TextFormat.shortDebugString(getProto());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setNodeAttributes(Set<NodeAttribute> attributes) {
|
||||||
|
maybeInitBuilder();
|
||||||
|
if (nodeAttributes == null) {
|
||||||
|
builder.clearNodeAttributes();
|
||||||
|
}
|
||||||
|
this.nodeAttributes = attributes;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<NodeAttribute> getNodeAttributes() {
|
||||||
|
initNodeAttributes();
|
||||||
|
return this.nodeAttributes;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,184 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attributes to nodes response.
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class GetAttributesToNodesResponsePBImpl
|
||||||
|
extends GetAttributesToNodesResponse {
|
||||||
|
|
||||||
|
private GetAttributesToNodesResponseProto proto =
|
||||||
|
GetAttributesToNodesResponseProto.getDefaultInstance();
|
||||||
|
private GetAttributesToNodesResponseProto.Builder builder = null;
|
||||||
|
private boolean viaProto = false;
|
||||||
|
|
||||||
|
private Map<NodeAttribute, Set<String>> attributesToNodes;
|
||||||
|
|
||||||
|
public GetAttributesToNodesResponsePBImpl() {
|
||||||
|
this.builder = GetAttributesToNodesResponseProto.newBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetAttributesToNodesResponsePBImpl(
|
||||||
|
GetAttributesToNodesResponseProto proto) {
|
||||||
|
this.proto = proto;
|
||||||
|
this.viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initAttributesToNodes() {
|
||||||
|
if (this.attributesToNodes != null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
YarnServiceProtos.GetAttributesToNodesResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
|
List<AttributeToNodesProto> list = p.getAttributeToNodesList();
|
||||||
|
this.attributesToNodes = new HashMap<>();
|
||||||
|
|
||||||
|
for (AttributeToNodesProto c : list) {
|
||||||
|
Set<String> setNodes = new HashSet<>(c.getHostnamesList());
|
||||||
|
if (!setNodes.isEmpty()) {
|
||||||
|
this.attributesToNodes
|
||||||
|
.put(convertFromProtoFormat(c.getNodeAttribute()), setNodes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void maybeInitBuilder() {
|
||||||
|
if (viaProto || builder == null) {
|
||||||
|
builder = GetAttributesToNodesResponseProto.newBuilder(proto);
|
||||||
|
}
|
||||||
|
viaProto = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addAttributesToNodesToProto() {
|
||||||
|
maybeInitBuilder();
|
||||||
|
builder.clearAttributeToNodes();
|
||||||
|
if (attributesToNodes == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Iterable<AttributeToNodesProto> iterable =
|
||||||
|
() -> new Iterator<AttributeToNodesProto>() {
|
||||||
|
|
||||||
|
private Iterator<Map.Entry<NodeAttribute, Set<String>>> iter =
|
||||||
|
attributesToNodes.entrySet().iterator();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AttributeToNodesProto next() {
|
||||||
|
Map.Entry<NodeAttribute, Set<String>> now = iter.next();
|
||||||
|
Set<String> hostNames = new HashSet<>();
|
||||||
|
for (String host : now.getValue()) {
|
||||||
|
hostNames.add(host);
|
||||||
|
}
|
||||||
|
return AttributeToNodesProto.newBuilder()
|
||||||
|
.setNodeAttribute(convertToProtoFormat(now.getKey()))
|
||||||
|
.addAllHostnames(hostNames).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return iter.hasNext();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
builder.addAllAttributeToNodes(iterable);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) {
|
||||||
|
return new NodeAttributePBImpl(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||||
|
return ((NodeAttributePBImpl) t).getProto();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToBuilder() {
|
||||||
|
if (this.attributesToNodes != null) {
|
||||||
|
addAttributesToNodesToProto();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToProto() {
|
||||||
|
if (viaProto) {
|
||||||
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
|
mergeLocalToBuilder();
|
||||||
|
proto = builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetAttributesToNodesResponseProto getProto() {
|
||||||
|
mergeLocalToProto();
|
||||||
|
proto = viaProto ? proto : builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
return proto;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
assert false : "hashCode not designed";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setAttributeToNodes(Map<NodeAttribute, Set<String>> map) {
|
||||||
|
initAttributesToNodes();
|
||||||
|
attributesToNodes.clear();
|
||||||
|
attributesToNodes.putAll(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<NodeAttribute, Set<String>> getAttributesToNodes() {
|
||||||
|
initAttributesToNodes();
|
||||||
|
return this.attributesToNodes;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,75 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||||
|
|
||||||
|
import com.google.protobuf.TextFormat;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesRequestProto;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Request to get cluster node attributes.
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class GetClusterNodeAttributesRequestPBImpl
|
||||||
|
extends GetClusterNodeAttributesRequest {
|
||||||
|
|
||||||
|
private GetClusterNodeAttributesRequestProto proto =
|
||||||
|
GetClusterNodeAttributesRequestProto.getDefaultInstance();
|
||||||
|
private GetClusterNodeAttributesRequestProto.Builder builder = null;
|
||||||
|
private boolean viaProto = false;
|
||||||
|
|
||||||
|
public GetClusterNodeAttributesRequestPBImpl() {
|
||||||
|
builder = GetClusterNodeAttributesRequestProto.newBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetClusterNodeAttributesRequestPBImpl(
|
||||||
|
GetClusterNodeAttributesRequestProto proto) {
|
||||||
|
this.proto = proto;
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetClusterNodeAttributesRequestProto getProto() {
|
||||||
|
proto = viaProto ? proto : builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
return proto;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return getProto().hashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return TextFormat.shortDebugString(getProto());
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,156 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.classification.InterfaceAudience.*;
|
||||||
|
import static org.apache.hadoop.classification.InterfaceStability.*;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cluster node attributes response.
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
@Unstable
|
||||||
|
public class GetClusterNodeAttributesResponsePBImpl
|
||||||
|
extends GetClusterNodeAttributesResponse {
|
||||||
|
|
||||||
|
private GetClusterNodeAttributesResponseProto proto =
|
||||||
|
GetClusterNodeAttributesResponseProto.getDefaultInstance();
|
||||||
|
private GetClusterNodeAttributesResponseProto.Builder builder = null;
|
||||||
|
private Set<NodeAttribute> updatedNodeAttributes;
|
||||||
|
private boolean viaProto = false;
|
||||||
|
|
||||||
|
public GetClusterNodeAttributesResponsePBImpl() {
|
||||||
|
builder = GetClusterNodeAttributesResponseProto.newBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public GetClusterNodeAttributesResponsePBImpl(
|
||||||
|
GetClusterNodeAttributesResponseProto proto) {
|
||||||
|
this.proto = proto;
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized GetClusterNodeAttributesResponseProto getProto() {
|
||||||
|
mergeLocalToProto();
|
||||||
|
proto = viaProto ? proto : builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
return proto;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToProto() {
|
||||||
|
if (viaProto) {
|
||||||
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
|
mergeLocalToBuilder();
|
||||||
|
proto = builder.build();
|
||||||
|
viaProto = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeLocalToBuilder() {
|
||||||
|
if (this.updatedNodeAttributes != null) {
|
||||||
|
addNodeAttributesToProto();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addNodeAttributesToProto() {
|
||||||
|
maybeInitBuilder();
|
||||||
|
builder.clearNodeAttributes();
|
||||||
|
List<NodeAttributeProto> protoList = new ArrayList<>();
|
||||||
|
for (NodeAttribute r : this.updatedNodeAttributes) {
|
||||||
|
protoList.add(convertToProtoFormat(r));
|
||||||
|
}
|
||||||
|
builder.addAllNodeAttributes(protoList);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object other) {
|
||||||
|
if (other == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
assert false : "hashCode not designed";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void maybeInitBuilder() {
|
||||||
|
if (viaProto || builder == null) {
|
||||||
|
builder = GetClusterNodeAttributesResponseProto.newBuilder(proto);
|
||||||
|
}
|
||||||
|
viaProto = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized void setNodeAttributes(Set<NodeAttribute> attributes) {
|
||||||
|
maybeInitBuilder();
|
||||||
|
this.updatedNodeAttributes = new HashSet<>();
|
||||||
|
if (attributes == null) {
|
||||||
|
builder.clearNodeAttributes();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.updatedNodeAttributes.addAll(attributes);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized Set<NodeAttribute> getNodeAttributes() {
|
||||||
|
if (this.updatedNodeAttributes != null) {
|
||||||
|
return this.updatedNodeAttributes;
|
||||||
|
}
|
||||||
|
initLocalNodeAttributes();
|
||||||
|
return this.updatedNodeAttributes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initLocalNodeAttributes() {
|
||||||
|
YarnServiceProtos.GetClusterNodeAttributesResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
|
List<NodeAttributeProto> attributesProtoList = p.getNodeAttributesList();
|
||||||
|
this.updatedNodeAttributes = new HashSet<>();
|
||||||
|
for (NodeAttributeProto r : attributesProtoList) {
|
||||||
|
this.updatedNodeAttributes.add(convertFromProtoFormat(r));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttribute convertFromProtoFormat(NodeAttributeProto p) {
|
||||||
|
return new NodeAttributePBImpl(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodeAttributeProto convertToProtoFormat(NodeAttribute t) {
|
||||||
|
return ((NodeAttributePBImpl) t).getProto();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return getProto().toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -88,12 +88,13 @@ public abstract class NodeAttributesManager extends AbstractService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Given a attribute set, return what all Nodes have attribute mapped to it.
|
* Given a attribute set, return what all Nodes have attribute mapped to it.
|
||||||
|
* If the attributes set is null or empty, all attributes mapping are
|
||||||
|
* returned.
|
||||||
*
|
*
|
||||||
* @return a Map, of attribute to set of hostnames
|
* @return a Map of attributes to set of hostnames.
|
||||||
*/
|
*/
|
||||||
//TODO need to handle as part of REST patch.
|
public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||||
/* public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
Set<NodeAttribute> attributes);
|
||||||
Set<NodeAttribute> attributes);*/
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* NodeAttribute to AttributeValue Map.
|
* NodeAttribute to AttributeValue Map.
|
||||||
|
|
|
@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl;
|
||||||
|
@ -1259,4 +1263,28 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
|
||||||
validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
|
validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
|
||||||
NodesToAttributesMappingRequestProto.class);
|
NodesToAttributesMappingRequestProto.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetAttributesToNodesRequestPBImpl() throws Exception {
|
||||||
|
validatePBImplRecord(GetAttributesToNodesRequestPBImpl.class,
|
||||||
|
YarnServiceProtos.GetAttributesToNodesRequestProto.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetAttributesToNodesResponsePBImpl() throws Exception {
|
||||||
|
validatePBImplRecord(GetAttributesToNodesResponsePBImpl.class,
|
||||||
|
YarnServiceProtos.GetAttributesToNodesResponseProto.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetClusterNodeAttributesRequestPBImpl() throws Exception {
|
||||||
|
validatePBImplRecord(GetClusterNodeAttributesRequestPBImpl.class,
|
||||||
|
YarnServiceProtos.GetClusterNodeAttributesRequestProto.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetClusterNodeAttributesResponsePBImpl() throws Exception {
|
||||||
|
validatePBImplRecord(GetClusterNodeAttributesResponsePBImpl.class,
|
||||||
|
YarnServiceProtos.GetClusterNodeAttributesResponseProto.class);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -895,6 +899,19 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request)
|
public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
|
|
|
@ -66,8 +66,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -127,6 +131,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
|
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeReport;
|
import org.apache.hadoop.yarn.api.records.NodeReport;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
import org.apache.hadoop.yarn.api.records.Priority;
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
|
@ -148,6 +153,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
|
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
||||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
|
||||||
|
@ -1838,6 +1844,30 @@ public class ClientRMService extends AbstractService implements
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
NodeAttributesManager attributesManager =
|
||||||
|
rmContext.getNodeAttributesManager();
|
||||||
|
GetAttributesToNodesResponse response = GetAttributesToNodesResponse
|
||||||
|
.newInstance(attributesManager
|
||||||
|
.getAttributesToNodes(request.getNodeAttributes()));
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
NodeAttributesManager attributesManager =
|
||||||
|
rmContext.getNodeAttributesManager();
|
||||||
|
Set<NodeAttribute> attributes =
|
||||||
|
attributesManager.getClusterNodeAttributes(null);
|
||||||
|
GetClusterNodeAttributesResponse response =
|
||||||
|
GetClusterNodeAttributesResponse.newInstance(attributes);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public void setDisplayPerUserApps(boolean displayPerUserApps) {
|
public void setDisplayPerUserApps(boolean displayPerUserApps) {
|
||||||
this.filterAppsByUser = displayPerUserApps;
|
this.filterAppsByUser = displayPerUserApps;
|
||||||
|
|
|
@ -36,7 +36,7 @@ import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.google.common.base.Strings;
|
import com.google.common.base.Strings;
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
@ -367,19 +367,26 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
|
||||||
return attributes;
|
return attributes;
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO need to handle as part of REST patch.
|
@Override
|
||||||
/*
|
public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
||||||
* @Override public Map<NodeAttribute, Set<String>> getAttributesToNodes(
|
Set<NodeAttribute> attributes) {
|
||||||
* Set<NodeAttribute> attributes) { try { readLock.lock(); boolean
|
try {
|
||||||
* fetchAllAttributes = (attributes == null || attributes.isEmpty());
|
readLock.lock();
|
||||||
* Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); for
|
boolean fetchAllAttributes = (attributes == null || attributes.isEmpty());
|
||||||
* (Entry<NodeAttribute, RMAttributeNodeLabel> attributeEntry :
|
Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>();
|
||||||
* attributeCollections .entrySet()) { if (fetchAllAttributes ||
|
for (Entry<NodeAttribute, RMNodeAttribute> attributeEntry :
|
||||||
* attributes.contains(attributeEntry.getKey())) {
|
clusterAttributes.entrySet()) {
|
||||||
* attributesToNodes.put(attributeEntry.getKey(),
|
if (fetchAllAttributes || attributes
|
||||||
* attributeEntry.getValue().getAssociatedNodeIds()); } } return
|
.contains(attributeEntry.getKey())) {
|
||||||
* attributesToNodes; } finally { readLock.unlock(); } }
|
attributesToNodes.put(attributeEntry.getKey(),
|
||||||
*/
|
attributeEntry.getValue().getAssociatedNodeIds());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return attributesToNodes;
|
||||||
|
} finally {
|
||||||
|
readLock.unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public Resource getResourceByAttribute(NodeAttribute attribute) {
|
public Resource getResourceByAttribute(NodeAttribute attribute) {
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -18,6 +18,13 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager;
|
package org.apache.hadoop.yarn.server.resourcemanager;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
|
||||||
|
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.mockito.Matchers.any;
|
import static org.mockito.Matchers.any;
|
||||||
|
@ -1999,6 +2006,125 @@ public class TestClientRMService {
|
||||||
rm.close();
|
rm.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 120000)
|
||||||
|
public void testGetClusterNodeAttributes() throws IOException, YarnException {
|
||||||
|
MockRM rm = new MockRM() {
|
||||||
|
protected ClientRMService createClientRMService() {
|
||||||
|
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||||
|
this.applicationACLsManager, this.queueACLsManager,
|
||||||
|
this.getRMContext().getRMDelegationTokenSecretManager());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
rm.start();
|
||||||
|
|
||||||
|
NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager();
|
||||||
|
NodeId host1 = NodeId.newInstance("host1", 0);
|
||||||
|
NodeId host2 = NodeId.newInstance("host2", 0);
|
||||||
|
NodeAttribute gpu = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
|
||||||
|
NodeAttributeType.STRING, "nvida");
|
||||||
|
NodeAttribute os = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
|
||||||
|
NodeAttributeType.STRING, "windows64");
|
||||||
|
NodeAttribute docker = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
|
||||||
|
NodeAttributeType.STRING, "docker0");
|
||||||
|
Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
|
||||||
|
nodes.put(host1.getHost(), ImmutableSet.of(gpu, os));
|
||||||
|
nodes.put(host2.getHost(), ImmutableSet.of(docker));
|
||||||
|
mgr.addNodeAttributes(nodes);
|
||||||
|
// Create a client.
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
YarnRPC rpc = YarnRPC.create(conf);
|
||||||
|
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
|
||||||
|
LOG.info("Connecting to ResourceManager at " + rmAddress);
|
||||||
|
ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
|
||||||
|
.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
|
||||||
|
|
||||||
|
GetClusterNodeAttributesRequest request =
|
||||||
|
GetClusterNodeAttributesRequest.newInstance();
|
||||||
|
GetClusterNodeAttributesResponse response =
|
||||||
|
client.getClusterNodeAttributes(request);
|
||||||
|
Set<NodeAttribute> attributes = response.getNodeAttributes();
|
||||||
|
Assert.assertEquals("Size not correct", 3, attributes.size());
|
||||||
|
Assert.assertTrue(attributes.contains(gpu));
|
||||||
|
Assert.assertTrue(attributes.contains(os));
|
||||||
|
Assert.assertTrue(attributes.contains(docker));
|
||||||
|
rpc.stopProxy(client, conf);
|
||||||
|
rm.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 120000)
|
||||||
|
public void testGetAttributesToNodes() throws IOException, YarnException {
|
||||||
|
MockRM rm = new MockRM() {
|
||||||
|
protected ClientRMService createClientRMService() {
|
||||||
|
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||||
|
this.applicationACLsManager, this.queueACLsManager,
|
||||||
|
this.getRMContext().getRMDelegationTokenSecretManager());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
rm.start();
|
||||||
|
|
||||||
|
NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager();
|
||||||
|
String node1 = "host1";
|
||||||
|
String node2 = "host2";
|
||||||
|
NodeAttribute gpu = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
|
||||||
|
NodeAttributeType.STRING, "nvida");
|
||||||
|
NodeAttribute os = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
|
||||||
|
NodeAttributeType.STRING, "windows64");
|
||||||
|
NodeAttribute docker = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
|
||||||
|
NodeAttributeType.STRING, "docker0");
|
||||||
|
NodeAttribute dist = NodeAttribute
|
||||||
|
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
|
||||||
|
NodeAttributeType.STRING, "3_0_2");
|
||||||
|
Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
|
||||||
|
nodes.put(node1, ImmutableSet.of(gpu, os, dist));
|
||||||
|
nodes.put(node2, ImmutableSet.of(docker, dist));
|
||||||
|
mgr.addNodeAttributes(nodes);
|
||||||
|
// Create a client.
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
YarnRPC rpc = YarnRPC.create(conf);
|
||||||
|
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
|
||||||
|
LOG.info("Connecting to ResourceManager at " + rmAddress);
|
||||||
|
ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
|
||||||
|
.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
|
||||||
|
|
||||||
|
GetAttributesToNodesRequest request =
|
||||||
|
GetAttributesToNodesRequest.newInstance();
|
||||||
|
GetAttributesToNodesResponse response =
|
||||||
|
client.getAttributesToNodes(request);
|
||||||
|
Map<NodeAttribute, Set<String>> attrs = response.getAttributesToNodes();
|
||||||
|
Assert.assertEquals(response.getAttributesToNodes().size(), 4);
|
||||||
|
Assert.assertEquals(attrs.get(dist).size(), 2);
|
||||||
|
Assert.assertEquals(attrs.get(os).size(), 1);
|
||||||
|
Assert.assertEquals(attrs.get(gpu).size(), 1);
|
||||||
|
Assert.assertTrue(attrs.get(dist).contains(node1));
|
||||||
|
Assert.assertTrue(attrs.get(dist).contains(node2));
|
||||||
|
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||||
|
|
||||||
|
GetAttributesToNodesRequest request2 =
|
||||||
|
GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker));
|
||||||
|
GetAttributesToNodesResponse response2 =
|
||||||
|
client.getAttributesToNodes(request2);
|
||||||
|
Map<NodeAttribute, Set<String>> attrs2 = response2.getAttributesToNodes();
|
||||||
|
Assert.assertEquals(response2.getAttributesToNodes().size(), 1);
|
||||||
|
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||||
|
|
||||||
|
GetAttributesToNodesRequest request3 =
|
||||||
|
GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os));
|
||||||
|
GetAttributesToNodesResponse response3 =
|
||||||
|
client.getAttributesToNodes(request3);
|
||||||
|
Map<NodeAttribute, Set<String>> attrs3 = response3.getAttributesToNodes();
|
||||||
|
Assert.assertEquals(response3.getAttributesToNodes().size(), 2);
|
||||||
|
Assert.assertTrue(attrs.get(os).contains(node1));
|
||||||
|
Assert.assertTrue(attrs.get(docker).contains(node2));
|
||||||
|
rpc.stopProxy(client, conf);
|
||||||
|
rm.close();
|
||||||
|
}
|
||||||
|
|
||||||
@Test(timeout = 120000)
|
@Test(timeout = 120000)
|
||||||
public void testUpdatePriorityAndKillAppWithZeroClusterResource()
|
public void testUpdatePriorityAndKillAppWithZeroClusterResource()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
|
@ -39,8 +39,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -327,6 +331,19 @@ public class DefaultClientRequestInterceptor
|
||||||
return clientRMProxy.getResourceTypeInfo(request);
|
return clientRMProxy.getResourceTypeInfo(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
return clientRMProxy.getAttributesToNodes(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return clientRMProxy.getClusterNodeAttributes(request);
|
||||||
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public void setRMClient(ApplicationClientProtocol clientRM) {
|
public void setRMClient(ApplicationClientProtocol clientRM) {
|
||||||
this.clientRMProxy = clientRM;
|
this.clientRMProxy = clientRM;
|
||||||
|
|
|
@ -55,8 +55,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -826,4 +830,17 @@ public class FederationClientInterceptor
|
||||||
executorService.shutdown();
|
executorService.shutdown();
|
||||||
super.shutdown();
|
super.shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
throw new NotImplementedException("Code is not implemented");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
throw new NotImplementedException("Code is not implemented");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -50,8 +50,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -430,6 +434,21 @@ public class RouterClientRMService extends AbstractService
|
||||||
return pipeline.getRootInterceptor().getResourceTypeInfo(request);
|
return pipeline.getRootInterceptor().getResourceTypeInfo(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
|
||||||
|
return pipeline.getRootInterceptor().getAttributesToNodes(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
|
||||||
|
return pipeline.getRootInterceptor().getClusterNodeAttributes(request);
|
||||||
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected RequestInterceptorChainWrapper getInterceptorChain()
|
protected RequestInterceptorChainWrapper getInterceptorChain()
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -36,8 +36,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
|
||||||
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
|
||||||
|
@ -288,4 +292,17 @@ public class PassThroughClientRequestInterceptor
|
||||||
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
|
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
|
||||||
return getNextInterceptor().getResourceTypeInfo(request);
|
return getNextInterceptor().getResourceTypeInfo(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetAttributesToNodesResponse getAttributesToNodes(
|
||||||
|
GetAttributesToNodesRequest request) throws YarnException, IOException {
|
||||||
|
return getNextInterceptor().getAttributesToNodes(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetClusterNodeAttributesResponse getClusterNodeAttributes(
|
||||||
|
GetClusterNodeAttributesRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
|
return getNextInterceptor().getClusterNodeAttributes(request);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue