From f6bfe986394ba6ff8daf164d6da7da7cd57478df Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 31 Jan 2014 23:08:23 +0000 Subject: [PATCH 01/54] HDFS-5859. DataNode#checkBlockToken should check block tokens even if security is not enabled (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563328 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6ae103f7c42..490dc9beced 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -303,6 +303,9 @@ Release 2.4.0 - UNRELEASED HDFS-5804. HDFS NFS Gateway fails to mount and proxy when using Kerberos. (Abin Shahab via jing9) + HDFS-5859. DataNode#checkBlockToken should check block tokens even if + security is not enabled. (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 6bf98b69465..1dabd4a94c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1194,7 +1194,7 @@ public class DataNode extends Configured private void checkBlockToken(ExtendedBlock block, Token token, AccessMode accessMode) throws IOException { - if (isBlockTokenEnabled && UserGroupInformation.isSecurityEnabled()) { + if (isBlockTokenEnabled) { BlockTokenIdentifier id = new BlockTokenIdentifier(); ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); From 6e8c1bf20077f2bc5676604af5fa63a27f28dd86 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 1 Feb 2014 00:09:53 +0000 Subject: [PATCH 02/54] YARN-1633. Defined user-facing entity, entity-info and event objects related to Application Timeline feature. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563356 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/apptimeline/ATSEntities.java | 88 +++++ .../api/records/apptimeline/ATSEntity.java | 314 ++++++++++++++++++ .../api/records/apptimeline/ATSEvent.java | 134 ++++++++ .../api/records/apptimeline/ATSEvents.java | 189 +++++++++++ .../api/records/apptimeline/package-info.java | 21 ++ .../TestApplicationTimelineRecords.java | 113 +++++++ 7 files changed, 862 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntities.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/package-info.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 914966a28d4..abc56d6961a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -77,6 +77,9 @@ Release 2.4.0 - UNRELEASED YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory server. (Mayank Bansal via vinodkv) + YARN-1633. Defined user-facing entity, entity-info and event objects related + to Application Timeline feature. (Zhijie Shen via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntities.java new file mode 100644 index 00000000000..ed02cac43d9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntities.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.apptimeline; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The class that hosts a list of application timeline entities. + */ +@XmlRootElement(name = "entities") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSEntities { + + private List entities = + new ArrayList(); + + public ATSEntities() { + + } + + /** + * Get a list of entities + * + * @return a list of entities + */ + @XmlElement(name = "entities") + public List getEntities() { + return entities; + } + + /** + * Add a single entity into the existing entity list + * + * @param entity + * a single entity + */ + public void addEntity(ATSEntity entity) { + entities.add(entity); + } + + /** + * All a list of entities into the existing entity list + * + * @param entities + * a list of entities + */ + public void addEntities(List entities) { + this.entities.addAll(entities); + } + + /** + * Set the entity list to the given list of entities + * + * @param entities + * a list of entities + */ + public void setEntities(List entities) { + this.entities = entities; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java new file mode 100644 index 00000000000..1884db7ac13 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java @@ -0,0 +1,314 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.apptimeline; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + *

+ * The class that contains the the meta information of some conceptual entity of + * an application and its related events. The entity can be an application, an + * application attempt, a container or whatever the user-defined object. + *

+ * + *

+ * Primary filters will be used to index the entities in + * ApplicationTimelineStore, such that users should carefully + * choose the information they want to store as the primary filters. The + * remaining can be stored as other information. + *

+ */ +@XmlRootElement(name = "entity") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSEntity { + + private String entityType; + private String entityId; + private long startTime; + private List events = new ArrayList(); + private Map> relatedEntities = + new HashMap>(); + private Map primaryFilters = + new HashMap(); + private Map otherInfo = + new HashMap(); + + public ATSEntity() { + + } + + /** + * Get the entity type + * + * @return the entity type + */ + @XmlElement(name = "entitytype") + public String getEntityType() { + return entityType; + } + + /** + * Set the entity type + * + * @param entityType + * the entity type + */ + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + /** + * Get the entity Id + * + * @return the entity Id + */ + @XmlElement(name = "entity") + public String getEntityId() { + return entityId; + } + + /** + * Set the entity Id + * + * @param entityId + * the entity Id + */ + public void setEntityId(String entityId) { + this.entityId = entityId; + } + + /** + * Get the start time of the entity + * + * @return the start time of the entity + */ + @XmlElement(name = "starttime") + public long getStartTime() { + return startTime; + } + + /** + * Set the start time of the entity + * + * @param startTime + * the start time of the entity + */ + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + /** + * Get a list of events related to the entity + * + * @return a list of events related to the entity + */ + @XmlElement(name = "events") + public List getEvents() { + return events; + } + + /** + * Add a single event related to the entity to the existing event list + * + * @param event + * a single event related to the entity + */ + public void addEvent(ATSEvent event) { + events.add(event); + } + + /** + * Add a list of events related to the entity to the existing event list + * + * @param events + * a list of events related to the entity + */ + public void addEvents(List events) { + this.events.addAll(events); + } + + /** + * Set the event list to the given list of events related to the entity + * + * @param events + * events a list of events related to the entity + */ + public void setEvents(List events) { + this.events = events; + } + + /** + * Get the related entities + * + * @return the related entities + */ + @XmlElement(name = "relatedentities") + public Map> getRelatedEntities() { + return relatedEntities; + } + + /** + * Add a list of entity of the same type to the existing related entity map + * + * @param entityType + * the entity type + * @param entityIds + * a list of entity Ids + */ + public void addRelatedEntity(String entityType, List entityIds) { + List thisRelatedEntity = relatedEntities.get(entityType); + relatedEntities.put(entityType, entityIds); + if (thisRelatedEntity == null) { + relatedEntities.put(entityType, entityIds); + } else { + thisRelatedEntity.addAll(entityIds); + } + } + + /** + * Add a map of related entities to the existing related entity map + * + * @param relatedEntities + * a map of related entities + */ + public void addRelatedEntities( + Map> relatedEntities) { + for (Map.Entry> relatedEntity : relatedEntities + .entrySet()) { + List thisRelatedEntity = + this.relatedEntities.get(relatedEntity.getKey()); + if (thisRelatedEntity == null) { + this.relatedEntities.put( + relatedEntity.getKey(), relatedEntity.getValue()); + } else { + thisRelatedEntity.addAll(relatedEntity.getValue()); + } + } + } + + /** + * Set the related entity map to the given map of related entities + * + * @param relatedEntities + * a map of related entities + */ + public void setRelatedEntities( + Map> relatedEntities) { + this.relatedEntities = relatedEntities; + } + + /** + * Get the primary filters + * + * @return the primary filters + */ + @XmlElement(name = "primaryfilters") + public Map getPrimaryFilters() { + return primaryFilters; + } + + /** + * Add a single piece of primary filter to the existing primary filter map + * + * @param key + * the primary filter key + * @param value + * the primary filter value + */ + public void addPrimaryFilter(String key, Object value) { + primaryFilters.put(key, value); + } + + /** + * Add a map of primary filters to the existing primary filter map + * + * @param primaryFilters + * a map of primary filters + */ + public void addPrimaryFilters(Map primaryFilters) { + this.primaryFilters.putAll(primaryFilters); + } + + /** + * Set the primary filter map to the given map of primary filters + * + * @param primaryFilters + * a map of primary filters + */ + public void setPrimaryFilters(Map primaryFilters) { + this.primaryFilters = primaryFilters; + } + + /** + * Get the other information of the entity + * + * @return the other information of the entity + */ + @XmlElement(name = "otherinfo") + public Map getOtherInfo() { + return otherInfo; + } + + /** + * Add one piece of other information of the entity to the existing other info + * map + * + * @param key + * the other information key + * @param value + * the other information value + */ + public void addOtherInfo(String key, Object value) { + this.otherInfo.put(key, value); + } + + /** + * Add a map of other information of the entity to the existing other info map + * + * @param otherInfo + * a map of other information + */ + public void addOtherInfo(Map otherInfo) { + this.otherInfo.putAll(otherInfo); + } + + /** + * Set the other info map to the given map of other information + * + * @param otherInfo + * a map of other information + */ + public void setOtherInfo(Map otherInfo) { + this.otherInfo = otherInfo; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java new file mode 100644 index 00000000000..6477a578e6a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.apptimeline; + +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The class that contains the information of an event that is related to some + * conceptual entity of an application. Users are free to define what the event + * means, such as starting an application, getting allocated a container and + * etc. + */ +@XmlRootElement(name = "event") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSEvent { + + private long timestamp; + private String eventType; + private Map eventInfo = new HashMap(); + + public ATSEvent() { + } + + /** + * Get the timestamp of the event + * + * @return the timestamp of the event + */ + @XmlElement(name = "timestamp") + public long getTimestamp() { + return timestamp; + } + + /** + * Set the timestamp of the event + * + * @param timestamp + * the timestamp of the event + */ + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + /** + * Get the event type + * + * @return the event type + */ + @XmlElement(name = "eventtype") + public String getEventType() { + return eventType; + } + + /** + * Set the event type + * + * @param eventType + * the event type + */ + public void setEventType(String eventType) { + this.eventType = eventType; + } + + /** + * Set the information of the event + * + * @return the information of the event + */ + @XmlElement(name = "eventinfo") + public Map getEventInfo() { + return eventInfo; + } + + /** + * Add one piece of the information of the event to the existing information + * map + * + * @param key + * the information key + * @param value + * the information value + */ + public void addEventInfo(String key, Object value) { + this.eventInfo.put(key, value); + } + + /** + * Add a map of the information of the event to the existing information map + * + * @param eventInfo + * a map of of the information of the event + */ + public void addEventInfo(Map eventInfo) { + this.eventInfo.putAll(eventInfo); + } + + /** + * Set the information map to the given map of the information of the event + * + * @param eventInfo + * a map of of the information of the event + */ + public void setEventInfo(Map eventInfo) { + this.eventInfo = eventInfo; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java new file mode 100644 index 00000000000..da7fd280886 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.apptimeline; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The class that hosts a list of events, which are categorized according to + * their related entities. + */ +@XmlRootElement(name = "events") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSEvents { + + private List allEvents = + new ArrayList(); + + public ATSEvents() { + + } + + /** + * Get a list of {@link ATSEventsOfOneEntity} instances + * + * @return a list of {@link ATSEventsOfOneEntity} instances + */ + @XmlElement(name = "events") + public List getAllEvents() { + return allEvents; + } + + /** + * Add a single {@link ATSEventsOfOneEntity} instance into the existing list + * + * @param eventsOfOneEntity + * a single {@link ATSEventsOfOneEntity} instance + */ + public void addEvent(ATSEventsOfOneEntity eventsOfOneEntity) { + allEvents.add(eventsOfOneEntity); + } + + /** + * Add a list of {@link ATSEventsOfOneEntity} instances into the existing list + * + * @param allEvents + * a list of {@link ATSEventsOfOneEntity} instances + */ + public void addEvents(List allEvents) { + this.allEvents.addAll(allEvents); + } + + /** + * Set the list to the given list of {@link ATSEventsOfOneEntity} instances + * + * @param allEvents + * a list of {@link ATSEventsOfOneEntity} instances + */ + public void setEvents(List allEvents) { + this.allEvents.clear(); + this.allEvents.addAll(allEvents); + } + + /** + * The class that hosts a list of events that are only related to one entity. + */ + @XmlRootElement(name = "events") + @XmlAccessorType(XmlAccessType.NONE) + @Public + @Unstable + public static class ATSEventsOfOneEntity { + + private String entityId; + private String entityType; + private List events = new ArrayList(); + + public ATSEventsOfOneEntity() { + + } + + /** + * Get the entity Id + * + * @return the entity Id + */ + @XmlElement(name = "entity") + public String getEntityId() { + return entityId; + } + + /** + * Set the entity Id + * + * @param entityId + * the entity Id + */ + public void setEntityId(String entityId) { + this.entityId = entityId; + } + + /** + * Get the entity type + * + * @return the entity type + */ + @XmlElement(name = "entitytype") + public String getEntityType() { + return entityType; + } + + /** + * Set the entity type + * + * @param entityType + * the entity type + */ + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + /** + * Get a list of events + * + * @return a list of events + */ + @XmlElement(name = "events") + public List getEvents() { + return events; + } + + /** + * Add a single event to the existing event list + * + * @param event + * a single event + */ + public void addEntity(ATSEvent event) { + events.add(event); + } + + /** + * Add a list of event to the existing event list + * + * @param events + * a list of events + */ + public void addEvents(List events) { + this.events.addAll(events); + } + + /** + * Set the event list to the given list of events + * + * @param events + * a list of events + */ + public void setEvents(List events) { + this.events = events; + } + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/package-info.java new file mode 100644 index 00000000000..b57cad4b204 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Public +package org.apache.hadoop.yarn.api.records.apptimeline; +import org.apache.hadoop.classification.InterfaceAudience; + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java new file mode 100644 index 00000000000..fe79e74eb6c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.apptimeline; + +import java.util.Arrays; + +import junit.framework.Assert; + +import org.junit.Test; + +public class TestApplicationTimelineRecords { + + @Test + public void testATSEntities() { + ATSEntities entities = new ATSEntities(); + for (int j = 0; j < 2; ++j) { + ATSEntity entity = new ATSEntity(); + entity.setEntityId("entity id " + j); + entity.setEntityType("entity type " + j); + entity.setStartTime(System.currentTimeMillis()); + for (int i = 0; i < 2; ++i) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType("event type " + i); + event.addEventInfo("key1", "val1"); + event.addEventInfo("key2", "val2"); + entity.addEvent(event); + } + entity.addRelatedEntity( + "test ref type 1", Arrays.asList((Object) "test ref id 1")); + entity.addRelatedEntity( + "test ref type 2", Arrays.asList((Object) "test ref id 2")); + entity.addPrimaryFilter("pkey1", "pval1"); + entity.addPrimaryFilter("pkey2", "pval2"); + entity.addOtherInfo("okey1", "oval1"); + entity.addOtherInfo("okey2", "oval2"); + entities.addEntity(entity); + } + Assert.assertEquals(2, entities.getEntities().size()); + ATSEntity entity1 = entities.getEntities().get(0); + Assert.assertEquals("entity id 0", entity1.getEntityId()); + Assert.assertEquals("entity type 0", entity1.getEntityType()); + Assert.assertEquals(2, entity1.getRelatedEntities().size()); + Assert.assertEquals(2, entity1.getEvents().size()); + Assert.assertEquals(2, entity1.getPrimaryFilters().size()); + Assert.assertEquals(2, entity1.getOtherInfo().size()); + ATSEntity entity2 = entities.getEntities().get(1); + Assert.assertEquals("entity id 1", entity2.getEntityId()); + Assert.assertEquals("entity type 1", entity2.getEntityType()); + Assert.assertEquals(2, entity2.getRelatedEntities().size()); + Assert.assertEquals(2, entity2.getEvents().size()); + Assert.assertEquals(2, entity2.getPrimaryFilters().size()); + Assert.assertEquals(2, entity2.getOtherInfo().size()); + } + + @Test + public void testATSEvents() { + ATSEvents events = new ATSEvents(); + for (int j = 0; j < 2; ++j) { + ATSEvents.ATSEventsOfOneEntity partEvents = + new ATSEvents.ATSEventsOfOneEntity(); + partEvents.setEntityId("entity id " + j); + partEvents.setEntityType("entity type " + j); + for (int i = 0; i < 2; ++i) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType("event type " + i); + event.addEventInfo("key1", "val1"); + event.addEventInfo("key2", "val2"); + partEvents.addEntity(event); + } + events.addEvent(partEvents); + } + Assert.assertEquals(2, events.getAllEvents().size()); + ATSEvents.ATSEventsOfOneEntity partEvents1 = events.getAllEvents().get(0); + Assert.assertEquals("entity id 0", partEvents1.getEntityId()); + Assert.assertEquals("entity type 0", partEvents1.getEntityType()); + Assert.assertEquals(2, partEvents1.getEvents().size()); + ATSEvent event11 = partEvents1.getEvents().get(0); + Assert.assertEquals("event type 0", event11.getEventType()); + Assert.assertEquals(2, event11.getEventInfo().size()); + ATSEvent event12 = partEvents1.getEvents().get(1); + Assert.assertEquals("event type 1", event12.getEventType()); + Assert.assertEquals(2, event12.getEventInfo().size()); + ATSEvents.ATSEventsOfOneEntity partEvents2 = events.getAllEvents().get(1); + Assert.assertEquals("entity id 1", partEvents2.getEntityId()); + Assert.assertEquals("entity type 1", partEvents2.getEntityType()); + Assert.assertEquals(2, partEvents2.getEvents().size()); + ATSEvent event21 = partEvents2.getEvents().get(0); + Assert.assertEquals("event type 0", event21.getEventType()); + Assert.assertEquals(2, event21.getEventInfo().size()); + ATSEvent event22 = partEvents2.getEvents().get(1); + Assert.assertEquals("event type 1", event22.getEventType()); + Assert.assertEquals(2, event22.getEventInfo().size()); + } + +} From 140246bad816ae7bf06eda193cc624e1e62923e0 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Sat, 1 Feb 2014 02:25:33 +0000 Subject: [PATCH 03/54] HDFS-5746. Add ShortCircuitSharedMemorySegment (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563362 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 4 +- hadoop-common-project/hadoop-common/pom.xml | 2 + .../hadoop-common/src/CMakeLists.txt | 2 + .../apache/hadoop/io/nativeio/NativeIO.java | 10 + .../nativeio/SharedFileDescriptorFactory.java | 90 ++++ .../apache/hadoop/net/unix/DomainSocket.java | 184 ++----- .../hadoop/net/unix/DomainSocketWatcher.java | 478 ++++++++++++++++++ .../hadoop/util/CloseableReferenceCount.java | 125 +++++ .../org/apache/hadoop/io/nativeio/NativeIO.c | 38 ++ .../io/nativeio/SharedFileDescriptorFactory.c | 162 ++++++ .../hadoop/net/unix/DomainSocketWatcher.c | 247 +++++++++ .../TestSharedFileDescriptorFactory.java | 82 +++ .../net/unix/TestDomainSocketWatcher.java | 150 ++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../ShortCircuitSharedMemorySegment.java | 302 +++++++++++ .../TestShortCircuitSharedMemorySegment.java | 104 ++++ 16 files changed, 1846 insertions(+), 136 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitSharedMemorySegment.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitSharedMemorySegment.java diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 7143b514060..10fcdb785e7 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -425,9 +425,9 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." - #There are 12 warnings that are caused by things that are caused by using sun internal APIs. + #There are 14 warnings that are caused by things that are caused by using sun internal APIs. #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. - OK_JAVADOC_WARNINGS=14; + OK_JAVADOC_WARNINGS=16; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 366aa38e0a5..683e92b01ce 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -543,6 +543,7 @@ org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor org.apache.hadoop.security.JniBasedUnixGroupsMapping org.apache.hadoop.io.nativeio.NativeIO + org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping org.apache.hadoop.io.compress.snappy.SnappyCompressor org.apache.hadoop.io.compress.snappy.SnappyDecompressor @@ -550,6 +551,7 @@ org.apache.hadoop.io.compress.lz4.Lz4Decompressor org.apache.hadoop.util.NativeCrc32 org.apache.hadoop.net.unix.DomainSocket + org.apache.hadoop.net.unix.DomainSocketWatcher ${project.build.directory}/native/javah diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 4a8f8a1cf1b..dec63c45d7a 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -178,7 +178,9 @@ add_dual_library(hadoop ${D}/io/nativeio/NativeIO.c ${D}/io/nativeio/errno_enum.c ${D}/io/nativeio/file_descriptor.c + ${D}/io/nativeio/SharedFileDescriptorFactory.c ${D}/net/unix/DomainSocket.c + ${D}/net/unix/DomainSocketWatcher.c ${D}/security/JniBasedUnixGroupsMapping.c ${D}/security/JniBasedUnixGroupsNetgroupMapping.c ${D}/security/hadoop_group_info.c diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 7ea7e59151b..db8da35e6b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -487,6 +487,16 @@ public class NativeIO { new ConcurrentHashMap(); private enum IdCache { USER, GROUP } + + public final static int MMAP_PROT_READ = 0x1; + public final static int MMAP_PROT_WRITE = 0x2; + public final static int MMAP_PROT_EXEC = 0x4; + + public static native long mmap(FileDescriptor fd, int prot, + boolean shared, long length) throws IOException; + + public static native void munmap(long addr, long length) + throws IOException; } private static boolean workaroundNonThreadSafePasswdCalls = false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java new file mode 100644 index 00000000000..25a861277a5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.FileDescriptor; + +import org.apache.commons.lang.SystemUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import com.google.common.base.Preconditions; + +/** + * A factory for creating shared file descriptors inside a given directory. + * Typically, the directory will be /dev/shm or /tmp. + * + * We will hand out file descriptors that correspond to unlinked files residing + * in that directory. These file descriptors are suitable for sharing across + * multiple processes and are both readable and writable. + * + * Because we unlink the temporary files right after creating them, a JVM crash + * usually does not leave behind any temporary files in the directory. However, + * it may happen that we crash right after creating the file and before + * unlinking it. In the constructor, we attempt to clean up after any such + * remnants by trying to unlink any temporary files created by previous + * SharedFileDescriptorFactory instances that also used our prefix. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class SharedFileDescriptorFactory { + private final String prefix; + private final String path; + + /** + * Create a SharedFileDescriptorFactory. + * + * @param prefix Prefix to add to all file names we use. + * @param path Path to use. + */ + public SharedFileDescriptorFactory(String prefix, String path) + throws IOException { + Preconditions.checkArgument(NativeIO.isAvailable()); + Preconditions.checkArgument(SystemUtils.IS_OS_UNIX); + this.prefix = prefix; + this.path = path; + deleteStaleTemporaryFiles0(prefix, path); + } + + /** + * Create a shared file descriptor which will be both readable and writable. + * + * @param length The starting file length. + * + * @return The file descriptor, wrapped in a FileInputStream. + * @throws IOException If there was an I/O or configuration error creating + * the descriptor. + */ + public FileInputStream createDescriptor(int length) throws IOException { + return new FileInputStream(createDescriptor0(prefix, path, length)); + } + + /** + * Delete temporary files in the directory, NOT following symlinks. + */ + private static native void deleteStaleTemporaryFiles0(String prefix, + String path) throws IOException; + + /** + * Create a file with O_EXCL, and then resize it to the desired size. + */ + private static native FileDescriptor createDescriptor0(String prefix, + String path, int length) throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java index bdf4d67a1af..e0d6b63c73c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java @@ -24,17 +24,15 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.SocketException; -import java.nio.channels.AsynchronousCloseException; import java.nio.channels.ClosedChannelException; import java.nio.channels.ReadableByteChannel; import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.SystemUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.CloseableReferenceCount; import com.google.common.annotations.VisibleForTesting; @@ -132,104 +130,14 @@ public class DomainSocket implements Closeable { } /** - * Tracks the reference count of the file descriptor, and also whether it is - * open or closed. + * The socket reference count and closed bit. */ - private static class Status { - /** - * Bit mask representing a closed domain socket. - */ - private static final int STATUS_CLOSED_MASK = 1 << 30; - - /** - * Status bits - * - * Bit 30: 0 = DomainSocket open, 1 = DomainSocket closed - * Bits 29 to 0: the reference count. - */ - private final AtomicInteger bits = new AtomicInteger(0); - - Status() { } - - /** - * Increment the reference count of the underlying file descriptor. - * - * @throws ClosedChannelException If the file descriptor is closed. - */ - void reference() throws ClosedChannelException { - int curBits = bits.incrementAndGet(); - if ((curBits & STATUS_CLOSED_MASK) != 0) { - bits.decrementAndGet(); - throw new ClosedChannelException(); - } - } - - /** - * Decrement the reference count of the underlying file descriptor. - * - * @param checkClosed Whether to throw an exception if the file - * descriptor is closed. - * - * @throws AsynchronousCloseException If the file descriptor is closed and - * checkClosed is set. - */ - void unreference(boolean checkClosed) throws AsynchronousCloseException { - int newCount = bits.decrementAndGet(); - assert (newCount & ~STATUS_CLOSED_MASK) >= 0; - if (checkClosed && ((newCount & STATUS_CLOSED_MASK) != 0)) { - throw new AsynchronousCloseException(); - } - } - - /** - * Return true if the file descriptor is currently open. - * - * @return True if the file descriptor is currently open. - */ - boolean isOpen() { - return ((bits.get() & STATUS_CLOSED_MASK) == 0); - } - - /** - * Mark the file descriptor as closed. - * - * Once the file descriptor is closed, it cannot be reopened. - * - * @return The current reference count. - * @throws ClosedChannelException If someone else closes the file - * descriptor before we do. - */ - int setClosed() throws ClosedChannelException { - while (true) { - int curBits = bits.get(); - if ((curBits & STATUS_CLOSED_MASK) != 0) { - throw new ClosedChannelException(); - } - if (bits.compareAndSet(curBits, curBits | STATUS_CLOSED_MASK)) { - return curBits & (~STATUS_CLOSED_MASK); - } - } - } - - /** - * Get the current reference count. - * - * @return The current reference count. - */ - int getReferenceCount() { - return bits.get() & (~STATUS_CLOSED_MASK); - } - } - - /** - * The socket status. - */ - private final Status status; + final CloseableReferenceCount refCount; /** * The file descriptor associated with this UNIX domain socket. */ - private final int fd; + final int fd; /** * The path associated with this UNIX domain socket. @@ -252,13 +160,21 @@ public class DomainSocket implements Closeable { private final DomainChannel channel = new DomainChannel(); private DomainSocket(String path, int fd) { - this.status = new Status(); + this.refCount = new CloseableReferenceCount(); this.fd = fd; this.path = path; } private static native int bind0(String path) throws IOException; + private void unreference(boolean checkClosed) throws ClosedChannelException { + if (checkClosed) { + refCount.unreferenceCheckClosed(); + } else { + refCount.unreference(); + } + } + /** * Create a new DomainSocket listening on the given path. * @@ -308,14 +224,14 @@ public class DomainSocket implements Closeable { * @throws SocketTimeoutException If the accept timed out. */ public DomainSocket accept() throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { DomainSocket ret = new DomainSocket(path, accept0(fd)); exc = false; return ret; } finally { - status.unreference(exc); + unreference(exc); } } @@ -335,14 +251,14 @@ public class DomainSocket implements Closeable { return new DomainSocket(path, fd); } - /** - * Return true if the file descriptor is currently open. - * - * @return True if the file descriptor is currently open. - */ - public boolean isOpen() { - return status.isOpen(); - } + /** + * Return true if the file descriptor is currently open. + * + * @return True if the file descriptor is currently open. + */ + public boolean isOpen() { + return refCount.isOpen(); + } /** * @return The socket path. @@ -381,20 +297,20 @@ public class DomainSocket implements Closeable { throws IOException; public void setAttribute(int type, int size) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { setAttribute0(fd, type, size); exc = false; } finally { - status.unreference(exc); + unreference(exc); } } private native int getAttribute0(int fd, int type) throws IOException; public int getAttribute(int type) throws IOException { - status.reference(); + refCount.reference(); int attribute; boolean exc = true; try { @@ -402,7 +318,7 @@ public class DomainSocket implements Closeable { exc = false; return attribute; } finally { - status.unreference(exc); + unreference(exc); } } @@ -419,9 +335,9 @@ public class DomainSocket implements Closeable { @Override public void close() throws IOException { // Set the closed bit on this DomainSocket - int refCount; + int count; try { - refCount = status.setClosed(); + count = refCount.setClosed(); } catch (ClosedChannelException e) { // Someone else already closed the DomainSocket. return; @@ -429,7 +345,7 @@ public class DomainSocket implements Closeable { // Wait for all references to go away boolean didShutdown = false; boolean interrupted = false; - while (refCount > 0) { + while (count > 0) { if (!didShutdown) { try { // Calling shutdown on the socket will interrupt blocking system @@ -446,7 +362,7 @@ public class DomainSocket implements Closeable { } catch (InterruptedException e) { interrupted = true; } - refCount = status.getReferenceCount(); + count = refCount.getReferenceCount(); } // At this point, nobody has a reference to the file descriptor, @@ -478,13 +394,13 @@ public class DomainSocket implements Closeable { */ public void sendFileDescriptors(FileDescriptor descriptors[], byte jbuf[], int offset, int length) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { sendFileDescriptors0(fd, descriptors, jbuf, offset, length); exc = false; } finally { - status.unreference(exc); + unreference(exc); } } @@ -515,14 +431,14 @@ public class DomainSocket implements Closeable { */ public int receiveFileDescriptors(FileDescriptor[] descriptors, byte jbuf[], int offset, int length) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { int nBytes = receiveFileDescriptors0(fd, descriptors, jbuf, offset, length); exc = false; return nBytes; } finally { - status.unreference(exc); + unreference(exc); } } @@ -539,7 +455,7 @@ public class DomainSocket implements Closeable { for (int i = 0; i < streams.length; i++) { streams[i] = null; } - status.reference(); + refCount.reference(); try { int ret = receiveFileDescriptors0(fd, descriptors, buf, offset, length); for (int i = 0, j = 0; i < descriptors.length; i++) { @@ -569,7 +485,7 @@ public class DomainSocket implements Closeable { } } } - status.unreference(!success); + unreference(!success); } } @@ -593,7 +509,7 @@ public class DomainSocket implements Closeable { public class DomainInputStream extends InputStream { @Override public int read() throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { byte b[] = new byte[1]; @@ -601,33 +517,33 @@ public class DomainSocket implements Closeable { exc = false; return (ret >= 0) ? b[0] : -1; } finally { - status.unreference(exc); + unreference(exc); } } @Override public int read(byte b[], int off, int len) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { int nRead = DomainSocket.readArray0(DomainSocket.this.fd, b, off, len); exc = false; return nRead; } finally { - status.unreference(exc); + unreference(exc); } } @Override public int available() throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { int nAvailable = DomainSocket.available0(DomainSocket.this.fd); exc = false; return nAvailable; } finally { - status.unreference(exc); + unreference(exc); } } @@ -649,7 +565,7 @@ public class DomainSocket implements Closeable { @Override public void write(int val) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { byte b[] = new byte[1]; @@ -657,19 +573,19 @@ public class DomainSocket implements Closeable { DomainSocket.writeArray0(DomainSocket.this.fd, b, 0, 1); exc = false; } finally { - status.unreference(exc); + unreference(exc); } } @Override public void write(byte[] b, int off, int len) throws IOException { - status.reference(); - boolean exc = true; + refCount.reference(); + boolean exc = true; try { DomainSocket.writeArray0(DomainSocket.this.fd, b, off, len); exc = false; } finally { - status.unreference(exc); + unreference(exc); } } } @@ -688,7 +604,7 @@ public class DomainSocket implements Closeable { @Override public int read(ByteBuffer dst) throws IOException { - status.reference(); + refCount.reference(); boolean exc = true; try { int nread = 0; @@ -710,7 +626,7 @@ public class DomainSocket implements Closeable { exc = false; return nread; } finally { - status.unreference(exc); + unreference(exc); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java new file mode 100644 index 00000000000..673129dd988 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java @@ -0,0 +1,478 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net.unix; + +import java.io.Closeable; +import java.io.EOFException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.IOUtils; + +import java.io.IOException; +import java.nio.channels.ClosedChannelException; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.TreeMap; +import java.util.Map; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.lang.SystemUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.NativeCodeLoader; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.Uninterruptibles; + +/** + * The DomainSocketWatcher watches a set of domain sockets to see when they + * become readable, or closed. When one of those events happens, it makes a + * callback. + * + * See {@link DomainSocket} for more information about UNIX domain sockets. + */ +@InterfaceAudience.LimitedPrivate("HDFS") +public final class DomainSocketWatcher extends Thread implements Closeable { + static { + if (SystemUtils.IS_OS_WINDOWS) { + loadingFailureReason = "UNIX Domain sockets are not available on Windows."; + } else if (!NativeCodeLoader.isNativeCodeLoaded()) { + loadingFailureReason = "libhadoop cannot be loaded."; + } else { + String problem; + try { + anchorNative(); + problem = null; + } catch (Throwable t) { + problem = "DomainSocketWatcher#anchorNative got error: " + + t.getMessage(); + } + loadingFailureReason = problem; + } + } + + static Log LOG = LogFactory.getLog(DomainSocketWatcher.class); + + /** + * The reason why DomainSocketWatcher is not available, or null if it is + * available. + */ + private final static String loadingFailureReason; + + /** + * Initializes the native library code. + */ + private static native void anchorNative(); + + interface Handler { + /** + * Handles an event on a socket. An event may be the socket becoming + * readable, or the remote end being closed. + * + * @param sock The socket that the event occurred on. + * @return Whether we should close the socket. + */ + boolean handle(DomainSocket sock); + } + + /** + * Handler for {DomainSocketWatcher#notificationSockets[1]} + */ + private class NotificationHandler implements Handler { + public boolean handle(DomainSocket sock) { + try { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": NotificationHandler: doing a read on " + + sock.fd); + } + if (sock.getInputStream().read() == -1) { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": NotificationHandler: got EOF on " + sock.fd); + } + throw new EOFException(); + } + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": NotificationHandler: read succeeded on " + + sock.fd); + } + return false; + } catch (IOException e) { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": NotificationHandler: setting closed to " + + "true for " + sock.fd); + } + closed = true; + return true; + } + } + } + + private static class Entry { + final DomainSocket socket; + final Handler handler; + + Entry(DomainSocket socket, Handler handler) { + this.socket = socket; + this.handler = handler; + } + + DomainSocket getDomainSocket() { + return socket; + } + + Handler getHandler() { + return handler; + } + } + + /** + * The FdSet is a set of file descriptors that gets passed to poll(2). + * It contains a native memory segment, so that we don't have to copy + * in the poll0 function. + */ + private static class FdSet { + private long data; + + private native static long alloc0(); + + FdSet() { + data = alloc0(); + } + + /** + * Add a file descriptor to the set. + * + * @param fd The file descriptor to add. + */ + native void add(int fd); + + /** + * Remove a file descriptor from the set. + * + * @param fd The file descriptor to remove. + */ + native void remove(int fd); + + /** + * Get an array containing all the FDs marked as readable. + * Also clear the state of all FDs. + * + * @return An array containing all of the currently readable file + * descriptors. + */ + native int[] getAndClearReadableFds(); + + /** + * Close the object and de-allocate the memory used. + */ + native void close(); + } + + /** + * Lock which protects toAdd, toRemove, and closed. + */ + private final ReentrantLock lock = new ReentrantLock(); + + /** + * Condition variable which indicates that toAdd and toRemove have been + * processed. + */ + private final Condition processedCond = lock.newCondition(); + + /** + * Entries to add. + */ + private final LinkedList toAdd = + new LinkedList(); + + /** + * Entries to remove. + */ + private final TreeMap toRemove = + new TreeMap(); + + /** + * Maximum length of time to go between checking whether the interrupted + * bit has been set for this thread. + */ + private final int interruptCheckPeriodMs; + + /** + * A pair of sockets used to wake up the thread after it has called poll(2). + */ + private final DomainSocket notificationSockets[]; + + /** + * Whether or not this DomainSocketWatcher is closed. + */ + private boolean closed = false; + + public DomainSocketWatcher(int interruptCheckPeriodMs) throws IOException { + if (loadingFailureReason != null) { + throw new UnsupportedOperationException(loadingFailureReason); + } + notificationSockets = DomainSocket.socketpair(); + this.interruptCheckPeriodMs = interruptCheckPeriodMs; + Preconditions.checkArgument(interruptCheckPeriodMs > 0); + watcherThread.start(); + } + + /** + * Close the DomainSocketWatcher and wait for its thread to terminate. + * + * If there is more than one close, all but the first will be ignored. + */ + @Override + public void close() throws IOException { + try { + lock.lock(); + if (closed) return; + LOG.info(this + ": closing"); + closed = true; + } finally { + lock.unlock(); + } + // Close notificationSockets[0], so that notificationSockets[1] gets an EOF + // event. This will wake up the thread immediately if it is blocked inside + // the select() system call. + notificationSockets[0].close(); + // Wait for the select thread to terminate. + Uninterruptibles.joinUninterruptibly(watcherThread); + } + + /** + * Add a socket. + * + * @param sock The socket to add. It is an error to re-add a socket that + * we are already watching. + * @param handler The handler to associate with this socket. This may be + * called any time after this function is called. + */ + public void add(DomainSocket sock, Handler handler) { + try { + lock.lock(); + checkNotClosed(); + Entry entry = new Entry(sock, handler); + try { + sock.refCount.reference(); + } catch (ClosedChannelException e) { + Preconditions.checkArgument(false, + "tried to add a closed DomainSocket to " + this); + } + toAdd.add(entry); + kick(); + while (true) { + try { + processedCond.await(); + } catch (InterruptedException e) { + this.interrupt(); + } + if (!toAdd.contains(entry)) { + break; + } + checkNotClosed(); + } + } finally { + lock.unlock(); + } + } + + /** + * Remove a socket. Its handler will be called. + * + * @param sock The socket to remove. + */ + public void remove(DomainSocket sock) { + try { + lock.lock(); + checkNotClosed(); + toRemove.put(sock.fd, sock); + kick(); + while (true) { + try { + processedCond.await(); + } catch (InterruptedException e) { + this.interrupt(); + } + if (!toRemove.containsKey(sock.fd)) { + break; + } + checkNotClosed(); + } + } finally { + lock.unlock(); + } + } + + /** + * Wake up the DomainSocketWatcher thread. + */ + private void kick() { + try { + notificationSockets[0].getOutputStream().write(0); + } catch (IOException e) { + LOG.error(this + ": error writing to notificationSockets[0]", e); + } + } + + /** + * Check that the DomainSocketWatcher is not closed. + * Must be called while holding the lock. + */ + private void checkNotClosed() { + Preconditions.checkState(lock.isHeldByCurrentThread()); + if (closed) { + throw new RuntimeException("DomainSocketWatcher is closed."); + } + } + + private void sendCallback(String caller, TreeMap entries, + FdSet fdSet, int fd) { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": " + caller + " starting sendCallback for fd " + fd); + } + Entry entry = entries.get(fd); + Preconditions.checkNotNull(entry, + this + ": fdSet contained " + fd + ", which we were " + + "not tracking."); + DomainSocket sock = entry.getDomainSocket(); + if (entry.getHandler().handle(sock)) { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": " + caller + ": closing fd " + fd + + " at the request of the handler."); + } + if (toRemove.remove(fd) != null) { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": " + caller + " : sendCallback processed fd " + + fd + " in toRemove."); + } + } + try { + sock.refCount.unreferenceCheckClosed(); + } catch (IOException e) { + Preconditions.checkArgument(false, + this + ": file descriptor " + sock.fd + " was closed while " + + "still in the poll(2) loop."); + } + IOUtils.cleanup(LOG, sock); + entries.remove(fd); + fdSet.remove(fd); + } else { + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": " + caller + ": sendCallback not " + + "closing fd " + fd); + } + } + } + + private final Thread watcherThread = new Thread(new Runnable() { + @Override + public void run() { + LOG.info(this + ": starting with interruptCheckPeriodMs = " + + interruptCheckPeriodMs); + final TreeMap entries = new TreeMap(); + FdSet fdSet = new FdSet(); + addNotificationSocket(entries, fdSet); + try { + while (true) { + lock.lock(); + try { + for (int fd : fdSet.getAndClearReadableFds()) { + sendCallback("getAndClearReadableFds", entries, fdSet, fd); + } + if (!(toAdd.isEmpty() && toRemove.isEmpty())) { + // Handle pending additions (before pending removes). + for (Iterator iter = toAdd.iterator(); iter.hasNext(); ) { + Entry entry = iter.next(); + DomainSocket sock = entry.getDomainSocket(); + Entry prevEntry = entries.put(sock.fd, entry); + Preconditions.checkState(prevEntry == null, + this + ": tried to watch a file descriptor that we " + + "were already watching: " + sock); + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": adding fd " + sock.fd); + } + fdSet.add(sock.fd); + iter.remove(); + } + // Handle pending removals + while (true) { + Map.Entry entry = toRemove.firstEntry(); + if (entry == null) break; + sendCallback("handlePendingRemovals", + entries, fdSet, entry.getValue().fd); + } + processedCond.signalAll(); + } + // Check if the thread should terminate. Doing this check now is + // easier than at the beginning of the loop, since we know toAdd and + // toRemove are now empty and processedCond has been notified if it + // needed to be. + if (closed) { + LOG.info(toString() + " thread terminating."); + return; + } + // Check if someone sent our thread an InterruptedException while we + // were waiting in poll(). + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } finally { + lock.unlock(); + } + doPoll0(interruptCheckPeriodMs, fdSet); + } + } catch (InterruptedException e) { + LOG.info(toString() + " terminating on InterruptedException"); + } catch (IOException e) { + LOG.error(toString() + " terminating on IOException", e); + } finally { + for (Entry entry : entries.values()) { + sendCallback("close", entries, fdSet, entry.getDomainSocket().fd); + } + entries.clear(); + fdSet.close(); + } + } + }); + + private void addNotificationSocket(final TreeMap entries, + FdSet fdSet) { + entries.put(notificationSockets[1].fd, + new Entry(notificationSockets[1], new NotificationHandler())); + try { + notificationSockets[1].refCount.reference(); + } catch (IOException e) { + throw new RuntimeException(e); + } + fdSet.add(notificationSockets[1].fd); + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": adding notificationSocket " + + notificationSockets[1].fd + ", connected to " + + notificationSockets[0].fd); + } + } + + public String toString() { + return "DomainSocketWatcher(" + System.identityHashCode(this) + ")"; + } + + private static native int doPoll0(int maxWaitMs, FdSet readFds) + throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java new file mode 100644 index 00000000000..388a087bbd3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CloseableReferenceCount.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.nio.channels.AsynchronousCloseException; +import java.nio.channels.ClosedChannelException; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.base.Preconditions; + +/** + * A closeable object that maintains a reference count. + * + * Once the object is closed, attempting to take a new reference will throw + * ClosedChannelException. + */ +public class CloseableReferenceCount { + /** + * Bit mask representing a closed domain socket. + */ + private static final int STATUS_CLOSED_MASK = 1 << 30; + + /** + * The status bits. + * + * Bit 30: 0 = open, 1 = closed. + * Bits 29 to 0: the reference count. + */ + private final AtomicInteger status = new AtomicInteger(0); + + public CloseableReferenceCount() { } + + /** + * Increment the reference count. + * + * @throws ClosedChannelException If the status is closed. + */ + public void reference() throws ClosedChannelException { + int curBits = status.incrementAndGet(); + if ((curBits & STATUS_CLOSED_MASK) != 0) { + status.decrementAndGet(); + throw new ClosedChannelException(); + } + } + + /** + * Decrement the reference count. + * + * @return True if the object is closed and has no outstanding + * references. + */ + public boolean unreference() { + int newVal = status.decrementAndGet(); + Preconditions.checkState(newVal != 0xffffffff, + "called unreference when the reference count was already at 0."); + return newVal == STATUS_CLOSED_MASK; + } + + /** + * Decrement the reference count, checking to make sure that the + * CloseableReferenceCount is not closed. + * + * @throws AsynchronousCloseException If the status is closed. + */ + public void unreferenceCheckClosed() throws ClosedChannelException { + int newVal = status.decrementAndGet(); + if ((newVal & STATUS_CLOSED_MASK) != 0) { + throw new AsynchronousCloseException(); + } + } + + /** + * Return true if the status is currently open. + * + * @return True if the status is currently open. + */ + public boolean isOpen() { + return ((status.get() & STATUS_CLOSED_MASK) == 0); + } + + /** + * Mark the status as closed. + * + * Once the status is closed, it cannot be reopened. + * + * @return The current reference count. + * @throws ClosedChannelException If someone else closes the object + * before we do. + */ + public int setClosed() throws ClosedChannelException { + while (true) { + int curBits = status.get(); + if ((curBits & STATUS_CLOSED_MASK) != 0) { + throw new ClosedChannelException(); + } + if (status.compareAndSet(curBits, curBits | STATUS_CLOSED_MASK)) { + return curBits & (~STATUS_CLOSED_MASK); + } + } + } + + /** + * Get the current reference count. + * + * @return The current reference count. + */ + public int getReferenceCount() { + return status.get() & (~STATUS_CLOSED_MASK); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index a26bf34c12e..2b5c0a4ab0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -18,6 +18,7 @@ #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_nativeio_NativeIO.h" +#include "org_apache_hadoop_io_nativeio_NativeIO_POSIX.h" #ifdef UNIX #include @@ -49,6 +50,10 @@ #include "file_descriptor.h" #include "errno_enum.h" +#define MMAP_PROT_READ org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_READ +#define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE +#define MMAP_PROT_EXEC org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_EXEC + // the NativeIO$POSIX$Stat inner class and its constructor static jclass stat_clazz; static jmethodID stat_ctor; @@ -661,6 +666,39 @@ cleanup: #endif } +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mmap( + JNIEnv *env, jclass clazz, jobject jfd, jint jprot, + jboolean jshared, jlong length) +{ + void *addr = 0; + int prot, flags, fd; + + prot = ((jprot & MMAP_PROT_READ) ? PROT_READ : 0) | + ((jprot & MMAP_PROT_WRITE) ? PROT_WRITE : 0) | + ((jprot & MMAP_PROT_EXEC) ? PROT_EXEC : 0); + flags = (jshared == JNI_TRUE) ? MAP_SHARED : MAP_PRIVATE; + fd = fd_get(env, jfd); + addr = mmap(NULL, length, prot, flags, fd, 0); + if (addr == MAP_FAILED) { + throw_ioe(env, errno); + } + return (jlong)(intptr_t)addr; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_munmap( + JNIEnv *env, jclass clazz, jlong jaddr, jlong length) +{ + void *addr; + + addr = (void*)(intptr_t)jaddr; + if (munmap(addr, length) < 0) { + throw_ioe(env, errno); + } +} + + /* * static native String getGroupName(int gid); * diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c new file mode 100644 index 00000000000..83684027550 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "org_apache_hadoop.h" + +#ifdef UNIX + +#include "exception.h" +#include "file_descriptor.h" +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static pthread_mutex_t g_rand_lock = PTHREAD_MUTEX_INITIALIZER; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_deleteStaleTemporaryFiles0( + JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath) +{ + const char *prefix = NULL, *path = NULL; + char target[PATH_MAX]; + jthrowable jthr; + DIR *dp = NULL; + struct dirent *de; + + prefix = (*env)->GetStringUTFChars(env, jprefix, NULL); + if (!prefix) goto done; // exception raised + path = (*env)->GetStringUTFChars(env, jpath, NULL); + if (!path) goto done; // exception raised + + dp = opendir(path); + if (!dp) { + int ret = errno; + jthr = newIOException(env, "opendir(%s) error %d: %s", + path, ret, terror(ret)); + (*env)->Throw(env, jthr); + goto done; + } + while ((de = readdir(dp))) { + if (strncmp(prefix, de->d_name, strlen(prefix)) == 0) { + int ret = snprintf(target, PATH_MAX, "%s/%s", path, de->d_name); + if ((0 < ret) && (ret < PATH_MAX)) { + unlink(target); + } + } + } + +done: + if (dp) { + closedir(dp); + } + if (prefix) { + (*env)->ReleaseStringUTFChars(env, jprefix, prefix); + } + if (path) { + (*env)->ReleaseStringUTFChars(env, jpath, path); + } +} + +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0( + JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath, jint length) +{ + const char *prefix = NULL, *path = NULL; + char target[PATH_MAX]; + int ret, fd = -1, rnd; + jthrowable jthr; + jobject jret = NULL; + + prefix = (*env)->GetStringUTFChars(env, jprefix, NULL); + if (!prefix) goto done; // exception raised + path = (*env)->GetStringUTFChars(env, jpath, NULL); + if (!path) goto done; // exception raised + + pthread_mutex_lock(&g_rand_lock); + rnd = rand(); + pthread_mutex_unlock(&g_rand_lock); + while (1) { + ret = snprintf(target, PATH_MAX, "%s/%s_%d", + path, prefix, rnd); + if (ret < 0) { + jthr = newIOException(env, "snprintf error"); + (*env)->Throw(env, jthr); + goto done; + } else if (ret >= PATH_MAX) { + jthr = newIOException(env, "computed path was too long."); + (*env)->Throw(env, jthr); + goto done; + } + fd = open(target, O_CREAT | O_EXCL | O_RDWR, 0700); + if (fd >= 0) break; // success + ret = errno; + if (ret == EEXIST) { + // Bad luck -- we got a very rare collision here between us and + // another DataNode (or process). Try again. + continue; + } else if (ret == EINTR) { + // Most of the time, this error is only possible when opening FIFOs. + // But let's be thorough. + continue; + } + jthr = newIOException(env, "open(%s, O_CREAT | O_EXCL | O_RDWR) " + "failed: error %d (%s)", target, ret, terror(ret)); + (*env)->Throw(env, jthr); + goto done; + } + if (unlink(target) < 0) { + jthr = newIOException(env, "unlink(%s) failed: error %d (%s)", + path, ret, terror(ret)); + (*env)->Throw(env, jthr); + goto done; + } + if (ftruncate(fd, length) < 0) { + jthr = newIOException(env, "ftruncate(%s, %d) failed: error %d (%s)", + path, length, ret, terror(ret)); + (*env)->Throw(env, jthr); + goto done; + } + jret = fd_create(env, fd); // throws exception on error. + +done: + if (prefix) { + (*env)->ReleaseStringUTFChars(env, jprefix, prefix); + } + if (path) { + (*env)->ReleaseStringUTFChars(env, jpath, path); + } + if (!jret) { + if (fd >= 0) { + close(fd); + } + } + return jret; +} + +#endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c new file mode 100644 index 00000000000..8281e643672 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_net_unix_DomainSocketWatcher.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static jfieldID fd_set_data_fid; + +#define FD_SET_DATA_MIN_SIZE 2 + +struct fd_set_data { + /** + * Number of fds we have allocated space for. + */ + int alloc_size; + + /** + * Number of fds actually in use. + */ + int used_size; + + /** + * Beginning of pollfd data. + */ + struct pollfd pollfd[0]; +}; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_anchorNative( +JNIEnv *env, jclass clazz) +{ + jclass fd_set_class; + + fd_set_class = (*env)->FindClass(env, + "org/apache/hadoop/net/unix/DomainSocketWatcher$FdSet"); + if (!fd_set_class) return; // exception raised + fd_set_data_fid = (*env)->GetFieldID(env, fd_set_class, "data", "J"); + if (!fd_set_data_fid) return; // exception raised +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_00024FdSet_alloc0( +JNIEnv *env, jclass clazz) +{ + struct fd_set_data *sd; + + sd = calloc(1, sizeof(struct fd_set_data) + + (sizeof(struct pollfd) * FD_SET_DATA_MIN_SIZE)); + if (!sd) { + (*env)->Throw(env, newRuntimeException(env, "out of memory allocating " + "DomainSocketWatcher#FdSet")); + return 0L; + } + sd->alloc_size = FD_SET_DATA_MIN_SIZE; + sd->used_size = 0; + return (jlong)(intptr_t)sd; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_00024FdSet_add( +JNIEnv *env, jobject obj, jint fd) +{ + struct fd_set_data *sd, *nd; + struct pollfd *pollfd; + + sd = (struct fd_set_data*)(intptr_t)(*env)-> + GetLongField(env, obj, fd_set_data_fid); + if (sd->used_size + 1 > sd->alloc_size) { + nd = realloc(sd, sizeof(struct fd_set_data) + + (sizeof(struct pollfd) * sd->alloc_size * 2)); + if (!nd) { + (*env)->Throw(env, newRuntimeException(env, "out of memory adding " + "another fd to DomainSocketWatcher#FdSet. we have %d already", + sd->alloc_size)); + return; + } + nd->alloc_size = nd->alloc_size * 2; + (*env)->SetLongField(env, obj, fd_set_data_fid, (jlong)(intptr_t)nd); + sd = nd; + } + pollfd = &sd->pollfd[sd->used_size]; + sd->used_size++; + pollfd->fd = fd; + pollfd->events = POLLIN; + pollfd->revents = 0; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_00024FdSet_remove( +JNIEnv *env, jobject obj, jint fd) +{ + struct fd_set_data *sd; + struct pollfd *pollfd, *last_pollfd; + int used_size, i; + + sd = (struct fd_set_data*)(intptr_t)(*env)-> + GetLongField(env, obj, fd_set_data_fid); + used_size = sd->used_size; + for (i = 0; i < used_size; i++) { + pollfd = sd->pollfd + i; + if (pollfd->fd == fd) break; + } + if (i == used_size) { + (*env)->Throw(env, newRuntimeException(env, "failed to remove fd %d " + "from the FdSet because it was never present.", fd)); + return; + } + last_pollfd = sd->pollfd + (used_size - 1); + if (used_size > 1) { + // Move last pollfd to the new empty slot if needed + pollfd->fd = last_pollfd->fd; + pollfd->events = last_pollfd->events; + pollfd->revents = last_pollfd->revents; + } + memset(last_pollfd, 0, sizeof(struct pollfd)); + sd->used_size--; +} + +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_00024FdSet_getAndClearReadableFds( +JNIEnv *env, jobject obj) +{ + int *carr = NULL; + jobject jarr = NULL; + struct fd_set_data *sd; + int used_size, num_readable = 0, i, j; + jthrowable jthr = NULL; + + sd = (struct fd_set_data*)(intptr_t)(*env)-> + GetLongField(env, obj, fd_set_data_fid); + used_size = sd->used_size; + for (i = 0; i < used_size; i++) { + if (sd->pollfd[i].revents & POLLIN) { + num_readable++; + } else { + sd->pollfd[i].revents = 0; + } + } + if (num_readable > 0) { + carr = malloc(sizeof(int) * num_readable); + if (!carr) { + jthr = newRuntimeException(env, "failed to allocate a temporary array " + "of %d ints", num_readable); + goto done; + } + j = 0; + for (i = 0; ((i < used_size) && (j < num_readable)); i++) { + if (sd->pollfd[i].revents & POLLIN) { + carr[j] = sd->pollfd[i].fd; + j++; + sd->pollfd[i].revents = 0; + } + } + if (j != num_readable) { + jthr = newRuntimeException(env, "failed to fill entire carr " + "array of size %d: only filled %d elements", num_readable, j); + goto done; + } + } + jarr = (*env)->NewIntArray(env, num_readable); + if (!jarr) { + jthr = (*env)->ExceptionOccurred(env); + (*env)->ExceptionClear(env); + goto done; + } + if (num_readable > 0) { + (*env)->SetIntArrayRegion(env, jarr, 0, num_readable, carr); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + goto done; + } + } + +done: + free(carr); + if (jthr) { + (*env)->DeleteLocalRef(env, jarr); + jarr = NULL; + } + return jarr; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_00024FdSet_close( +JNIEnv *env, jobject obj) +{ + struct fd_set_data *sd; + + sd = (struct fd_set_data*)(intptr_t)(*env)-> + GetLongField(env, obj, fd_set_data_fid); + if (sd) { + free(sd); + (*env)->SetLongField(env, obj, fd_set_data_fid, 0L); + } +} + +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_net_unix_DomainSocketWatcher_doPoll0( +JNIEnv *env, jclass clazz, jint checkMs, jobject fdSet) +{ + struct fd_set_data *sd; + int ret, err; + + sd = (struct fd_set_data*)(intptr_t)(*env)-> + GetLongField(env, fdSet, fd_set_data_fid); + ret = poll(sd->pollfd, sd->used_size, checkMs); + if (ret >= 0) { + return ret; + } + err = errno; + if (err != EINTR) { // treat EINTR as 0 fds ready + (*env)->Throw(env, newIOException(env, + "poll(2) failed with error code %d: %s", err, terror(err))); + } + return 0; +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java new file mode 100644 index 00000000000..153478148d1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.apache.commons.lang.SystemUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; + +public class TestSharedFileDescriptorFactory { + static final Log LOG = LogFactory.getLog(TestSharedFileDescriptorFactory.class); + + private static final File TEST_BASE = + new File(System.getProperty("test.build.data", "/tmp")); + + @Test(timeout=10000) + public void testReadAndWrite() throws Exception { + Assume.assumeTrue(NativeIO.isAvailable()); + Assume.assumeTrue(SystemUtils.IS_OS_UNIX); + File path = new File(TEST_BASE, "testReadAndWrite"); + path.mkdirs(); + SharedFileDescriptorFactory factory = + new SharedFileDescriptorFactory("woot_", path.getAbsolutePath()); + FileInputStream inStream = factory.createDescriptor(4096); + FileOutputStream outStream = new FileOutputStream(inStream.getFD()); + outStream.write(101); + inStream.getChannel().position(0); + Assert.assertEquals(101, inStream.read()); + inStream.close(); + outStream.close(); + FileUtil.fullyDelete(path); + } + + static private void createTempFile(String path) throws Exception { + FileOutputStream fos = new FileOutputStream(path); + fos.write(101); + fos.close(); + } + + @Test(timeout=10000) + public void testCleanupRemainders() throws Exception { + Assume.assumeTrue(NativeIO.isAvailable()); + Assume.assumeTrue(SystemUtils.IS_OS_UNIX); + File path = new File(TEST_BASE, "testCleanupRemainders"); + path.mkdirs(); + String remainder1 = path.getAbsolutePath() + + Path.SEPARATOR + "woot2_remainder1"; + String remainder2 = path.getAbsolutePath() + + Path.SEPARATOR + "woot2_remainder2"; + createTempFile(remainder1); + createTempFile(remainder2); + new SharedFileDescriptorFactory("woot2_", path.getAbsolutePath()); + // creating the SharedFileDescriptorFactory should have removed + // the remainders + Assert.assertFalse(new File(remainder1).exists()); + Assert.assertFalse(new File(remainder2).exists()); + FileUtil.fullyDelete(path); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java new file mode 100644 index 00000000000..b9d76cb603d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net.unix; + +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.Uninterruptibles; + +public class TestDomainSocketWatcher { + static final Log LOG = LogFactory.getLog(TestDomainSocketWatcher.class); + + @Before + public void before() { + Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null); + } + + /** + * Test that we can create a DomainSocketWatcher and then shut it down. + */ + @Test(timeout=60000) + public void testCreateShutdown() throws Exception { + DomainSocketWatcher watcher = new DomainSocketWatcher(10000000); + watcher.close(); + } + + /** + * Test that we can get notifications out a DomainSocketWatcher. + */ + @Test(timeout=180000) + public void testDeliverNotifications() throws Exception { + DomainSocketWatcher watcher = new DomainSocketWatcher(10000000); + DomainSocket pair[] = DomainSocket.socketpair(); + final CountDownLatch latch = new CountDownLatch(1); + watcher.add(pair[1], new DomainSocketWatcher.Handler() { + @Override + public boolean handle(DomainSocket sock) { + latch.countDown(); + return true; + } + }); + pair[0].close(); + latch.await(); + watcher.close(); + } + + /** + * Test that a java interruption can stop the watcher thread + */ + @Test(timeout=60000) + public void testInterruption() throws Exception { + DomainSocketWatcher watcher = new DomainSocketWatcher(10); + watcher.interrupt(); + Uninterruptibles.joinUninterruptibly(watcher); + } + + @Test(timeout=300000) + public void testStress() throws Exception { + final int SOCKET_NUM = 250; + final ReentrantLock lock = new ReentrantLock(); + final DomainSocketWatcher watcher = new DomainSocketWatcher(10000000); + final ArrayList pairs = new ArrayList(); + final AtomicInteger handled = new AtomicInteger(0); + + final Thread adderThread = new Thread(new Runnable() { + @Override + public void run() { + try { + for (int i = 0; i < SOCKET_NUM; i++) { + DomainSocket pair[] = DomainSocket.socketpair(); + watcher.add(pair[1], new DomainSocketWatcher.Handler() { + @Override + public boolean handle(DomainSocket sock) { + handled.incrementAndGet(); + return true; + } + }); + lock.lock(); + try { + pairs.add(pair); + } finally { + lock.unlock(); + } + } + } catch (Throwable e) { + LOG.error(e); + throw new RuntimeException(e); + } + } + }); + + final Thread removerThread = new Thread(new Runnable() { + @Override + public void run() { + final Random random = new Random(); + try { + while (handled.get() != SOCKET_NUM) { + lock.lock(); + try { + if (!pairs.isEmpty()) { + int idx = random.nextInt(pairs.size()); + DomainSocket pair[] = pairs.remove(idx); + if (random.nextBoolean()) { + pair[0].close(); + } else { + watcher.remove(pair[1]); + } + } + } finally { + lock.unlock(); + } + } + } catch (Throwable e) { + LOG.error(e); + throw new RuntimeException(e); + } + } + }); + + adderThread.start(); + removerThread.start(); + Uninterruptibles.joinUninterruptibly(adderThread); + Uninterruptibles.joinUninterruptibly(removerThread); + watcher.close(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 490dc9beced..5867e7bf2f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -306,6 +306,8 @@ Release 2.4.0 - UNRELEASED HDFS-5859. DataNode#checkBlockToken should check block tokens even if security is not enabled. (cmccabe) + HDFS-5746. Add ShortCircuitSharedMemorySegment (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitSharedMemorySegment.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitSharedMemorySegment.java new file mode 100644 index 00000000000..ec3084cab0c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitSharedMemorySegment.java @@ -0,0 +1,302 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.client; + +import java.io.Closeable; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.reflect.Field; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.io.nativeio.NativeIO.POSIX; +import org.apache.hadoop.util.CloseableReferenceCount; +import org.apache.hadoop.util.Shell; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import sun.misc.Unsafe; + +public class ShortCircuitSharedMemorySegment implements Closeable { + private static final Log LOG = + LogFactory.getLog(ShortCircuitSharedMemorySegment.class); + + private static final int BYTES_PER_SLOT = 64; + + private static final Unsafe unsafe; + + static { + Unsafe theUnsafe = null; + try { + Field f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + theUnsafe = (Unsafe)f.get(null); + } catch (Throwable e) { + LOG.error("failed to load misc.Unsafe", e); + } + unsafe = theUnsafe; + } + + /** + * A slot containing information about a replica. + * + * The format is: + * word 0 + * bit 0:32 Slot flags (see below). + * bit 33:63 Anchor count. + * word 1:7 + * Reserved for future use, such as statistics. + * Padding is also useful for avoiding false sharing. + * + * Little-endian versus big-endian is not relevant here since both the client + * and the server reside on the same computer and use the same orientation. + */ + public class Slot implements Closeable { + /** + * Flag indicating that the slot is in use. + */ + private static final long SLOT_IN_USE_FLAG = 1L<<63; + + /** + * Flag indicating that the slot can be anchored. + */ + private static final long ANCHORABLE_FLAG = 1L<<62; + + private long slotAddress; + + Slot(long slotAddress) { + this.slotAddress = slotAddress; + } + + /** + * Make a given slot anchorable. + */ + public void makeAnchorable() { + Preconditions.checkState(slotAddress != 0, + "Called makeAnchorable on a slot that was closed."); + long prev; + do { + prev = unsafe.getLongVolatile(null, this.slotAddress); + if ((prev & ANCHORABLE_FLAG) != 0) { + return; + } + } while (!unsafe.compareAndSwapLong(null, this.slotAddress, + prev, prev | ANCHORABLE_FLAG)); + } + + /** + * Make a given slot unanchorable. + */ + public void makeUnanchorable() { + Preconditions.checkState(slotAddress != 0, + "Called makeUnanchorable on a slot that was closed."); + long prev; + do { + prev = unsafe.getLongVolatile(null, this.slotAddress); + if ((prev & ANCHORABLE_FLAG) == 0) { + return; + } + } while (!unsafe.compareAndSwapLong(null, this.slotAddress, + prev, prev & (~ANCHORABLE_FLAG))); + } + + /** + * Try to add an anchor for a given slot. + * + * When a slot is anchored, we know that the block it refers to is resident + * in memory. + * + * @return True if the slot is anchored. + */ + public boolean addAnchor() { + long prev; + do { + prev = unsafe.getLongVolatile(null, this.slotAddress); + if ((prev & 0x7fffffff) == 0x7fffffff) { + // Too many other threads have anchored the slot (2 billion?) + return false; + } + if ((prev & ANCHORABLE_FLAG) == 0) { + // Slot can't be anchored right now. + return false; + } + } while (!unsafe.compareAndSwapLong(null, this.slotAddress, + prev, prev + 1)); + return true; + } + + /** + * Remove an anchor for a given slot. + */ + public void removeAnchor() { + long prev; + do { + prev = unsafe.getLongVolatile(null, this.slotAddress); + Preconditions.checkState((prev & 0x7fffffff) != 0, + "Tried to remove anchor for slot " + slotAddress +", which was " + + "not anchored."); + } while (!unsafe.compareAndSwapLong(null, this.slotAddress, + prev, prev - 1)); + } + + /** + * @return The index of this slot. + */ + public int getIndex() { + Preconditions.checkState(slotAddress != 0); + return Ints.checkedCast( + (slotAddress - baseAddress) / BYTES_PER_SLOT); + } + + @Override + public void close() throws IOException { + if (slotAddress == 0) return; + long prev; + do { + prev = unsafe.getLongVolatile(null, this.slotAddress); + Preconditions.checkState((prev & SLOT_IN_USE_FLAG) != 0, + "tried to close slot that wasn't open"); + } while (!unsafe.compareAndSwapLong(null, this.slotAddress, + prev, 0)); + slotAddress = 0; + if (ShortCircuitSharedMemorySegment.this.refCount.unreference()) { + ShortCircuitSharedMemorySegment.this.free(); + } + } + } + + /** + * The stream that we're going to use to create this shared memory segment. + * + * Although this is a FileInputStream, we are going to assume that the + * underlying file descriptor is writable as well as readable. + * It would be more appropriate to use a RandomAccessFile here, but that class + * does not have any public accessor which returns a FileDescriptor, unlike + * FileInputStream. + */ + private final FileInputStream stream; + + /** + * Length of the shared memory segment. + */ + private final int length; + + /** + * The base address of the memory-mapped file. + */ + private final long baseAddress; + + /** + * Reference count and 'closed' status. + */ + private final CloseableReferenceCount refCount = new CloseableReferenceCount(); + + public ShortCircuitSharedMemorySegment(FileInputStream stream) + throws IOException { + if (!NativeIO.isAvailable()) { + throw new UnsupportedOperationException("NativeIO is not available."); + } + if (Shell.WINDOWS) { + throw new UnsupportedOperationException( + "ShortCircuitSharedMemorySegment is not yet implemented " + + "for Windows."); + } + if (unsafe == null) { + throw new UnsupportedOperationException( + "can't use ShortCircuitSharedMemorySegment because we failed to " + + "load misc.Unsafe."); + } + this.refCount.reference(); + this.stream = stream; + this.length = getEffectiveLength(stream); + this.baseAddress = POSIX.mmap(this.stream.getFD(), + POSIX.MMAP_PROT_READ | POSIX.MMAP_PROT_WRITE, true, this.length); + } + + /** + * Calculate the effective usable size of the shared memory segment. + * We round down to a multiple of the slot size and do some validation. + * + * @param stream The stream we're using. + * @return The effective usable size of the shared memory segment. + */ + private static int getEffectiveLength(FileInputStream stream) + throws IOException { + int intSize = Ints.checkedCast(stream.getChannel().size()); + int slots = intSize / BYTES_PER_SLOT; + Preconditions.checkState(slots > 0, "size of shared memory segment was " + + intSize + ", but that is not enough to hold even one slot."); + return slots * BYTES_PER_SLOT; + } + + private boolean allocateSlot(long address) { + long prev; + do { + prev = unsafe.getLongVolatile(null, address); + if ((prev & Slot.SLOT_IN_USE_FLAG) != 0) { + return false; + } + } while (!unsafe.compareAndSwapLong(null, address, + prev, prev | Slot.SLOT_IN_USE_FLAG)); + return true; + } + + /** + * Allocate a new Slot in this shared memory segment. + * + * @return A newly allocated Slot, or null if there were no available + * slots. + */ + public Slot allocateNextSlot() throws IOException { + ShortCircuitSharedMemorySegment.this.refCount.reference(); + Slot slot = null; + try { + final int numSlots = length / BYTES_PER_SLOT; + for (int i = 0; i < numSlots; i++) { + long address = this.baseAddress + (i * BYTES_PER_SLOT); + if (allocateSlot(address)) { + slot = new Slot(address); + break; + } + } + } finally { + if (slot == null) { + if (refCount.unreference()) { + free(); + } + } + } + return slot; + } + + @Override + public void close() throws IOException { + refCount.setClosed(); + if (refCount.unreference()) { + free(); + } + } + + void free() throws IOException { + IOUtils.cleanup(LOG, stream); + POSIX.munmap(baseAddress, length); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitSharedMemorySegment.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitSharedMemorySegment.java new file mode 100644 index 00000000000..4a9abbe1a6f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitSharedMemorySegment.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.client; + +import java.io.File; +import java.io.FileInputStream; +import java.util.ArrayList; + +import org.apache.commons.lang.SystemUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory; +import org.apache.hadoop.hdfs.client.ShortCircuitSharedMemorySegment.Slot; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.Assert; + +public class TestShortCircuitSharedMemorySegment { + public static final Log LOG = + LogFactory.getLog(TestShortCircuitSharedMemorySegment.class); + + private static final File TEST_BASE = + new File(System.getProperty("test.build.data", "/tmp")); + + @Before + public void before() { + Assume.assumeTrue(NativeIO.isAvailable()); + Assume.assumeTrue(SystemUtils.IS_OS_UNIX); + } + + @Test(timeout=60000) + public void testStartupShutdown() throws Exception { + File path = new File(TEST_BASE, "testStartupShutdown"); + path.mkdirs(); + SharedFileDescriptorFactory factory = + new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + FileInputStream stream = factory.createDescriptor(4096); + ShortCircuitSharedMemorySegment shm = + new ShortCircuitSharedMemorySegment(stream); + shm.close(); + stream.close(); + FileUtil.fullyDelete(path); + } + + @Test(timeout=60000) + public void testAllocateSlots() throws Exception { + File path = new File(TEST_BASE, "testAllocateSlots"); + path.mkdirs(); + SharedFileDescriptorFactory factory = + new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + FileInputStream stream = factory.createDescriptor(4096); + ShortCircuitSharedMemorySegment shm = + new ShortCircuitSharedMemorySegment(stream); + int numSlots = 0; + ArrayList slots = new ArrayList(); + while (true) { + Slot slot = shm.allocateNextSlot(); + if (slot == null) { + LOG.info("allocated " + numSlots + " slots before running out."); + break; + } + slots.add(slot); + numSlots++; + } + int slotIdx = 0; + for (Slot slot : slots) { + Assert.assertFalse(slot.addAnchor()); + Assert.assertEquals(slotIdx++, slot.getIndex()); + } + for (Slot slot : slots) { + slot.makeAnchorable(); + } + for (Slot slot : slots) { + Assert.assertTrue(slot.addAnchor()); + } + for (Slot slot : slots) { + slot.removeAnchor(); + } + shm.close(); + for (Slot slot : slots) { + slot.close(); + } + stream.close(); + FileUtil.fullyDelete(path); + } +} From 1079c371289cd31478ed4bc123c1c4dd846c76ee Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Sat, 1 Feb 2014 04:03:36 +0000 Subject: [PATCH 04/54] YARN-1504. RM changes for moving apps between queues (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563371 13f79535-47bb-0310-9956-ffa450edef68 --- .../scheduler/ResourceSchedulerWrapper.java | 7 + hadoop-yarn-project/CHANGES.txt | 2 + .../resourcemanager/ClientRMService.java | 71 ++++++- .../server/resourcemanager/RMAuditLogger.java | 1 + .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../resourcemanager/rmapp/RMAppImpl.java | 32 ++++ .../resourcemanager/rmapp/RMAppMoveEvent.java | 44 +++++ .../scheduler/AbstractYarnScheduler.java | 10 +- .../scheduler/YarnScheduler.java | 14 ++ .../scheduler/fair/FairScheduler.java | 5 +- .../scheduler/fifo/FifoScheduler.java | 3 +- .../resourcemanager/TestClientRMService.java | 15 ++ .../resourcemanager/TestMoveApplication.java | 180 ++++++++++++++++++ 13 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMoveEvent.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 9a3cb2450ee..f923733d5b3 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -865,4 +866,10 @@ public class ResourceSchedulerWrapper implements public RMContainer getRMContainer(ContainerId containerId) { return null; } + + @Override + public String moveApplication(ApplicationId appId, String newQueue) + throws YarnException { + return scheduler.moveApplication(appId, newQueue); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index abc56d6961a..84634df909c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -12,6 +12,8 @@ Trunk - Unreleased YARN-1498. Common scheduler changes for moving apps between queues (Sandy Ryza) + YARN-1504. RM changes for moving apps between queues (Sandy Ryza) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 1df67f82a86..8800f290cdc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -94,6 +94,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -104,6 +106,9 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; + /** * The client interface to the Resource Manager. This module handles all the rpc @@ -686,10 +691,74 @@ public class ClientRMService extends AbstractService implements } } + @SuppressWarnings("unchecked") @Override public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues( MoveApplicationAcrossQueuesRequest request) throws YarnException { - throw new UnsupportedOperationException("Move not yet supported"); + ApplicationId applicationId = request.getApplicationId(); + + UserGroupInformation callerUGI; + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException ie) { + LOG.info("Error getting UGI ", ie); + RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST, + "UNKNOWN", "ClientRMService" , "Error getting UGI", + applicationId); + throw RPCUtil.getRemoteException(ie); + } + + RMApp application = this.rmContext.getRMApps().get(applicationId); + if (application == null) { + RMAuditLogger.logFailure(callerUGI.getUserName(), + AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", + "Trying to move an absent application", applicationId); + throw new ApplicationNotFoundException("Trying to move an absent" + + " application " + applicationId); + } + + if (!checkAccess(callerUGI, application.getUser(), + ApplicationAccessType.MODIFY_APP, application)) { + RMAuditLogger.logFailure(callerUGI.getShortUserName(), + AuditConstants.MOVE_APP_REQUEST, + "User doesn't have permissions to " + + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", + AuditConstants.UNAUTHORIZED_USER, applicationId); + throw RPCUtil.getRemoteException(new AccessControlException("User " + + callerUGI.getShortUserName() + " cannot perform operation " + + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); + } + + // Moves only allowed when app is in a state that means it is tracked by + // the scheduler + if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED, + RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED, + RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED) + .contains(application.getState())) { + String msg = "App in " + application.getState() + " state cannot be moved."; + RMAuditLogger.logFailure(callerUGI.getShortUserName(), + AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg); + throw new YarnException(msg); + } + + SettableFuture future = SettableFuture.create(); + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppMoveEvent(applicationId, request.getTargetQueue(), future)); + + try { + Futures.get(future, YarnException.class); + } catch (YarnException ex) { + RMAuditLogger.logFailure(callerUGI.getShortUserName(), + AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", + ex.getMessage()); + throw ex; + } + + RMAuditLogger.logSuccess(callerUGI.getShortUserName(), + AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId); + MoveApplicationAcrossQueuesResponse response = recordFactory + .newRecordInstance(MoveApplicationAcrossQueuesResponse.class); + return response; } private String getRenewerForToken(Token token) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index b9261cac200..f7d12136f03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -45,6 +45,7 @@ public class RMAuditLogger { public static final String KILL_APP_REQUEST = "Kill Application Request"; public static final String SUBMIT_APP_REQUEST = "Submit Application Request"; + public static final String MOVE_APP_REQUEST = "Move Application Request"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index bb63e6f3f73..3ab5db4bcea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -23,6 +23,7 @@ public enum RMAppEventType { START, RECOVER, KILL, + MOVE, // Move app to a new queue // Source: Scheduler and RMAppManager APP_REJECTED, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 59132ce9e89..d4cf416dd33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; @@ -166,6 +167,8 @@ public class RMAppImpl implements RMApp, Recoverable { // Transitions from SUBMITTED state .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) + .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, + RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING, RMAppEventType.APP_REJECTED, new FinalSavingTransition( @@ -181,6 +184,8 @@ public class RMAppImpl implements RMApp, Recoverable { // Transitions from ACCEPTED state .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) + .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, + RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING, RMAppEventType.ATTEMPT_REGISTERED) .addTransition(RMAppState.ACCEPTED, @@ -204,6 +209,8 @@ public class RMAppImpl implements RMApp, Recoverable { // Transitions from RUNNING state .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) + .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, + RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING, RMAppEventType.ATTEMPT_UNREGISTERED, new FinalSavingTransition( @@ -692,6 +699,31 @@ public class RMAppImpl implements RMApp, Recoverable { }; } + /** + * Move an app to a new queue. + * This transition must set the result on the Future in the RMAppMoveEvent, + * either as an exception for failure or null for success, or the client will + * be left waiting forever. + */ + @SuppressWarnings("unchecked") + private static final class RMAppMoveTransition extends RMAppTransition { + public void transition(RMAppImpl app, RMAppEvent event) { + RMAppMoveEvent moveEvent = (RMAppMoveEvent) event; + try { + app.queue = app.scheduler.moveApplication(app.applicationId, + moveEvent.getTargetQueue()); + } catch (YarnException ex) { + moveEvent.getResult().setException(ex); + return; + } + + // TODO: Write out change to state store (YARN-1558) + + moveEvent.getResult().set(null); + } + } + + @SuppressWarnings("unchecked") private static final class RMAppRecoveredTransition implements MultipleArcTransition { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMoveEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMoveEvent.java new file mode 100644 index 00000000000..5fc63c9294e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMoveEvent.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + +import com.google.common.util.concurrent.SettableFuture; + +public class RMAppMoveEvent extends RMAppEvent { + private String targetQueue; + private SettableFuture result; + + public RMAppMoveEvent(ApplicationId id, String newQueue, + SettableFuture resultFuture) { + super(id, RMAppEventType.MOVE); + this.targetQueue = newQueue; + this.result = resultFuture; + } + + public String getTargetQueue() { + return targetQueue; + } + + public SettableFuture getResult() { + return result; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e460f1cb5e5..4208d1db5e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -27,11 +27,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -public class AbstractYarnScheduler { +public abstract class AbstractYarnScheduler implements ResourceScheduler { protected RMContext rmContext; protected Map applications; @@ -61,4 +62,11 @@ public class AbstractYarnScheduler { public Map getSchedulerApplications() { return applications; } + + @Override + public String moveApplication(ApplicationId appId, String newQueue) + throws YarnException { + throw new YarnException(getClass().getSimpleName() + + " does not support moving apps between queues"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 4f1cb74c8b7..2348603465d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; /** @@ -180,4 +182,16 @@ public interface YarnScheduler extends EventHandler { @LimitedPrivate("yarn") @Unstable public RMContainer getRMContainer(ContainerId containerId); + + /** + * Moves the given application to the given queue + * @param appId + * @param newQueue + * @return the name of the queue the application was placed into + * @throws YarnException if the move cannot be carried out + */ + @LimitedPrivate("yarn") + @Evolving + public String moveApplication(ApplicationId appId, String newQueue) + throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 3ff3b04e63a..b88ad503dd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; @@ -75,7 +76,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; @@ -121,8 +121,7 @@ import com.google.common.annotations.VisibleForTesting; @LimitedPrivate("yarn") @Unstable @SuppressWarnings("unchecked") -public class FairScheduler extends AbstractYarnScheduler implements - ResourceScheduler { +public class FairScheduler extends AbstractYarnScheduler { private boolean initialized; private FairSchedulerConfiguration conf; private Resource minimumAllocation; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 026f22c49c3..e33348a10d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -77,7 +77,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; @@ -106,7 +105,7 @@ import com.google.common.annotations.VisibleForTesting; @Evolving @SuppressWarnings("unchecked") public class FifoScheduler extends AbstractYarnScheduler implements - ResourceScheduler, Configurable { + Configurable { private static final Log LOG = LogFactory.getLog(FifoScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 7e3d5fe073b..1894a115ac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -232,6 +233,20 @@ public class TestClientRMService { "application " + request.getApplicationId()); } } + + @Test (expected = ApplicationNotFoundException.class) + public void testMoveAbsentApplication() throws YarnException { + RMContext rmContext = mock(RMContext.class); + when(rmContext.getRMApps()).thenReturn( + new ConcurrentHashMap()); + ClientRMService rmService = new ClientRMService(rmContext, null, null, + null, null, null); + ApplicationId applicationId = + BuilderUtils.newApplicationId(System.currentTimeMillis(), 0); + MoveApplicationAcrossQueuesRequest request = + MoveApplicationAcrossQueuesRequest.newInstance(applicationId, "newqueue"); + rmService.moveApplicationAcrossQueues(request); + } @Test public void testGetQueueInfo() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java new file mode 100644 index 00000000000..1b60fce4d33 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java @@ -0,0 +1,180 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.security.AccessControlException; +import java.security.PrivilegedExceptionAction; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestMoveApplication { + private ResourceManager resourceManager = null; + private static boolean failMove; + + @Before + public void setUp() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class, + FifoSchedulerWithMove.class); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, " "); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + resourceManager = new ResourceManager(); + resourceManager.init(conf); + resourceManager.getRMContainerTokenSecretManager().rollMasterKey(); + resourceManager.getRMNMTokenSecretManager().rollMasterKey(); + resourceManager.start(); + failMove = false; + } + + @After + public void tearDown() { + resourceManager.stop(); + } + + @Test + public void testMoveRejectedByScheduler() throws Exception { + failMove = true; + + // Submit application + Application application = new Application("user1", resourceManager); + application.submit(); + + ClientRMService clientRMService = resourceManager.getClientRMService(); + try { + // FIFO scheduler does not support moves + clientRMService.moveApplicationAcrossQueues( + MoveApplicationAcrossQueuesRequest.newInstance( + application.getApplicationId(), "newqueue")); + fail("Should have hit exception"); + } catch (YarnException ex) { + assertEquals("Move not supported", ex.getCause().getMessage()); + } + } + + @Test (timeout = 10000) + public void testMoveTooLate() throws Exception { + // Submit application + Application application = new Application("user1", resourceManager); + ApplicationId appId = application.getApplicationId(); + application.submit(); + + ClientRMService clientRMService = resourceManager.getClientRMService(); + // Kill the application + clientRMService.forceKillApplication( + KillApplicationRequest.newInstance(appId)); + RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId); + // wait until it's dead + while (rmApp.getState() != RMAppState.KILLED) { + Thread.sleep(100); + } + + try { + clientRMService.moveApplicationAcrossQueues( + MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue")); + fail("Should have hit exception"); + } catch (YarnException ex) { + assertEquals(YarnException.class, + ex.getClass()); + assertEquals("App in KILLED state cannot be moved.", ex.getMessage()); + } + } + + @Test (timeout = 5000) + public void testMoveSuccessful() throws Exception { + // Submit application + Application application = new Application("user1", resourceManager); + ApplicationId appId = application.getApplicationId(); + application.submit(); + + // Wait for app to be accepted + RMApp app = resourceManager.rmContext.getRMApps().get(appId); + while (app.getState() != RMAppState.ACCEPTED) { + Thread.sleep(100); + } + + ClientRMService clientRMService = resourceManager.getClientRMService(); + // FIFO scheduler does not support moves + clientRMService.moveApplicationAcrossQueues( + MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue")); + + RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId); + assertEquals("newqueue", rmApp.getQueue()); + } + + @Test + public void testMoveRejectedByPermissions() throws Exception { + failMove = true; + + // Submit application + final Application application = new Application("user1", resourceManager); + application.submit(); + + final ClientRMService clientRMService = resourceManager.getClientRMService(); + try { + UserGroupInformation.createRemoteUser("otheruser").doAs( + new PrivilegedExceptionAction() { + @Override + public MoveApplicationAcrossQueuesResponse run() throws Exception { + return clientRMService.moveApplicationAcrossQueues( + MoveApplicationAcrossQueuesRequest.newInstance( + application.getApplicationId(), "newqueue")); + } + + }); + fail("Should have hit exception"); + } catch (Exception ex) { + assertEquals(AccessControlException.class, ex.getCause().getCause().getClass()); + } + } + + public static class FifoSchedulerWithMove extends FifoScheduler { + @Override + public String moveApplication(ApplicationId appId, String newQueue) + throws YarnException { + if (failMove) { + throw new YarnException("Move not supported"); + } + return newQueue; + } + + + @Override + public synchronized boolean checkAccess(UserGroupInformation callerUGI, + QueueACL acl, String queueName) { + return acl != QueueACL.ADMINISTER_QUEUE; + } + } +} From 94b512bc51ecb5f4eafcf17676a340a6033aefe4 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sun, 2 Feb 2014 15:39:02 +0000 Subject: [PATCH 05/54] HDFS-5828. BlockPlacementPolicyWithNodeGroup can place multiple replicas on the same node group when dfs.namenode.avoid.write.stale.datanode is true. (Buddy via junping_du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563640 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../server/blockmanagement/BlockPlacementPolicyDefault.java | 2 +- .../blockmanagement/TestReplicationPolicyWithNodeGroup.java | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5867e7bf2f8..0b9caec75b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -324,6 +324,10 @@ Release 2.4.0 - UNRELEASED HDFS-5856. DataNode.checkDiskError might throw NPE. (Josh Elser via suresh) + HDFS-5828. BlockPlacementPolicyWithNodeGroup can place multiple replicas on + the same node group when dfs.namenode.avoid.write.stale.datanode is true. + (Buddy via junping_du) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index ff1f0f4c64d..f4dc208d731 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -317,7 +317,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // We need to additionally exclude the nodes that were added to the // result list in the successful calls to choose*() above. for (DatanodeStorageInfo resultStorage : results) { - oldExcludedNodes.add(resultStorage.getDatanodeDescriptor()); + addToExcludedNodes(resultStorage.getDatanodeDescriptor(), oldExcludedNodes); } // Set numOfReplicas, since it can get out of sync with the result list // if the NotEnoughReplicasException was thrown in chooseRandom(). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index 23209f0e28b..ffcc18cc789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -124,6 +124,8 @@ public class TestReplicationPolicyWithNodeGroup { CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, NetworkTopologyWithNodeGroup.class.getName()); + CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); + File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class); CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, From 88d8ea95092d8391c106532511d40c897ade5707 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sun, 2 Feb 2014 19:51:39 +0000 Subject: [PATCH 06/54] HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563694 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/service/CompositeService.java | 5 +- .../hadoop/service}/TestCompositeService.java | 232 ++++++++++++++++-- 3 files changed, 217 insertions(+), 23 deletions(-) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util => hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service}/TestCompositeService.java (66%) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c858d7d4266..d4db1854569 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -313,6 +313,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10320. Javadoc in InterfaceStability.java lacks final . (René Nyffenegger via cnauroth) + + HADOOP-10085. CompositeService should allow adding services while being + inited. (Steve Loughran via kasha) Release 2.3.0 - UNRELEASED diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java index 123ca52ba96..ca667e2bcf4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java @@ -19,7 +19,6 @@ package org.apache.hadoop.service; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -54,13 +53,13 @@ public class CompositeService extends AbstractService { } /** - * Get an unmodifiable list of services + * Get a cloned list of services * @return a list of child services at the time of invocation - * added services will not be picked up. */ public List getServices() { synchronized (serviceList) { - return Collections.unmodifiableList(serviceList); + return new ArrayList(serviceList); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java similarity index 66% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java index 3dbdc135ac4..f2ede7d9b65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java @@ -16,26 +16,20 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.util; +package org.apache.hadoop.service; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service.STATE; +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.service.BreakableService; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.service.ServiceStateException; -import org.apache.hadoop.service.Service.STATE; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.junit.Before; -import org.junit.Test; - public class TestCompositeService { private static final int NUM_OF_SERVICES = 5; @@ -156,7 +150,7 @@ public class TestCompositeService { try { serviceManager.start(); fail("Exception should have been thrown due to startup failure of last service"); - } catch (YarnRuntimeException e) { + } catch (ServiceTestRuntimeException e) { for (int i = 0; i < NUM_OF_SERVICES - 1; i++) { if (i >= FAILED_SERVICE_SEQ_NUMBER && STOP_ONLY_STARTED_SERVICES) { // Failed service state should be INITED @@ -197,7 +191,7 @@ public class TestCompositeService { // Stop the composite service try { serviceManager.stop(); - } catch (YarnRuntimeException e) { + } catch (ServiceTestRuntimeException e) { } assertInState(STATE.STOPPED, services); } @@ -335,7 +329,41 @@ public class TestCompositeService { testService.init(new Configuration()); assertEquals("Incorrect number of services", - 1, testService.getServices().size()); + 1, testService.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddInitedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddUninitedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + try { + parent.start(); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); } @Test @@ -365,6 +393,105 @@ public class TestCompositeService { 2, testService.getServices().size()); } + @Test(timeout = 1000) + public void testAddStartedChildBeforeInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService child = new BreakableService(); + child.init(new Configuration()); + child.start(); + AddSiblingService.addChildToService(parent, child); + try { + parent.init(new Configuration()); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + } + + @Test(timeout = 1000) + public void testAddStoppedChildBeforeInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService child = new BreakableService(); + child.init(new Configuration()); + child.start(); + child.stop(); + AddSiblingService.addChildToService(parent, child); + try { + parent.init(new Configuration()); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInStart() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STARTED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddUninitedSiblingInStart() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STARTED)); + parent.init(new Configuration()); + assertInState(STATE.NOTINITED, sibling); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + assertInState(STATE.STARTED, sibling); + parent.start(); + assertInState(STATE.STARTED, sibling); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + assertInState(STATE.STOPPED, sibling); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInStop() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STOPPED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + public static class CompositeServiceAddingAChild extends CompositeService{ Service child; @@ -379,7 +506,18 @@ public class TestCompositeService { super.serviceInit(conf); } } - + + public static class ServiceTestRuntimeException extends RuntimeException { + public ServiceTestRuntimeException(String message) { + super(message); + } + } + + /** + * This is a composite service that keeps a count of the number of lifecycle + * events called, and can be set to throw a {@link ServiceTestRuntimeException } + * during service start or stop + */ public static class CompositeServiceImpl extends CompositeService { public static boolean isPolicyToStopOnlyStartedServices() { @@ -408,7 +546,7 @@ public class TestCompositeService { @Override protected void serviceStart() throws Exception { if (throwExceptionOnStart) { - throw new YarnRuntimeException("Fake service start exception"); + throw new ServiceTestRuntimeException("Fake service start exception"); } counter++; callSequenceNumber = counter; @@ -420,7 +558,7 @@ public class TestCompositeService { counter++; callSequenceNumber = counter; if (throwExceptionOnStop) { - throw new YarnRuntimeException("Fake service stop exception"); + throw new ServiceTestRuntimeException("Fake service stop exception"); } super.serviceStop(); } @@ -457,6 +595,9 @@ public class TestCompositeService { } + /** + * Composite service that makes the addService method public to all + */ public static class ServiceManager extends CompositeService { public void addTestService(CompositeService service) { @@ -468,4 +609,55 @@ public class TestCompositeService { } } + public static class AddSiblingService extends CompositeService { + private final CompositeService parent; + private final Service serviceToAdd; + private STATE triggerState; + + public AddSiblingService(CompositeService parent, + Service serviceToAdd, + STATE triggerState) { + super("ParentStateManipulatorService"); + this.parent = parent; + this.serviceToAdd = serviceToAdd; + this.triggerState = triggerState; + } + + /** + * Add the serviceToAdd to the parent if this service + * is in the state requested + */ + private void maybeAddSibling() { + if (getServiceState() == triggerState) { + parent.addService(serviceToAdd); + } + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + maybeAddSibling(); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + maybeAddSibling(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + maybeAddSibling(); + super.serviceStop(); + } + + /** + * Expose addService method + * @param parent parent service + * @param child child to add + */ + public static void addChildToService(CompositeService parent, Service child) { + parent.addService(child); + } + } } From ca72e11158047f3d00db9e9e61c632e757383287 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 19:07:55 +0000 Subject: [PATCH 07/54] YARN-1611. Introduced the concept of a configuration provider which can be used by ResourceManager to read configuration locally or from remote systems so as to help RM failover. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564002 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../yarn/conf/ConfigurationProvider.java | 64 +++++++ .../conf/ConfigurationProviderFactory.java | 57 ++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 13 ++ .../FileSystemBasedConfigurationProvider.java | 72 ++++++++ .../yarn/LocalConfigurationProvider.java | 48 +++++ .../src/main/resources/yarn-default.xml | 12 ++ .../server/resourcemanager/AdminService.java | 34 +++- .../scheduler/capacity/CapacityScheduler.java | 14 +- .../CapacitySchedulerConfiguration.java | 13 +- .../resourcemanager/TestRMAdminService.java | 171 ++++++++++++++++++ 11 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 84634df909c..4ef7f01c315 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,10 @@ Release 2.4.0 - UNRELEASED YARN-1633. Defined user-facing entity, entity-info and event objects related to Application Timeline feature. (Zhijie Shen via vinodkv) + YARN-1611. Introduced the concept of a configuration provider which can be + used by ResourceManager to read configuration locally or from remote systems + so as to help RM failover. (Xuan Gong via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java new file mode 100644 index 00000000000..78c34d9de98 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +/** + * Base class to implement ConfigurationProvider. + * Real ConfigurationProvider implementations need to derive from it and + * implement load methods to actually load the configuration. + */ +public abstract class ConfigurationProvider { + + public void init(Configuration conf) throws Exception { + initInternal(conf); + } + + public void close() throws Exception { + closeInternal(); + } + + /** + * Get the configuration. + * @param name The configuration file name + * @return configuration + * @throws YarnException + * @throws IOException + */ + public abstract Configuration getConfiguration(String name) + throws YarnException, IOException; + + /** + * Derived classes initialize themselves using this method. + */ + public abstract void initInternal(Configuration conf) throws Exception; + + /** + * Derived classes close themselves using this method. + */ + public abstract void closeInternal() throws Exception; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java new file mode 100644 index 00000000000..4adc72e1f11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +@Private +@Unstable +/** + * Factory for {@link ConfigurationProvider} implementations. + */ +public class ConfigurationProviderFactory { + /** + * Creates an instance of {@link ConfigurationProvider} using given + * configuration. + * @param conf + * @return configurationProvider + */ + @SuppressWarnings("unchecked") + public static ConfigurationProvider + getConfigurationProvider(Configuration conf) { + Class defaultProviderClass; + try { + defaultProviderClass = (Class) + Class.forName( + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS); + } catch (Exception e) { + throw new YarnRuntimeException( + "Invalid default configuration provider class" + + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS, e); + } + ConfigurationProvider configurationProvider = ReflectionUtils.newInstance( + conf.getClass(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + defaultProviderClass, ConfigurationProvider.class), conf); + return configurationProvider; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 32665d79bb7..a324e97b297 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; @Evolving public class YarnConfiguration extends Configuration { + @Private + public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; @@ -329,6 +332,16 @@ public class YarnConfiguration extends Configuration { public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; public static final String RM_HA_ID = RM_HA_PREFIX + "id"; + /** Store the related configuration files in File System */ + public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX + + "configuration.file-system-based-store"; + public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; + + public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX + + "configuration.provider-class"; + public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = + "org.apache.hadoop.yarn.LocalConfigurationProvider"; + @Private public static final List RM_SERVICES_ADDRESS_CONF_KEYS = Collections.unmodifiableList(Arrays.asList( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java new file mode 100644 index 00000000000..709f54a3529 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +public class FileSystemBasedConfigurationProvider + extends ConfigurationProvider { + + private static final Log LOG = LogFactory + .getLog(FileSystemBasedConfigurationProvider.class); + private FileSystem fs; + private Path configDir; + + @Override + public synchronized Configuration getConfiguration(String name) + throws IOException, YarnException { + Path configPath = new Path(this.configDir, name); + if (!fs.exists(configPath)) { + throw new YarnException("Can not find Configuration: " + name + " in " + + configDir); + } + Configuration conf = new Configuration(false); + conf.addResource(fs.open(configPath)); + return conf; + } + + @Override + public synchronized void initInternal(Configuration conf) throws Exception { + configDir = + new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, + YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE)); + fs = configDir.getFileSystem(conf); + if (!fs.exists(configDir)) { + fs.mkdirs(configDir); + } + } + + @Override + public synchronized void closeInternal() throws Exception { + fs.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java new file mode 100644 index 00000000000..d152c353f08 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +public class LocalConfigurationProvider extends ConfigurationProvider { + + @Override + public Configuration getConfiguration(String name) + throws IOException, YarnException { + return new Configuration(); + } + + @Override + public void initInternal(Configuration conf) throws Exception { + // Do nothing + } + + @Override + public void closeInternal() throws Exception { + // Do nothing + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c4937421f55..beb1d3005cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -588,6 +588,18 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore + + The class to use as the configuration provider. + If org.apache.hadoop.yarn.LocalConfigurationProvider is used, + the local configuration will be loaded. + If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used, + the configuration which will be loaded should be uploaded to remote File system first. + > + yarn.resourcemanager.configuration.provider-class + org.apache.hadoop.yarn.LocalConfigurationProvider + + + The hostname of the NM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 971603a90ed..c7fe0e2e6e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -47,6 +47,8 @@ import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -89,6 +91,8 @@ public class AdminService extends CompositeService implements private InetSocketAddress masterServiceAddress; private AccessControlList adminAcl; + private ConfigurationProvider configurationProvider = null; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -109,6 +113,10 @@ public class AdminService extends CompositeService implements } } + this.configurationProvider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + configurationProvider.init(conf); + masterServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, @@ -129,6 +137,9 @@ public class AdminService extends CompositeService implements @Override protected synchronized void serviceStop() throws Exception { stopServer(); + if (this.configurationProvider != null) { + configurationProvider.close(); + } super.serviceStop(); } @@ -295,23 +306,28 @@ public class AdminService extends CompositeService implements @Override public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshQueues"); + String argName = "refreshQueues"; + UserGroupInformation user = checkAcls(argName); if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh queues."); throwStandbyException(); } + RefreshQueuesResponse response = + recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - rmContext.getScheduler().reinitialize(getConfig(), this.rmContext); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues", + Configuration conf = + getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE); + rmContext.getScheduler().reinitialize(conf, this.rmContext); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); - return recordFactory.newRecordInstance(RefreshQueuesResponse.class); + return response; } catch (IOException ioe) { LOG.info("Exception refreshing queues ", ioe); - RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "Exception refreshing queues"); throw RPCUtil.getRemoteException(ioe); @@ -483,5 +499,9 @@ public class AdminService extends CompositeService implements UpdateNodeResourceResponse.class); return response; } - + + private synchronized Configuration getConfiguration(String confFileName) + throws YarnException, IOException { + return this.configurationProvider.getConfiguration(confFileName); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 0197c5bf998..1b47d69fa2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -195,6 +195,7 @@ public class CapacityScheduler extends AbstractYarnScheduler private ResourceCalculator calculator; private boolean usePortForNodeName; + private boolean useLocalConfigurationProvider; public CapacityScheduler() {} @@ -261,7 +262,13 @@ public class CapacityScheduler extends AbstractYarnScheduler public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.conf = new CapacitySchedulerConfiguration(conf); + this.useLocalConfigurationProvider = conf.get( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( + "org.apache.hadoop.yarn.LocalConfigurationProvider"); + this.conf = + new CapacitySchedulerConfiguration(conf, + this.useLocalConfigurationProvider); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); this.maximumAllocation = this.conf.getMaximumAllocation(); @@ -279,9 +286,10 @@ public class CapacityScheduler extends AbstractYarnScheduler "minimumAllocation=<" + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" + getMaximumResourceCapability() + ">"); } else { - CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = new CapacitySchedulerConfiguration(conf); + this.conf = + new CapacitySchedulerConfiguration(conf, + this.useLocalConfigurationProvider); validateConf(this.conf); try { LOG.info("Re-initializing queues..."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 6fceabf0dec..267f819ffaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -140,10 +140,17 @@ public class CapacitySchedulerConfiguration extends Configuration { } public CapacitySchedulerConfiguration(Configuration configuration) { - super(configuration); - addResource(CS_CONFIGURATION_FILE); + this(configuration, true); } - + + public CapacitySchedulerConfiguration(Configuration configuration, + boolean useLocalConfigurationProvider) { + super(configuration); + if (useLocalConfigurationProvider) { + addResource(CS_CONFIGURATION_FILE); + } + } + private String getQueuePrefix(String queue) { String queueName = PREFIX + queue + DOT; return queueName; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java new file mode 100644 index 00000000000..d800f5067d9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import static org.junit.Assert.fail; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +public class TestRMAdminService { + + private final Configuration configuration = new YarnConfiguration(); + private MockRM rm = null; + private FileSystem fs; + private Path workingPath; + private Path tmpDir; + + @Before + public void setup() throws IOException { + fs = FileSystem.get(configuration); + workingPath = + new Path(new File("target", this.getClass().getSimpleName() + + "-remoteDir").getAbsolutePath()); + configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE, + workingPath.toString()); + tmpDir = new Path(new File("target", this.getClass().getSimpleName() + + "-tmpDir").getAbsolutePath()); + fs.delete(workingPath, true); + fs.delete(tmpDir, true); + fs.mkdirs(workingPath); + fs.mkdirs(tmpDir); + } + + @After + public void tearDown() throws IOException { + if (rm != null) { + rm.stop(); + } + fs.delete(workingPath, true); + fs.delete(tmpDir, true); + } + @Test + public void testAdminRefreshQueuesWithLocalConfigurationProvider() + throws IOException, YarnException { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + CapacityScheduler cs = + (CapacityScheduler) rm.getRMContext().getScheduler(); + int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + Assert.assertEquals(maxAppsBefore, cs.getConfiguration() + .getMaximumSystemApplications()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + CapacityScheduler cs = + (CapacityScheduler) rm.getRMContext().getScheduler(); + int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: capacity-scheduler.xml")); + } + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); + String csConfFile = writeConfigurationXML(csConf, + "capacity-scheduler.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + + int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications(); + Assert.assertEquals(maxAppsAfter, 5000); + Assert.assertTrue(maxAppsAfter != maxAppsBefore); + } + + private String writeConfigurationXML(Configuration conf, String confXMLName) + throws IOException { + DataOutputStream output = null; + try { + final File confFile = new File(tmpDir.toString(), confXMLName); + if (confFile.exists()) { + confFile.delete(); + } + if (!confFile.createNewFile()) { + Assert.fail("Can not create " + confXMLName); + } + output = new DataOutputStream( + new FileOutputStream(confFile)); + conf.writeXml(output); + return confFile.getAbsolutePath(); + } finally { + if (output != null) { + output.close(); + } + } + } + + private void uploadToRemoteFileSystem(Path filePath) + throws IOException { + fs.copyFromLocalFile(filePath, workingPath); + } + + private void cleanRemoteDirectory() throws IOException { + if (fs.exists(workingPath)) { + for (FileStatus file : fs.listStatus(workingPath)) { + fs.delete(file.getPath(), true); + } + } + } +} From cb5e0787a6fc0b0748753b7e7c4c3fdbfd2714b2 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Mon, 3 Feb 2014 19:30:55 +0000 Subject: [PATCH 08/54] YARN-1498 addendum to fix findbugs warning git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564018 13f79535-47bb-0310-9956-ffa450edef68 --- .../resourcemanager/scheduler/SchedulerApplicationAttempt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 7e7fdb765e1..7785e56bdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -432,7 +432,7 @@ public class SchedulerApplicationAttempt { .transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo); } - public void move(Queue newQueue) { + public synchronized void move(Queue newQueue) { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String user = getUser(); From 3e7d56678c334a359049adf99018683113849fc8 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 19:53:46 +0000 Subject: [PATCH 09/54] YARN-1639. Modified RM HA configuration handling to have a way of not requiring separate configuration files for each RM. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564032 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 49 ++++++++++++++++--- .../src/main/resources/yarn-default.xml | 9 +++- .../yarn/server/resourcemanager/TestRMHA.java | 41 ++++++++++++++-- 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4ef7f01c315..022c180b9c4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -103,6 +103,9 @@ Release 2.4.0 - UNRELEASED YARN-1617. Remove ancient comment and surround LOG.debug in AppSchedulingInfo.allocate (Sandy Ryza) + YARN-1639. Modified RM HA configuration handling to have a way of not + requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index 3cbde93a3e9..ec2c64b5c2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -21,10 +21,13 @@ package org.apache.hadoop.yarn.conf; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import java.net.InetSocketAddress; import java.util.Collection; @InterfaceAudience.Private @@ -108,8 +111,7 @@ public class HAUtil { String errmsg = iae.getMessage(); if (confKey == null) { // Error at addSuffix - errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, - getRMHAId(conf)); + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id); } throwBadConfigurationException(errmsg); } @@ -122,10 +124,18 @@ public class HAUtil { } private static void verifyAndSetCurrentRMHAId(Configuration conf) { - String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + String rmId = getRMHAId(conf); if (rmId == null) { - throwBadConfigurationException( - getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID)); + StringBuilder msg = new StringBuilder(); + msg.append("Can not find valid RM_HA_ID. None of "); + for (String id : conf + .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { + msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " "); + } + msg.append(" are matching" + + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + + " specified in HA Configuration"); + throwBadConfigurationException(msg.toString()); } else { Collection ids = getRMHAIds(conf); if (!ids.contains(rmId)) { @@ -179,7 +189,34 @@ public class HAUtil { * @return RM Id on success */ public static String getRMHAId(Configuration conf) { - return conf.get(YarnConfiguration.RM_HA_ID); + int found = 0; + String currentRMId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + if(currentRMId == null) { + for(String rmId : getRMHAIds(conf)) { + String key = addSuffix(YarnConfiguration.RM_ADDRESS, rmId); + String addr = conf.get(key); + if (addr == null) { + continue; + } + InetSocketAddress s; + try { + s = NetUtils.createSocketAddr(addr); + } catch (Exception e) { + LOG.warn("Exception in creating socket address " + addr, e); + continue; + } + if (!s.isUnresolved() && NetUtils.isLocalAddress(s.getAddress())) { + currentRMId = rmId.trim(); + found++; + } + } + } + if (found > 1) { // Only one address must match the local address + String msg = "The HA Configuration has multiple addresses that match " + + "local node's address."; + throw new HadoopIllegalArgumentException(msg); + } + return currentRMId; } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index beb1d3005cb..78ceb64b3d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -395,7 +395,9 @@ the Active mode when prompted to. (2) The nodes in the RM ensemble are listed in yarn.resourcemanager.ha.rm-ids - (3) The id of each RM comes from yarn.resourcemanager.ha.id + (3) The id of each RM either comes from yarn.resourcemanager.ha.id + if yarn.resourcemanager.ha.id is explicitly specified or can be + figured out by matching yarn.resourcemanager.address.{id} with local address (4) The actual physical addresses come from the configs of the pattern - {rpc-config}.{id} yarn.resourcemanager.ha.enabled @@ -442,7 +444,10 @@ The id (string) of the current RM. When HA is enabled, this - is a required config. See description of yarn.resourcemanager.ha.enabled + is an optional config. The id of current RM can be set by explicitly + specifying yarn.resourcemanager.ha.id or figured out by matching + yarn.resourcemanager.address.{id} with local address + See description of yarn.resourcemanager.ha.enabled for full details on how this is used. yarn.resourcemanager.ha.id diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 9d1a46776c5..5b4f5709d75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -36,6 +36,8 @@ import org.junit.Test; import java.io.IOException; +import junit.framework.Assert; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -48,12 +50,15 @@ public class TestRMHA { private static final String STATE_ERR = "ResourceManager is in wrong HA state"; - private static final String RM1_ADDRESS = "0.0.0.0:0"; + private static final String RM1_ADDRESS = "1.1.1.1:1"; private static final String RM1_NODE_ID = "rm1"; - private static final String RM2_ADDRESS = "1.1.1.1:1"; + private static final String RM2_ADDRESS = "0.0.0.0:0"; private static final String RM2_NODE_ID = "rm2"; + private static final String RM3_ADDRESS = "2.2.2.2:2"; + private static final String RM3_NODE_ID = "rm3"; + @Before public void setUp() throws Exception { configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); @@ -61,8 +66,8 @@ public class TestRMHA { for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } - configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); } private void checkMonitorHealth() throws IOException { @@ -278,6 +283,36 @@ public class TestRMHA { rm.stop(); } + @Test + public void testHAIDLookup() { + //test implicitly lookup HA-ID + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM2_NODE_ID); + + //test explicitly lookup HA-ID + configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM1_NODE_ID); + + //test if RM_HA_ID can not be found + configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID+ "," + RM3_NODE_ID); + configuration.unset(YarnConfiguration.RM_HA_ID); + conf = new YarnConfiguration(configuration); + try { + rm = new MockRM(conf); + rm.init(conf); + fail("Should get an exception here."); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Invalid configuration! Can not find valid RM_HA_ID.")); + } + } + @SuppressWarnings("rawtypes") class MyCountingDispatcher extends AbstractService implements Dispatcher { From eac066b13f1d2882415d6888d95d9fe87c665dcc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 20:07:01 +0000 Subject: [PATCH 10/54] YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for implementing different storage impls for storing timeline information. Contributed by Billie Rinaldi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564037 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../api/records/apptimeline/ATSEntity.java | 41 +++-- .../api/records/apptimeline/ATSEvent.java | 40 ++++- .../api/records/apptimeline/ATSEvents.java | 2 +- .../api/records/apptimeline/ATSPutErrors.java | 163 ++++++++++++++++++ .../TestApplicationTimelineRecords.java | 43 ++++- .../ApplicationTimelineReader.java | 125 ++++++++++++++ .../apptimeline/ApplicationTimelineStore.java | 29 ++++ .../ApplicationTimelineWriter.java | 43 +++++ .../apptimeline/NameValuePair.java | 59 +++++++ .../apptimeline/package-info.java | 20 +++ 11 files changed, 541 insertions(+), 28 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 022c180b9c4..b07ecd77144 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -86,6 +86,10 @@ Release 2.4.0 - UNRELEASED used by ResourceManager to read configuration locally or from remote systems so as to help RM failover. (Xuan Gong via vinodkv) + YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for + implementing different storage impls for storing timeline information. + (Billie Rinaldi via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java index 1884db7ac13..6b3ea1013e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -53,10 +54,10 @@ public class ATSEntity { private String entityType; private String entityId; - private long startTime; + private Long startTime; private List events = new ArrayList(); - private Map> relatedEntities = - new HashMap>(); + private Map> relatedEntities = + new HashMap>(); private Map primaryFilters = new HashMap(); private Map otherInfo = @@ -112,7 +113,7 @@ public class ATSEntity { * @return the start time of the entity */ @XmlElement(name = "starttime") - public long getStartTime() { + public Long getStartTime() { return startTime; } @@ -122,7 +123,7 @@ public class ATSEntity { * @param startTime * the start time of the entity */ - public void setStartTime(long startTime) { + public void setStartTime(Long startTime) { this.startTime = startTime; } @@ -172,26 +173,25 @@ public class ATSEntity { * @return the related entities */ @XmlElement(name = "relatedentities") - public Map> getRelatedEntities() { + public Map> getRelatedEntities() { return relatedEntities; } /** - * Add a list of entity of the same type to the existing related entity map + * Add an entity to the existing related entity map * * @param entityType * the entity type - * @param entityIds - * a list of entity Ids + * @param entityId + * the entity Id */ - public void addRelatedEntity(String entityType, List entityIds) { - List thisRelatedEntity = relatedEntities.get(entityType); - relatedEntities.put(entityType, entityIds); + public void addRelatedEntity(String entityType, String entityId) { + List thisRelatedEntity = relatedEntities.get(entityType); if (thisRelatedEntity == null) { - relatedEntities.put(entityType, entityIds); - } else { - thisRelatedEntity.addAll(entityIds); + thisRelatedEntity = new ArrayList(); + relatedEntities.put(entityType, thisRelatedEntity); } + thisRelatedEntity.add(entityId); } /** @@ -200,11 +200,10 @@ public class ATSEntity { * @param relatedEntities * a map of related entities */ - public void addRelatedEntities( - Map> relatedEntities) { - for (Map.Entry> relatedEntity : relatedEntities - .entrySet()) { - List thisRelatedEntity = + public void addRelatedEntities(Map> relatedEntities) { + for (Entry> relatedEntity : + relatedEntities.entrySet()) { + List thisRelatedEntity = this.relatedEntities.get(relatedEntity.getKey()); if (thisRelatedEntity == null) { this.relatedEntities.put( @@ -222,7 +221,7 @@ public class ATSEntity { * a map of related entities */ public void setRelatedEntities( - Map> relatedEntities) { + Map> relatedEntities) { this.relatedEntities = relatedEntities; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java index 6477a578e6a..27bac16a69f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java @@ -39,7 +39,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEvent { +public class ATSEvent implements Comparable { private long timestamp; private String eventType; @@ -131,4 +131,42 @@ public class ATSEvent { this.eventInfo = eventInfo; } + @Override + public int compareTo(ATSEvent other) { + if (timestamp > other.timestamp) { + return -1; + } else if (timestamp < other.timestamp) { + return 1; + } else { + return eventType.compareTo(other.eventType); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ATSEvent atsEvent = (ATSEvent) o; + + if (timestamp != atsEvent.timestamp) + return false; + if (!eventType.equals(atsEvent.eventType)) + return false; + if (eventInfo != null ? !eventInfo.equals(atsEvent.eventInfo) : + atsEvent.eventInfo != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + eventType.hashCode(); + result = 31 * result + (eventInfo != null ? eventInfo.hashCode() : 0); + return result; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java index da7fd280886..a08537d7203 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java @@ -160,7 +160,7 @@ public class ATSEvents { * @param event * a single event */ - public void addEntity(ATSEvent event) { + public void addEvent(ATSEvent event) { events.add(event); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java new file mode 100644 index 00000000000..91458e1419f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * A class that holds a list of put errors. This is the response returned + * when a list of {@link ATSEntity} objects is added to the application + * timeline. If there are errors in storing individual entity objects, + * they will be indicated in the list of errors. + */ +@XmlRootElement(name = "errors") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSPutErrors { + + private List errors = new ArrayList(); + + public ATSPutErrors() { + + } + + /** + * Get a list of {@link ATSPutError} instances + * + * @return a list of {@link ATSPutError} instances + */ + @XmlElement(name = "errors") + public List getErrors() { + return errors; + } + + /** + * Add a single {@link ATSPutError} instance into the existing list + * + * @param error + * a single {@link ATSPutError} instance + */ + public void addError(ATSPutError error) { + errors.add(error); + } + + /** + * Add a list of {@link ATSPutError} instances into the existing list + * + * @param errors + * a list of {@link ATSPutError} instances + */ + public void addErrors(List errors) { + this.errors.addAll(errors); + } + + /** + * Set the list to the given list of {@link ATSPutError} instances + * + * @param errors + * a list of {@link ATSPutError} instances + */ + public void setErrors(List errors) { + this.errors.clear(); + this.errors.addAll(errors); + } + + /** + * A class that holds the error code for one entity. + */ + @XmlRootElement(name = "error") + @XmlAccessorType(XmlAccessType.NONE) + @Public + @Unstable + public static class ATSPutError { + private String entityId; + private String entityType; + private Integer errorCode; + + /** + * Get the entity Id + * + * @return the entity Id + */ + @XmlElement(name = "entity") + public String getEntityId() { + return entityId; + } + + /** + * Set the entity Id + * + * @param entityId + * the entity Id + */ + public void setEntityId(String entityId) { + this.entityId = entityId; + } + + /** + * Get the entity type + * + * @return the entity type + */ + @XmlElement(name = "entitytype") + public String getEntityType() { + return entityType; + } + + /** + * Set the entity type + * + * @param entityType + * the entity type + */ + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + /** + * Get the error code + * + * @return an error code + */ + @XmlElement(name = "errorcode") + public Integer getErrorCode() { + return errorCode; + } + + /** + * Set the error code to the given error code + * + * @param errorCode + * an error code + */ + public void setErrorCode(Integer errorCode) { + this.errorCode = errorCode; + } + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java index fe79e74eb6c..f2a6d3ef461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.api.records.apptimeline; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; import org.junit.Test; public class TestApplicationTimelineRecords { @@ -42,10 +45,8 @@ public class TestApplicationTimelineRecords { event.addEventInfo("key2", "val2"); entity.addEvent(event); } - entity.addRelatedEntity( - "test ref type 1", Arrays.asList((Object) "test ref id 1")); - entity.addRelatedEntity( - "test ref type 2", Arrays.asList((Object) "test ref id 2")); + entity.addRelatedEntity("test ref type 1", "test ref id 1"); + entity.addRelatedEntity("test ref type 2", "test ref id 2"); entity.addPrimaryFilter("pkey1", "pval1"); entity.addPrimaryFilter("pkey2", "pval2"); entity.addOtherInfo("okey1", "oval1"); @@ -83,7 +84,7 @@ public class TestApplicationTimelineRecords { event.setEventType("event type " + i); event.addEventInfo("key1", "val1"); event.addEventInfo("key2", "val2"); - partEvents.addEntity(event); + partEvents.addEvent(event); } events.addEvent(partEvents); } @@ -110,4 +111,36 @@ public class TestApplicationTimelineRecords { Assert.assertEquals(2, event22.getEventInfo().size()); } + @Test + public void testATSPutErrors() { + ATSPutErrors atsPutErrors = new ATSPutErrors(); + ATSPutError error1 = new ATSPutError(); + error1.setEntityId("entity id 1"); + error1.setEntityId("entity type 1"); + error1.setErrorCode(1); + atsPutErrors.addError(error1); + List errors = new ArrayList(); + errors.add(error1); + ATSPutError error2 = new ATSPutError(); + error2.setEntityId("entity id 2"); + error2.setEntityId("entity type 2"); + error2.setErrorCode(2); + errors.add(error2); + atsPutErrors.addErrors(errors); + + Assert.assertEquals(3, atsPutErrors.getErrors().size()); + ATSPutError e = atsPutErrors.getErrors().get(0); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(1); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(2); + Assert.assertEquals(error2.getEntityId(), e.getEntityId()); + Assert.assertEquals(error2.getEntityType(), e.getEntityType()); + Assert.assertEquals(error2.getErrorCode(), e.getErrorCode()); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java new file mode 100644 index 00000000000..97a217dc98a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; + +/** + * This interface is for retrieving application timeline information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineReader { + + /** + * Possible fields to retrieve for {@link #getEntities} and {@link + * #getEntity}. + */ + enum Field { + EVENTS, + RELATED_ENTITIES, + PRIMARY_FILTERS, + OTHER_INFO, + LAST_EVENT_ONLY + } + + /** + * Default limit for {@link #getEntities} and {@link #getEntityTimelines}. + */ + final long DEFAULT_LIMIT = 100; + + /** + * This method retrieves a list of entity information, {@link ATSEntity}, + * sorted by the starting timestamp for the entity, descending. + * + * @param entityType The type of entities to return (required). + * @param limit A limit on the number of entities to return. If null, + * defaults to {@link #DEFAULT_LIMIT}. + * @param windowStart The earliest start timestamp to retrieve (exclusive). + * If null, defaults to retrieving all entities until the + * limit is reached. + * @param windowEnd The latest start timestamp to retrieve (inclusive). + * If null, defaults to {@link Long#MAX_VALUE} + * @param primaryFilter Retrieves only entities that have the specified + * primary filter. If null, retrieves all entities. + * This is an indexed retrieval, and no entities that + * do not match the filter are scanned. + * @param secondaryFilters Retrieves only entities that have exact matches + * for all the specified filters in their primary + * filters or other info. This is not an indexed + * retrieval, so all entities are scanned but only + * those matching the filters are returned. + * @param fieldsToRetrieve Specifies which fields of the entity object to + * retrieve (see {@link Field}). If the set of fields + * contains {@link Field#LAST_EVENT_ONLY} and not + * {@link Field#EVENTS}, the most recent event for + * each entity is retrieved. + * @return An {@link ATSEntities} object. + */ + ATSEntities getEntities(String entityType, + Long limit, Long windowStart, Long windowEnd, + NameValuePair primaryFilter, Collection secondaryFilters, + EnumSet fieldsToRetrieve); + + /** + * This method retrieves the entity information for a given entity. + * + * @param entity The entity whose information will be retrieved. + * @param entityType The type of the entity. + * @param fieldsToRetrieve Specifies which fields of the entity object to + * retrieve (see {@link Field}). If the set of + * fields contains {@link Field#LAST_EVENT_ONLY} and + * not {@link Field#EVENTS}, the most recent event + * for each entity is retrieved. + * @return An {@link ATSEntity} object. + */ + ATSEntity getEntity(String entity, String entityType, EnumSet + fieldsToRetrieve); + + /** + * This method retrieves the events for a list of entities all of the same + * entity type. The events for each entity are sorted in order of their + * timestamps, descending. + * + * @param entityType The type of entities to retrieve events for. + * @param entityIds The entity IDs to retrieve events for. + * @param limit A limit on the number of events to return for each entity. + * If null, defaults to {@link #DEFAULT_LIMIT} events per + * entity. + * @param windowStart If not null, retrieves only events later than the + * given time (exclusive) + * @param windowEnd If not null, retrieves only events earlier than the + * given time (inclusive) + * @param eventTypes Restricts the events returned to the given types. If + * null, events of all types will be returned. + * @return An {@link ATSEvents} object. + */ + ATSEvents getEntityTimelines(String entityType, + SortedSet entityIds, Long limit, Long windowStart, + Long windowEnd, Set eventTypes); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java new file mode 100644 index 00000000000..b231418a89f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.service.Service; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineStore extends + Service, ApplicationTimelineReader, ApplicationTimelineWriter { +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java new file mode 100644 index 00000000000..b7bd0708e43 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; + +/** + * This interface is for storing application timeline information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineWriter { + + /** + * Stores entity information to the application timeline store. Any errors + * occurring for individual put request objects will be reported in the + * response. + * + * @param data An {@link ATSEntities} object. + * @return An {@link ATSPutErrors} object. + */ + ATSPutErrors put(ATSEntities data); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java new file mode 100644 index 00000000000..66a21bbf658 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A class holding a name and value pair, used for specifying filters in + * {@link ApplicationTimelineReader}. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class NameValuePair { + String name; + Object value; + + public NameValuePair(String name, Object value) { + this.name = name; + this.value = value; + } + + /** + * Get the name. + * @return The name. + */ + public String getName() { + + return name; + } + + /** + * Get the value. + * @return The value. + */ + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "{ name: " + name + ", value: " + value + " }"; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java new file mode 100644 index 00000000000..c3aaafe6747 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; +import org.apache.hadoop.classification.InterfaceAudience; From eff1e809f284d50ee49bb9ed2acea52331838c5e Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 20:16:28 +0000 Subject: [PATCH 11/54] YARN-1668. Modified RM HA handling of admin-acls to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564043 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 4 +- .../server/resourcemanager/AdminService.java | 22 ++++++-- .../resourcemanager/TestRMAdminService.java | 56 ++++++++++++++++++- 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b07ecd77144..9a947f713cd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -110,6 +110,10 @@ Release 2.4.0 - UNRELEASED YARN-1639. Modified RM HA configuration handling to have a way of not requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + YARN-1668. Modified RM HA handling of admin-acls to be available across RM + failover by making using of a remote configuration-provider. (Xuan Gong via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a324e97b297..af385f81db2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -40,8 +40,10 @@ public class YarnConfiguration extends Configuration { @Private public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + @Private + public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; static { Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index c7fe0e2e6e9..9a33b706f90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRespo import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; public class AdminService extends CompositeService implements @@ -407,14 +408,22 @@ public class AdminService extends CompositeService implements @Override public RefreshAdminAclsResponse refreshAdminAcls( - RefreshAdminAclsRequest request) throws YarnException { - UserGroupInformation user = checkAcls("refreshAdminAcls"); + RefreshAdminAclsRequest request) throws YarnException, IOException { + String argName = "refreshAdminAcls"; + UserGroupInformation user = checkAcls(argName); - Configuration conf = new Configuration(); + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh user-groups."); + throwStandbyException(); + } + Configuration conf = + getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls", + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class); @@ -504,4 +513,9 @@ public class AdminService extends CompositeService implements throws YarnException, IOException { return this.configurationProvider.getConfiguration(confFileName); } + + @VisibleForTesting + public AccessControlList getAccessControlList() { + return this.adminAcl; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index d800f5067d9..4b7018528fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -50,6 +51,7 @@ public class TestRMAdminService { @Before public void setup() throws IOException { + Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); fs = FileSystem.get(configuration); workingPath = new Path(new File("target", this.getClass().getSimpleName() @@ -72,6 +74,7 @@ public class TestRMAdminService { fs.delete(workingPath, true); fs.delete(tmpDir, true); } + @Test public void testAdminRefreshQueuesWithLocalConfigurationProvider() throws IOException, YarnException { @@ -95,7 +98,6 @@ public class TestRMAdminService { @Test public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { - Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); rm = new MockRM(configuration); @@ -134,6 +136,58 @@ public class TestRMAdminService { Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + @Test + public void testAdminAclsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testAdminAclsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: yarn-site.xml")); + } + + String aclStringBefore = + rm.adminService.getAccessControlList().getAclString().trim(); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); + String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(yarnConfFile)); + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + + String aclStringAfter = + rm.adminService.getAccessControlList().getAclString().trim(); + + Assert.assertTrue(!aclStringAfter.equals(aclStringBefore)); + Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; From 24fa232707b56f6ccc88a460dc0d708d35a28ff4 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 22:10:56 +0000 Subject: [PATCH 12/54] YARN-1667. Modified RM HA handling of super users (with proxying ability) to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564100 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/security/authorize/ProxyUsers.java | 11 ++++ hadoop-yarn-project/CHANGES.txt | 33 ++++++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../server/resourcemanager/AdminService.java | 15 ++--- .../resourcemanager/TestRMAdminService.java | 61 +++++++++++++++++++ 5 files changed, 116 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java index 341285e1a75..52952588739 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java @@ -30,6 +30,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class ProxyUsers { @@ -177,4 +179,13 @@ public class ProxyUsers { (list.contains("*")); } + @VisibleForTesting + public static Map> getProxyGroups() { + return proxyGroups; + } + + @VisibleForTesting + public static Map> getProxyHosts() { + return proxyHosts; + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9a947f713cd..e4b8900843a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -114,6 +114,39 @@ Release 2.4.0 - UNRELEASED failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1667. Modified RM HA handling of super users (with proxying ability) to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + + OPTIMIZATIONS + + BUG FIXES + + YARN-935. Correcting pom.xml to build applicationhistoryserver module + successfully. (Zhijie Shen via vinodkv) + + YARN-962. Fixed bug in application-history proto file and renamed it be just + a client proto file. (Zhijie Shen via vinodkv) + + YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl + package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv) + + YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) + + YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar + Vavilapalli via mayank) + + YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to + YARN-888. (Vinod Kumar Vavilapalli via zjshen) + + YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1595. Made enabling history service configurable and fixed test failures on + branch YARN-321. (Vinod Kumar Vavilapalli via zjshen) OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index af385f81db2..919ed901439 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -43,6 +43,9 @@ public class YarnConfiguration extends Configuration { @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + @Private + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; static { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 9a33b706f90..3bfd47d1373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -363,21 +363,22 @@ public class AdminService extends CompositeService implements @Override public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration"); + throws YarnException, IOException { + String argName = "refreshSuperUserGroupsConfiguration"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh super-user-groups."); throwStandbyException(); } - ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration()); + Configuration conf = + getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", "AdminService"); + argName, "AdminService"); return recordFactory.newRecordInstance( RefreshSuperUserGroupsConfigurationResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 4b7018528fe..797b4226842 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -29,10 +29,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -188,6 +190,65 @@ public class TestRMAdminService { Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); } + @Test + public void + testRefreshSuperUserGroupsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: core-site.xml")); + } + + Configuration coreConf = new Configuration(false); + coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); + coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); + String coreConfFile = writeConfigurationXML(coreConf, + "core-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(coreConfFile)); + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").contains("test_groups")); + + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; From 60c0fe8f6d9fcd1bbfeef2f81afe490cbabad4a7 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Tue, 4 Feb 2014 00:11:54 +0000 Subject: [PATCH 13/54] HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry McCay via omalley) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564137 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/crypto/key/TestKeyShell.java | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d4db1854569..06d97b1428e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -113,6 +113,9 @@ Trunk (Unreleased) HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley) + HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry + McCay via omalley) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java index 2d2f7ee431b..a05e9bbc2b5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java @@ -41,7 +41,7 @@ public class TestKeyShell { @Test public void testKeySuccessfulKeyLifecycle() throws Exception { - outContent.flush(); + outContent.reset(); String[] args1 = {"create", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; int rc = 0; @@ -52,14 +52,14 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "created.")); - outContent.flush(); + outContent.reset(); String[] args2 = {"list", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args2); assertEquals(0, rc); assertTrue(outContent.toString().contains("key1")); - outContent.flush(); + outContent.reset(); String[] args3 = {"roll", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args3); @@ -67,7 +67,7 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "rolled.")); - outContent.flush(); + outContent.reset(); String[] args4 = {"delete", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args4); @@ -75,12 +75,12 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "deleted.")); - outContent.flush(); + outContent.reset(); String[] args5 = {"list", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args5); assertEquals(0, rc); - assertTrue(outContent.toString().contains("key1")); + assertFalse(outContent.toString(), outContent.toString().contains("key1")); } @Test @@ -165,7 +165,7 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "created.")); - outContent.flush(); + outContent.reset(); String[] args2 = {"delete", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args2); From 6439cd0f691069cefb6da4ba261ffe60cc13bbd0 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 4 Feb 2014 00:27:25 +0000 Subject: [PATCH 14/54] HDFS-5767. NFS implementation assumes userName userId mapping to be unique, which is not true sometimes. Contributed by Yongjun Zhang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564141 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 4 +- .../apache/hadoop/nfs/nfs3/IdUserGroup.java | 52 +++++++++++-------- .../hadoop/nfs/nfs3/TestIdUserGroup.java | 41 +++++++++------ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 10fcdb785e7..7143b514060 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -425,9 +425,9 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." - #There are 14 warnings that are caused by things that are caused by using sun internal APIs. + #There are 12 warnings that are caused by things that are caused by using sun internal APIs. #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. - OK_JAVADOC_WARNINGS=16; + OK_JAVADOC_WARNINGS=14; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java index a1d48aadc85..bf2b542d85b 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java @@ -50,14 +50,6 @@ public class IdUserGroup { private BiMap gidNameMap = HashBiMap.create(); private long lastUpdateTime = 0; // Last time maps were updated - - static public class DuplicateNameOrIdException extends IOException { - private static final long serialVersionUID = 1L; - - public DuplicateNameOrIdException(String msg) { - super(msg); - } - } public IdUserGroup() throws IOException { updateMaps(); @@ -80,7 +72,8 @@ public class IdUserGroup { } } - private static final String DUPLICATE_NAME_ID_DEBUG_INFO = "NFS gateway can't start with duplicate name or id on the host system.\n" + private static final String DUPLICATE_NAME_ID_DEBUG_INFO = + "NFS gateway could have problem starting with duplicate name or id on the host system.\n" + "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n" + "The host system with duplicated user/group name or id might work fine most of the time by itself.\n" + "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n" @@ -88,6 +81,16 @@ public class IdUserGroup { + " and on Linux systms,\n" + " and on MacOS."; + private static void reportDuplicateEntry(final String header, + final Integer key, final String value, + final Integer ekey, final String evalue) { + LOG.warn("\n" + header + String.format( + "new entry (%d, %s), existing entry: (%d, %s).\n%s\n%s", + key, value, ekey, evalue, + "The new entry is to be ignored for the following reason.", + DUPLICATE_NAME_ID_DEBUG_INFO)); + } + /** * Get the whole list of users and groups and save them in the maps. * @throws IOException @@ -108,22 +111,27 @@ public class IdUserGroup { } LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); // HDFS can't differentiate duplicate names with simple authentication - Integer key = Integer.valueOf(nameId[1]); - String value = nameId[0]; + final Integer key = Integer.valueOf(nameId[1]); + final String value = nameId[0]; if (map.containsKey(key)) { - LOG.error(String.format( - "Got duplicate id:(%d, %s), existing entry: (%d, %s).\n%s", key, - value, key, map.get(key), DUPLICATE_NAME_ID_DEBUG_INFO)); - throw new DuplicateNameOrIdException("Got duplicate id."); + final String prevValue = map.get(key); + if (value.equals(prevValue)) { + // silently ignore equivalent entries + continue; + } + reportDuplicateEntry( + "Got multiple names associated with the same id: ", + key, value, key, prevValue); + continue; } - if (map.containsValue(nameId[0])) { - LOG.error(String.format( - "Got duplicate name:(%d, %s), existing entry: (%d, %s) \n%s", - key, value, map.inverse().get(value), value, - DUPLICATE_NAME_ID_DEBUG_INFO)); - throw new DuplicateNameOrIdException("Got duplicate name"); + if (map.containsValue(value)) { + final Integer prevKey = map.inverse().get(value); + reportDuplicateEntry( + "Got multiple ids associated with the same name: ", + key, value, prevKey, value); + continue; } - map.put(Integer.valueOf(nameId[1]), nameId[0]); + map.put(key, value); } LOG.info("Updated " + mapName + " map size:" + map.size()); diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java index db2b27016e4..4331238b166 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java @@ -17,11 +17,10 @@ */ package org.apache.hadoop.nfs.nfs3; -import static org.junit.Assert.fail; - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; -import org.apache.hadoop.nfs.nfs3.IdUserGroup.DuplicateNameOrIdException; import org.junit.Test; import com.google.common.collect.BiMap; @@ -33,24 +32,36 @@ public class TestIdUserGroup { public void testDuplicates() throws IOException { String GET_ALL_USERS_CMD = "echo \"root:x:0:0:root:/root:/bin/bash\n" + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" - + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\"" + + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "hdfs1:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "hdfs2:x:11502:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "bin:x:2:2:bin:/bin:/bin/sh\n" + + "bin:x:1:1:bin:/bin:/sbin/nologin\n" + + "daemon:x:1:1:daemon:/usr/sbin:/bin/sh\n" + + "daemon:x:2:2:daemon:/sbin:/sbin/nologin\"" + " | cut -d: -f1,3"; String GET_ALL_GROUPS_CMD = "echo \"hdfs:*:11501:hrt_hdfs\n" - + "mapred:x:497\n" + "mapred2:x:497\"" + " | cut -d: -f1,3"; + + "mapred:x:497\n" + + "mapred2:x:497\n" + + "mapred:x:498\n" + + "mapred3:x:498\"" + + " | cut -d: -f1,3"; // Maps for id to name map BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - try { - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); - fail("didn't detect the duplicate name"); - } catch (DuplicateNameOrIdException e) { - } + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); + assertTrue(uMap.size() == 5); + assertEquals(uMap.get(0), "root"); + assertEquals(uMap.get(11501), "hdfs"); + assertEquals(uMap.get(11502), "hdfs2"); + assertEquals(uMap.get(2), "bin"); + assertEquals(uMap.get(1), "daemon"); - try { - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); - fail("didn't detect the duplicate id"); - } catch (DuplicateNameOrIdException e) { - } + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); + assertTrue(gMap.size() == 3); + assertEquals(gMap.get(11501), "hdfs"); + assertEquals(gMap.get(497), "mapred"); + assertEquals(gMap.get(498), "mapred3"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0b9caec75b6..f9abed76c5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -328,6 +328,9 @@ Release 2.4.0 - UNRELEASED the same node group when dfs.namenode.avoid.write.stale.datanode is true. (Buddy via junping_du) + HDFS-5767. NFS implementation assumes userName userId mapping to be unique, + which is not true sometimes (Yongjun Zhang via brandonli) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES From dd7fa037192d7a64bc29fc184c080efbbb14c9cb Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:05:58 +0000 Subject: [PATCH 15/54] Fix YARN's CHANGES.txt issue caused during the commit for YARN-1667. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564185 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e4b8900843a..7d7542dfbd5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -118,37 +118,6 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) - OPTIMIZATIONS - - BUG FIXES - - YARN-935. Correcting pom.xml to build applicationhistoryserver module - successfully. (Zhijie Shen via vinodkv) - - YARN-962. Fixed bug in application-history proto file and renamed it be just - a client proto file. (Zhijie Shen via vinodkv) - - YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl - package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv) - - YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) - - YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar - Vavilapalli via mayank) - - YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to - YARN-888. (Vinod Kumar Vavilapalli via zjshen) - - YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli - via zjshen) - - YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli - via zjshen) - - YARN-1595. Made enabling history service configurable and fixed test failures on - branch YARN-321. (Vinod Kumar Vavilapalli via zjshen) - OPTIMIZATIONS - BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module From a8c780d378df86aafba09751c0c43dd4e0d54c0a Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:08:09 +0000 Subject: [PATCH 16/54] YARN-1673. Fix option parsing in YARN's application CLI after it is broken by YARN-967. Contributed by Mayank Bansal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564188 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/yarn/client/cli/ApplicationCLI.java | 2 +- .../org/apache/hadoop/yarn/client/cli/TestYarnCLI.java | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7d7542dfbd5..4110dfe7421 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.4.0 - UNRELEASED YARN-1632. TestApplicationMasterServices should be under org.apache.hadoop.yarn.server.resourcemanager package (Chen He via jeagles) + YARN-1673. Fix option parsing in YARN's application CLI after it is broken + by YARN-967. (Mayank Bansal via vinodkv) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 9b465b78b28..d520866e51c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -197,7 +197,7 @@ public class ApplicationCLI extends YarnCLI { listApplications(appTypes, appStates); } } else if (cliParser.hasOption(KILL_CMD)) { - if (args.length != 2) { + if (args.length != 3) { printUsage(opts); return exitCode; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index dd6be0d8a57..48ac5484893 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -681,7 +681,7 @@ public class TestYarnCLI { sysOutStream.reset(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); result = - cli.run(new String[] { "-kill", applicationId.toString(), "args" }); + cli.run(new String[] {"application", "-kill", applicationId.toString(), "args" }); verify(spyCli).printUsage(any(Options.class)); Assert.assertEquals(createApplicationCLIHelpMessage(), sysOutStream.toString()); @@ -717,7 +717,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); - int result = cli.run(new String[] { "-kill", applicationId.toString() }); + int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client, times(0)).killApplication(any(ApplicationId.class)); verify(sysOut).println( @@ -730,7 +730,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); - result = cli.run(new String[] { "-kill", applicationId.toString() }); + result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client).killApplication(any(ApplicationId.class)); verify(sysOut).println("Killing application application_1234_0005"); @@ -740,7 +740,7 @@ public class TestYarnCLI { .getApplicationReport(applicationId); cli = createAndGetAppCLI(); try { - cli.run(new String[] { "-kill", applicationId.toString() }); + cli.run(new String[] { "application","-kill", applicationId.toString() }); Assert.fail(); } catch (Exception ex) { Assert.assertTrue(ex instanceof ApplicationNotFoundException); From 39ecc46ff6c4d11175f6995289e9697bc23436c5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:21:40 +0000 Subject: [PATCH 17/54] YARN-1285. Changed the default value of yarn.acl.enable in yarn-default.xml to be consistent with what exists (false) in the code and documentation. Contributed by Kenji Kikushima. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564190 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4110dfe7421..cf0a269047e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -118,6 +118,10 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1285. Changed the default value of yarn.acl.enable in yarn-default.xml + to be consistent with what exists (false) in the code and documentation. + (Kenji Kikushima via vinodkv) + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 78ceb64b3d3..a5906986665 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -130,7 +130,7 @@ Are acls enabled. yarn.acl.enable - true + false From 786d71bfb57dd6b0e6ca6583e9fe9a95130b38e3 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Tue, 4 Feb 2014 04:44:00 +0000 Subject: [PATCH 18/54] YARN-1684. Fixed history server heap size in yarn script. Contributed by Billie Rinaldi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564193 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cf0a269047e..79be4506d7f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -172,6 +172,9 @@ Release 2.4.0 - UNRELEASED YARN-1673. Fix option parsing in YARN's application CLI after it is broken by YARN-967. (Mayank Bansal via vinodkv) + YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi + via zjshen) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index ac42a9a442a..7b805ffe86f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -204,7 +204,7 @@ elif [ "$COMMAND" = "historyserver" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS" - if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then + if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m" fi elif [ "$COMMAND" = "nodemanager" ] ; then diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 121f864f838..8fba1b5a03f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -207,7 +207,7 @@ goto :eof set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\ahs-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer set YARN_OPTS=%YARN_OPTS% %HADOOP_HISTORYSERVER_OPTS% - if defined YARN_RESOURCEMANAGER_HEAPSIZE ( + if defined YARN_HISTORYSERVER_HEAPSIZE ( set JAVA_HEAP_MAX=-Xmx%YARN_HISTORYSERVER_HEAPSIZE%m ) goto :eof From cd1e8d2e3470a5ce7aced0108f0587a655652008 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 4 Feb 2014 05:11:11 +0000 Subject: [PATCH 19/54] HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding error during testing. Contributed by Haohui Mai git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564198 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f9abed76c5f..bf8633747d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -331,6 +331,9 @@ Release 2.4.0 - UNRELEASED HDFS-5767. NFS implementation assumes userName userId mapping to be unique, which is not true sometimes (Yongjun Zhang via brandonli) + HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding + error during testing (Haohui Mai via brandonli) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index 883fdeacf79..c4f30b3ebf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -52,6 +52,7 @@ public class TestHttpsFileSystem { conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); FileUtil.fullyDelete(base); From b812af964d100c50d065cdd9007cef31ea2642a8 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 22:46:52 +0000 Subject: [PATCH 20/54] YARN-1669. Modified RM HA handling of protocol level service-ACLS to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564549 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/ipc/Server.java | 8 ++ .../ServiceAuthorizationManager.java | 29 +++-- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../server/resourcemanager/AdminService.java | 52 +++++--- .../ApplicationMasterService.java | 25 +++- .../resourcemanager/ClientRMService.java | 19 ++- .../ResourceTrackerService.java | 20 ++- .../scheduler/capacity/CapacityScheduler.java | 9 +- .../resourcemanager/TestRMAdminService.java | 119 +++++++++++++++++- 10 files changed, 254 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index c7cada825d9..9c67146265b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -450,6 +450,14 @@ public abstract class Server { serviceAuthorizationManager.refresh(conf, provider); } + /** + * Refresh the service authorization ACL for the service handled by this server + * using the specified Configuration. + */ + public void refreshServiceAclWithConfigration(Configuration conf, + PolicyProvider provider) { + serviceAuthorizationManager.refreshWithConfiguration(conf, provider); + } /** * Returns a handle to the serviceAuthorizationManager (required in tests) * @return instance of ServiceAuthorizationManager for this server diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 8523f38dec7..cf032ba0980 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,6 +33,8 @@ import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import com.google.common.annotations.VisibleForTesting; + /** * An authorization manager which handles service-level authorization * for incoming service requests. @@ -120,19 +122,23 @@ public class ServiceAuthorizationManager { // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); - + refreshWithConfiguration(policyConf, provider); + } + + public synchronized void refreshWithConfiguration(Configuration conf, + PolicyProvider provider) { final Map, AccessControlList> newAcls = - new IdentityHashMap, AccessControlList>(); + new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { - AccessControlList acl = - new AccessControlList( - policyConf.get(service.getServiceKey(), - AccessControlList.WILDCARD_ACL_VALUE) - ); + AccessControlList acl = + new AccessControlList( + conf.get(service.getServiceKey(), + AccessControlList.WILDCARD_ACL_VALUE) + ); newAcls.put(service.getProtocol(), acl); } } @@ -141,8 +147,13 @@ public class ServiceAuthorizationManager { protocolToAcl = newAcls; } - // Package-protected for use in tests. - Set> getProtocolsWithAcls() { + @VisibleForTesting + public Set> getProtocolsWithAcls() { return protocolToAcl.keySet(); } + + @VisibleForTesting + public AccessControlList getProtocolsAcls(Class className) { + return protocolToAcl.get(className); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 79be4506d7f..8e8095dfb32 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -122,6 +122,10 @@ Release 2.4.0 - UNRELEASED to be consistent with what exists (false) in the code and documentation. (Kenji Kikushima via vinodkv) + YARN-1669. Modified RM HA handling of protocol level service-ACLS to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 919ed901439..44f6e9b3724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -40,6 +40,10 @@ public class YarnConfiguration extends Configuration { @Private public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + @Private + public static final String HADOOP_POLICY_CONFIGURATION_FILE = + "hadoop-policy.xml"; + @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3bfd47d1373..d9c239e220a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.conf.ConfigurationProvider; @@ -432,9 +433,8 @@ public class AdminService extends CompositeService implements @Override public RefreshServiceAclsResponse refreshServiceAcls( - RefreshServiceAclsRequest request) throws YarnException { - Configuration conf = new Configuration(); - if (!conf.getBoolean( + RefreshServiceAclsRequest request) throws YarnException, IOException { + if (!getConfig().getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { throw RPCUtil.getRemoteException( @@ -442,27 +442,38 @@ public class AdminService extends CompositeService implements CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION + ") not enabled.")); } - - PolicyProvider policyProvider = new RMPolicyProvider(); - - refreshServiceAcls(conf, policyProvider); - if (isRMActive()) { - rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); - rmContext.getApplicationMasterService().refreshServiceAcls( - conf, policyProvider); - rmContext.getResourceTrackerService().refreshServiceAcls( - conf, policyProvider); - } else { - LOG.warn("ResourceManager is not active. Not refreshing ACLs for " + - "Clients, ApplicationMasters and NodeManagers"); + + String argName = "refreshServiceAcls"; + if (!isRMActive()) { + RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser() + .getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh Service ACLs."); + throwStandbyException(); } + + PolicyProvider policyProvider = new RMPolicyProvider(); + Configuration conf = + getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + + refreshServiceAcls(conf, policyProvider); + rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); + rmContext.getApplicationMasterService().refreshServiceAcls( + conf, policyProvider); + rmContext.getResourceTrackerService().refreshServiceAcls( + conf, policyProvider); return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); } - void refreshServiceAcls(Configuration configuration, + synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.configurationProvider instanceof LocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -519,4 +530,9 @@ public class AdminService extends CompositeService implements public AccessControlList getAccessControlList() { return this.adminAcl; } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 57605c09362..2c4be13ee92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -39,6 +39,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -86,6 +87,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import com.google.common.annotations.VisibleForTesting; + @SuppressWarnings("unchecked") @Private public class ApplicationMasterService extends AbstractService implements @@ -102,6 +105,7 @@ public class ApplicationMasterService extends AbstractService implements private final AllocateResponse resync = recordFactory.newRecordInstance(AllocateResponse.class); private final RMContext rmContext; + private boolean useLocalConfigurationProvider; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); @@ -111,6 +115,15 @@ public class ApplicationMasterService extends AbstractService implements this.rmContext = rmContext; } + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); + super.serviceInit(conf); + } + @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); @@ -578,7 +591,12 @@ public class ApplicationMasterService extends AbstractService implements public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -604,4 +622,9 @@ public class ApplicationMasterService extends AbstractService implements this.response = response; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 8800f290cdc..086a6d823e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -43,6 +43,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -106,6 +107,7 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; @@ -133,6 +135,7 @@ public class ClientRMService extends AbstractService implements private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; + private boolean useLocalConfigurationProvider; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -150,6 +153,10 @@ public class ClientRMService extends AbstractService implements @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -773,7 +780,12 @@ public class ClientRMService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } private boolean isAllowedDelegationTokenOp() throws IOException { @@ -787,4 +799,9 @@ public class ClientRMService extends AbstractService implements return true; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f80ce85d9d1..4f74179717f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -29,6 +29,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.VersionUtil; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -66,6 +67,8 @@ import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import com.google.common.annotations.VisibleForTesting; + public class ResourceTrackerService extends AbstractService implements ResourceTracker { @@ -92,6 +95,7 @@ public class ResourceTrackerService extends AbstractService implements private int minAllocMb; private int minAllocVcores; + private boolean useLocalConfigurationProvider; static { resync.setNodeAction(NodeAction.RESYNC); @@ -141,6 +145,10 @@ public class ResourceTrackerService extends AbstractService implements YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -415,6 +423,16 @@ public class ResourceTrackerService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } + } + + @VisibleForTesting + public Server getServer() { + return this.server; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 1b47d69fa2c..b019a762515 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -262,10 +263,10 @@ public class CapacityScheduler extends AbstractYarnScheduler public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.useLocalConfigurationProvider = conf.get( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( - "org.apache.hadoop.yarn.LocalConfigurationProvider"); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); this.conf = new CapacitySchedulerConfiguration(conf, this.useLocalConfigurationProvider); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 797b4226842..5372c18832d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,16 +24,19 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -190,6 +193,120 @@ public class TestRMAdminService { Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); } + @Test + public void testServiceAclsRefreshWithLocalConfigurationProvider() { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + ResourceManager resourceManager = null; + + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + @SuppressWarnings("resource") + @Test + public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + ResourceManager resourceManager = null; + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + resourceManager.adminService + .refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: hadoop-policy.xml")); + } + + String aclsString = "alice,bob users,wheel"; + Configuration conf = new Configuration(); + conf.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + conf.set("security.applicationclient.protocol.acl", aclsString); + String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(hadoopConfFile)); + + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + + // verify service Acls refresh for AdminService + ServiceAuthorizationManager adminServiceServiceManager = + resourceManager.adminService.getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(adminServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ClientRMService + ServiceAuthorizationManager clientRMServiceServiceManager = + resourceManager.getRMContext().getClientRMService().getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(clientRMServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ApplicationMasterService + ServiceAuthorizationManager appMasterService = + resourceManager.getRMContext().getApplicationMasterService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(appMasterService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ResourceTrackerService + ServiceAuthorizationManager RTService = + resourceManager.getRMContext().getResourceTrackerService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(RTService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, + Class protocol, String aclString) { + for (Class protocolClass : manager.getProtocolsWithAcls()) { + AccessControlList accessList = + manager.getProtocolsAcls(protocolClass); + if (protocolClass == protocol) { + Assert.assertEquals(accessList.getAclString(), + aclString); + } else { + Assert.assertEquals(accessList.getAclString(), "*"); + } + } + } + @Test public void testRefreshSuperUserGroupsWithLocalConfigurationProvider() { From 96578f0e01ba751175d4bcbad48d6f679e662382 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 5 Feb 2014 00:32:02 +0000 Subject: [PATCH 21/54] YARN-1634. Added a testable in-memory implementation of ApplicationTimelineStore. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564583 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 5 + .../api/records/apptimeline/ATSEntity.java | 90 ++- .../apptimeline/EntityId.java | 100 ++++ .../MemoryApplicationTimelineStore.java | 288 ++++++++++ .../ApplicationTimelineStoreTestUtils.java | 532 ++++++++++++++++++ .../TestMemoryApplicationTimelineStore.java | 73 +++ 6 files changed, 1087 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8e8095dfb32..a9af3e0121a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -90,6 +90,9 @@ Release 2.4.0 - UNRELEASED implementing different storage impls for storing timeline information. (Billie Rinaldi via vinodkv) + YARN-1634. Added a testable in-memory implementation of + ApplicationTimelineStore. (Zhijie Shen via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via @@ -126,6 +129,8 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + OPTIMIZATIONS + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java index 6b3ea1013e8..709c79568e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java @@ -50,7 +50,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEntity { +public class ATSEntity implements Comparable { private String entityType; private String entityId; @@ -310,4 +310,92 @@ public class ATSEntity { this.otherInfo = otherInfo; } + @Override + public int hashCode() { + // generated by eclipse + final int prime = 31; + int result = 1; + result = prime * result + ((entityId == null) ? 0 : entityId.hashCode()); + result = + prime * result + ((entityType == null) ? 0 : entityType.hashCode()); + result = prime * result + ((events == null) ? 0 : events.hashCode()); + result = prime * result + ((otherInfo == null) ? 0 : otherInfo.hashCode()); + result = + prime * result + + ((primaryFilters == null) ? 0 : primaryFilters.hashCode()); + result = + prime * result + + ((relatedEntities == null) ? 0 : relatedEntities.hashCode()); + result = prime * result + ((startTime == null) ? 0 : startTime.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + // generated by eclipse + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ATSEntity other = (ATSEntity) obj; + if (entityId == null) { + if (other.entityId != null) + return false; + } else if (!entityId.equals(other.entityId)) + return false; + if (entityType == null) { + if (other.entityType != null) + return false; + } else if (!entityType.equals(other.entityType)) + return false; + if (events == null) { + if (other.events != null) + return false; + } else if (!events.equals(other.events)) + return false; + if (otherInfo == null) { + if (other.otherInfo != null) + return false; + } else if (!otherInfo.equals(other.otherInfo)) + return false; + if (primaryFilters == null) { + if (other.primaryFilters != null) + return false; + } else if (!primaryFilters.equals(other.primaryFilters)) + return false; + if (relatedEntities == null) { + if (other.relatedEntities != null) + return false; + } else if (!relatedEntities.equals(other.relatedEntities)) + return false; + if (startTime == null) { + if (other.startTime != null) + return false; + } else if (!startTime.equals(other.startTime)) + return false; + return true; + } + + @Override + public int compareTo(ATSEntity other) { + int comparison = entityType.compareTo(other.entityType); + if (comparison == 0) { + long thisStartTime = + startTime == null ? Long.MIN_VALUE : startTime; + long otherStartTime = + other.startTime == null ? Long.MIN_VALUE : other.startTime; + if (thisStartTime > otherStartTime) { + return -1; + } else if (thisStartTime < otherStartTime) { + return 1; + } else { + return entityId.compareTo(other.entityId); + } + } else { + return comparison; + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java new file mode 100644 index 00000000000..26431f87569 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The unique identifier for an entity + */ +@Private +@Unstable +public class EntityId implements Comparable { + + private String id; + private String type; + + public EntityId(String id, String type) { + this.id = id; + this.type = type; + } + + /** + * Get the entity Id. + * @return The entity Id. + */ + public String getId() { + return id; + } + + /** + * Get the entity type. + * @return The entity type. + */ + public String getType() { + return type; + } + + @Override + public int compareTo(EntityId other) { + int c = type.compareTo(other.type); + if (c != 0) return c; + return id.compareTo(other.id); + } + + @Override + public int hashCode() { + // generated by eclipse + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + // generated by eclipse + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + EntityId other = (EntityId) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + return true; + } + + @Override + public String toString() { + return "{ id: " + id + ", type: "+ type + " }"; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java new file mode 100644 index 00000000000..45f0a11d764 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents.ATSEventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; + +/** + * In-memory implementation of {@link ApplicationTimelineStore}. This + * implementation is for test purpose only. If users improperly instantiate it, + * they may encounter reading and writing history data in different memory + * store. + * + */ +@Private +@Unstable +public class MemoryApplicationTimelineStore + extends AbstractService implements ApplicationTimelineStore { + + private Map entities = + new HashMap(); + + public MemoryApplicationTimelineStore() { + super(MemoryApplicationTimelineStore.class.getName()); + } + + @Override + public ATSEntities getEntities(String entityType, Long limit, + Long windowStart, Long windowEnd, NameValuePair primaryFilter, + Collection secondaryFilters, EnumSet fields) { + if (limit == null) { + limit = DEFAULT_LIMIT; + } + if (windowStart == null) { + windowStart = Long.MIN_VALUE; + } + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + if (fields == null) { + fields = EnumSet.allOf(Field.class); + } + List entitiesSelected = new ArrayList(); + for (ATSEntity entity : new PriorityQueue(entities.values())) { + if (entitiesSelected.size() >= limit) { + break; + } + if (!entity.getEntityType().equals(entityType)) { + continue; + } + if (entity.getStartTime() <= windowStart) { + continue; + } + if (entity.getStartTime() > windowEnd) { + continue; + } + if (primaryFilter != null && + !matchFilter(entity.getPrimaryFilters(), primaryFilter)) { + continue; + } + if (secondaryFilters != null) { // OR logic + boolean flag = false; + for (NameValuePair secondaryFilter : secondaryFilters) { + if (secondaryFilter != null && + matchFilter(entity.getOtherInfo(), secondaryFilter)) { + flag = true; + break; + } + } + if (!flag) { + continue; + } + } + entitiesSelected.add(entity); + } + List entitiesToReturn = new ArrayList(); + for (ATSEntity entitySelected : entitiesSelected) { + entitiesToReturn.add(maskFields(entitySelected, fields)); + } + Collections.sort(entitiesToReturn); + ATSEntities entitiesWrapper = new ATSEntities(); + entitiesWrapper.setEntities(entitiesToReturn); + return entitiesWrapper; + } + + @Override + public ATSEntity getEntity(String entityId, String entityType, + EnumSet fieldsToRetrieve) { + if (fieldsToRetrieve == null) { + fieldsToRetrieve = EnumSet.allOf(Field.class); + } + ATSEntity entity = entities.get(new EntityId(entityId, entityType)); + if (entity == null) { + return null; + } else { + return maskFields(entity, fieldsToRetrieve); + } + } + + @Override + public ATSEvents getEntityTimelines(String entityType, + SortedSet entityIds, Long limit, Long windowStart, + Long windowEnd, + Set eventTypes) { + ATSEvents allEvents = new ATSEvents(); + if (entityIds == null) { + return allEvents; + } + if (limit == null) { + limit = DEFAULT_LIMIT; + } + if (windowStart == null) { + windowStart = Long.MIN_VALUE; + } + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + for (String entityId : entityIds) { + EntityId entityID = new EntityId(entityId, entityType); + ATSEntity entity = entities.get(entityID); + if (entity == null) { + continue; + } + ATSEventsOfOneEntity events = new ATSEventsOfOneEntity(); + events.setEntityId(entityId); + events.setEntityType(entityType); + for (ATSEvent event : entity.getEvents()) { + if (events.getEvents().size() >= limit) { + break; + } + if (event.getTimestamp() <= windowStart) { + continue; + } + if (event.getTimestamp() > windowEnd) { + continue; + } + if (eventTypes != null && !eventTypes.contains(event.getEventType())) { + continue; + } + events.addEvent(event); + } + allEvents.addEvent(events); + } + return allEvents; + } + + @Override + public ATSPutErrors put(ATSEntities data) { + ATSPutErrors errors = new ATSPutErrors(); + for (ATSEntity entity : data.getEntities()) { + EntityId entityId = + new EntityId(entity.getEntityId(), entity.getEntityType()); + // store entity info in memory + ATSEntity existingEntity = entities.get(entityId); + if (existingEntity == null) { + existingEntity = new ATSEntity(); + existingEntity.setEntityId(entity.getEntityId()); + existingEntity.setEntityType(entity.getEntityType()); + existingEntity.setStartTime(entity.getStartTime()); + entities.put(entityId, existingEntity); + } + if (entity.getEvents() != null) { + if (existingEntity.getEvents() == null) { + existingEntity.setEvents(entity.getEvents()); + } else { + existingEntity.addEvents(entity.getEvents()); + } + Collections.sort(existingEntity.getEvents()); + } + // check startTime + if (existingEntity.getStartTime() == null) { + if (existingEntity.getEvents() == null + || existingEntity.getEvents().isEmpty()) { + ATSPutError error = new ATSPutError(); + error.setEntityId(entityId.getId()); + error.setEntityType(entityId.getType()); + error.setErrorCode(1); + errors.addError(error); + entities.remove(entityId); + continue; + } else { + existingEntity.setStartTime(entity.getEvents().get(0).getTimestamp()); + } + } + if (entity.getPrimaryFilters() != null) { + if (existingEntity.getPrimaryFilters() == null) { + existingEntity.setPrimaryFilters(entity.getPrimaryFilters()); + } else { + existingEntity.addPrimaryFilters(entity.getPrimaryFilters()); + } + } + if (entity.getOtherInfo() != null) { + if (existingEntity.getOtherInfo() == null) { + existingEntity.setOtherInfo(entity.getOtherInfo()); + } else { + existingEntity.addOtherInfo(entity.getOtherInfo()); + } + } + // relate it to other entities + if (entity.getRelatedEntities() == null) { + continue; + } + for (Map.Entry> partRelatedEntities : entity + .getRelatedEntities().entrySet()) { + if (partRelatedEntities == null) { + continue; + } + for (String idStr : partRelatedEntities.getValue()) { + EntityId relatedEntityId = + new EntityId(idStr, partRelatedEntities.getKey()); + ATSEntity relatedEntity = entities.get(relatedEntityId); + if (relatedEntity != null) { + relatedEntity.addRelatedEntity( + existingEntity.getEntityType(), existingEntity.getEntityId()); + } + } + } + } + return errors; + } + + private static ATSEntity maskFields( + ATSEntity entity, EnumSet fields) { + // Conceal the fields that are not going to be exposed + ATSEntity entityToReturn = new ATSEntity(); + entityToReturn.setEntityId(entity.getEntityId()); + entityToReturn.setEntityType(entity.getEntityType()); + entityToReturn.setStartTime(entity.getStartTime()); + entityToReturn.setEvents(fields.contains(Field.EVENTS) ? + entity.getEvents() : fields.contains(Field.LAST_EVENT_ONLY) ? + Arrays.asList(entity.getEvents().get(0)) : null); + entityToReturn.setRelatedEntities(fields.contains(Field.RELATED_ENTITIES) ? + entity.getRelatedEntities() : null); + entityToReturn.setPrimaryFilters(fields.contains(Field.PRIMARY_FILTERS) ? + entity.getPrimaryFilters() : null); + entityToReturn.setOtherInfo(fields.contains(Field.OTHER_INFO) ? + entity.getOtherInfo() : null); + return entityToReturn; + } + + private static boolean matchFilter(Map tags, + NameValuePair filter) { + Object value = tags.get(filter.getName()); + if (value == null) { // doesn't have the filter + return false; + } else if (!value.equals(filter.getValue())) { // doesn't match the filter + return false; + } + return true; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java new file mode 100644 index 00000000000..5825af192b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java @@ -0,0 +1,532 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents.ATSEventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineReader.Field; + +public class ApplicationTimelineStoreTestUtils { + + private static final Map EMPTY_MAP = Collections.emptyMap(); + private static final Map> EMPTY_REL_ENTITIES = + new HashMap>(); + + protected ApplicationTimelineStore store; + private String entity1; + private String entityType1; + private String entity1b; + private String entity2; + private String entityType2; + private Map primaryFilters; + private Map secondaryFilters; + private Map allFilters; + private Map otherInfo; + private Map> relEntityMap; + private NameValuePair userFilter; + private Collection goodTestingFilters; + private Collection badTestingFilters; + private ATSEvent ev1; + private ATSEvent ev2; + private ATSEvent ev3; + private ATSEvent ev4; + private Map eventInfo; + private List events1; + private List events2; + + /** + * Load test data into the given store + */ + protected void loadTestData() { + ATSEntities atsEntities = new ATSEntities(); + Map primaryFilters = new HashMap(); + primaryFilters.put("user", "username"); + primaryFilters.put("appname", 12345l); + Map secondaryFilters = new HashMap(); + secondaryFilters.put("startTime", 123456l); + secondaryFilters.put("status", "RUNNING"); + Map otherInfo1 = new HashMap(); + otherInfo1.put("info1", "val1"); + otherInfo1.putAll(secondaryFilters); + + String entity1 = "id_1"; + String entityType1 = "type_1"; + String entity1b = "id_2"; + String entity2 = "id_2"; + String entityType2 = "type_2"; + + Map> relatedEntities = + new HashMap>(); + relatedEntities.put(entityType2, Collections.singletonList(entity2)); + + ATSEvent ev3 = createEvent(789l, "launch_event", null); + ATSEvent ev4 = createEvent(-123l, "init_event", null); + List events = new ArrayList(); + events.add(ev3); + events.add(ev4); + atsEntities.setEntities(Collections.singletonList(createEntity(entity2, + entityType2, null, events, null, null, null))); + ATSPutErrors response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + ATSEvent ev1 = createEvent(123l, "start_event", null); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1, + entityType1, 123l, Collections.singletonList(ev1), + relatedEntities, primaryFilters, otherInfo1))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1b, + entityType1, null, Collections.singletonList(ev1), relatedEntities, + primaryFilters, otherInfo1))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + Map eventInfo = new HashMap(); + eventInfo.put("event info 1", "val1"); + ATSEvent ev2 = createEvent(456l, "end_event", eventInfo); + Map otherInfo2 = new HashMap(); + otherInfo2.put("info2", "val2"); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1, + entityType1, null, Collections.singletonList(ev2), null, + primaryFilters, otherInfo2))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1b, + entityType1, 123l, Collections.singletonList(ev2), null, + primaryFilters, otherInfo2))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + atsEntities.setEntities(Collections.singletonList(createEntity( + "badentityid", "badentity", null, null, null, null, otherInfo1))); + response = store.put(atsEntities); + assertEquals(1, response.getErrors().size()); + ATSPutError error = response.getErrors().get(0); + assertEquals("badentityid", error.getEntityId()); + assertEquals("badentity", error.getEntityType()); + assertEquals((Integer) 1, error.getErrorCode()); + } + + /** + * Load veification data + */ + protected void loadVerificationData() throws Exception { + userFilter = new NameValuePair("user", + "username"); + goodTestingFilters = new ArrayList(); + goodTestingFilters.add(new NameValuePair("appname", 12345l)); + goodTestingFilters.add(new NameValuePair("status", "RUNNING")); + badTestingFilters = new ArrayList(); + badTestingFilters.add(new NameValuePair("appname", 12345l)); + badTestingFilters.add(new NameValuePair("status", "FINISHED")); + + primaryFilters = new HashMap(); + primaryFilters.put("user", "username"); + primaryFilters.put("appname", 12345l); + secondaryFilters = new HashMap(); + secondaryFilters.put("startTime", 123456l); + secondaryFilters.put("status", "RUNNING"); + allFilters = new HashMap(); + allFilters.putAll(secondaryFilters); + allFilters.putAll(primaryFilters); + otherInfo = new HashMap(); + otherInfo.put("info1", "val1"); + otherInfo.put("info2", "val2"); + otherInfo.putAll(secondaryFilters); + + entity1 = "id_1"; + entityType1 = "type_1"; + entity1b = "id_2"; + entity2 = "id_2"; + entityType2 = "type_2"; + + ev1 = createEvent(123l, "start_event", null); + + eventInfo = new HashMap(); + eventInfo.put("event info 1", "val1"); + ev2 = createEvent(456l, "end_event", eventInfo); + events1 = new ArrayList(); + events1.add(ev2); + events1.add(ev1); + + relEntityMap = + new HashMap>(); + List ids = new ArrayList(); + ids.add(entity1); + ids.add(entity1b); + relEntityMap.put(entityType1, ids); + + ev3 = createEvent(789l, "launch_event", null); + ev4 = createEvent(-123l, "init_event", null); + events2 = new ArrayList(); + events2.add(ev3); + events2.add(ev4); + } + + public void testGetSingleEntity() { + // test getting entity info + verifyEntityInfo(null, null, null, null, null, null, + store.getEntity("id_1", "type_2", EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, store.getEntity(entity1, entityType1, + EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, store.getEntity(entity1b, entityType1, + EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity2, entityType2, events2, relEntityMap, EMPTY_MAP, + EMPTY_MAP, store.getEntity(entity2, entityType2, + EnumSet.allOf(Field.class))); + + // test getting single fields + verifyEntityInfo(entity1, entityType1, events1, null, null, null, + store.getEntity(entity1, entityType1, EnumSet.of(Field.EVENTS))); + + verifyEntityInfo(entity1, entityType1, Collections.singletonList(ev2), + null, null, null, store.getEntity(entity1, entityType1, + EnumSet.of(Field.LAST_EVENT_ONLY))); + + verifyEntityInfo(entity1, entityType1, null, null, primaryFilters, null, + store.getEntity(entity1, entityType1, + EnumSet.of(Field.PRIMARY_FILTERS))); + + verifyEntityInfo(entity1, entityType1, null, null, null, otherInfo, + store.getEntity(entity1, entityType1, EnumSet.of(Field.OTHER_INFO))); + + verifyEntityInfo(entity2, entityType2, null, relEntityMap, null, null, + store.getEntity(entity2, entityType2, + EnumSet.of(Field.RELATED_ENTITIES))); + } + + public void testGetEntities() { + // test getting entities + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_0", null, null, null, null, null, + null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_3", null, null, null, null, null, + null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_0", null, null, null, userFilter, + null, null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_3", null, null, null, userFilter, + null, null).getEntities().size()); + + List entities = + store.getEntities("type_1", null, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_2", null, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity2, entityType2, events2, relEntityMap, EMPTY_MAP, + EMPTY_MAP, entities.get(0)); + + entities = store.getEntities("type_1", 1l, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", 1l, 0l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", null, 234l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 123l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 234l, 345l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, 345l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, 123l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + } + + public void testGetEntitiesWithPrimaryFilters() { + // test using primary filter + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_1", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_2", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_3", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + + List entities = store.getEntities("type_1", null, null, null, + userFilter, null, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_2", null, null, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", 1l, null, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", 1l, 0l, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", null, 234l, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 234l, 345l, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, 345l, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + } + + public void testGetEntitiesWithSecondaryFilters() { + // test using secondary filter + List entities = store.getEntities("type_1", null, null, null, + null, goodTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, null, userFilter, + goodTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, null, null, + badTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, null, userFilter, + badTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + } + + public void testGetEvents() { + // test getting entity timelines + SortedSet sortedSet = new TreeSet(); + sortedSet.add(entity1); + List timelines = + store.getEntityTimelines(entityType1, sortedSet, null, null, + null, null).getAllEvents(); + assertEquals(1, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2, ev1); + + sortedSet.add(entity1b); + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, 1l, + null, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + 345l, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + 123l, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, 345l, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, 123l, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, null, Collections.singleton("end_event")).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + sortedSet.add(entity2); + timelines = store.getEntityTimelines(entityType2, sortedSet, null, + null, null, null).getAllEvents(); + assertEquals(1, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity2, entityType2, ev3, ev4); + } + + /** + * Verify a single entity + */ + private static void verifyEntityInfo(String entity, String entityType, + List events, Map> relatedEntities, + Map primaryFilters, Map otherInfo, + ATSEntity retrievedEntityInfo) { + if (entity == null) { + assertNull(retrievedEntityInfo); + return; + } + assertEquals(entity, retrievedEntityInfo.getEntityId()); + assertEquals(entityType, retrievedEntityInfo.getEntityType()); + if (events == null) + assertNull(retrievedEntityInfo.getEvents()); + else + assertEquals(events, retrievedEntityInfo.getEvents()); + if (relatedEntities == null) + assertNull(retrievedEntityInfo.getRelatedEntities()); + else + assertEquals(relatedEntities, retrievedEntityInfo.getRelatedEntities()); + if (primaryFilters == null) + assertNull(retrievedEntityInfo.getPrimaryFilters()); + else + assertTrue(primaryFilters.equals( + retrievedEntityInfo.getPrimaryFilters())); + if (otherInfo == null) + assertNull(retrievedEntityInfo.getOtherInfo()); + else + assertTrue(otherInfo.equals(retrievedEntityInfo.getOtherInfo())); + } + + /** + * Verify timeline events + */ + private static void verifyEntityTimeline( + ATSEventsOfOneEntity retrievedEvents, String entity, String entityType, + ATSEvent... actualEvents) { + assertEquals(entity, retrievedEvents.getEntityId()); + assertEquals(entityType, retrievedEvents.getEntityType()); + assertEquals(actualEvents.length, retrievedEvents.getEvents().size()); + for (int i = 0; i < actualEvents.length; i++) { + assertEquals(actualEvents[i], retrievedEvents.getEvents().get(i)); + } + } + + /** + * Create a test entity + */ + private static ATSEntity createEntity(String entity, String entityType, + Long startTime, List events, + Map> relatedEntities, + Map primaryFilters, Map otherInfo) { + ATSEntity atsEntity = new ATSEntity(); + atsEntity.setEntityId(entity); + atsEntity.setEntityType(entityType); + atsEntity.setStartTime(startTime); + atsEntity.setEvents(events); + if (relatedEntities != null) + for (Entry> e : relatedEntities.entrySet()) + for (String v : e.getValue()) + atsEntity.addRelatedEntity(e.getKey(), v); + else + atsEntity.setRelatedEntities(null); + atsEntity.setPrimaryFilters(primaryFilters); + atsEntity.setOtherInfo(otherInfo); + return atsEntity; + } + + /** + * Create a test event + */ + private static ATSEvent createEvent(long timestamp, String type, Map info) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(timestamp); + event.setEventType(type); + event.setEventInfo(info); + return event; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java new file mode 100644 index 00000000000..aa88b74a901 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +public class TestMemoryApplicationTimelineStore + extends ApplicationTimelineStoreTestUtils { + + @Before + public void setup() throws Exception { + store = new MemoryApplicationTimelineStore(); + store.init(new YarnConfiguration()); + store.start(); + loadTestData(); + loadVerificationData(); + } + + @After + public void tearDown() throws Exception { + store.stop(); + } + + public ApplicationTimelineStore getApplicationTimelineStore() { + return store; + } + + @Test + public void testGetSingleEntity() { + super.testGetSingleEntity(); + } + + @Test + public void testGetEntities() { + super.testGetEntities(); + } + + @Test + public void testGetEntitiesWithPrimaryFilters() { + super.testGetEntitiesWithPrimaryFilters(); + } + + @Test + public void testGetEntitiesWithSecondaryFilters() { + super.testGetEntitiesWithSecondaryFilters(); + } + + @Test + public void testGetEvents() { + super.testGetEvents(); + } + +} From 0aa09f6d5a97f523e9ee6f30bb44f206433ead0a Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 5 Feb 2014 04:18:31 +0000 Subject: [PATCH 22/54] HDFS-5399. Revisit SafeModeException and corresponding retry policies. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564629 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/io/retry/RetryPolicies.java | 20 ++++++++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSClient.java | 6 ++++++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../apache/hadoop/hdfs/NameNodeProxies.java | 16 ++++++++++----- .../hdfs/server/namenode/FSNamesystem.java | 15 +++++++++++++- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 5 ++++- .../server/namenode/ha/TestHASafeMode.java | 5 +++++ 8 files changed, 66 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index 3d5992716f4..14ded8ea24d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -151,6 +151,13 @@ public class RetryPolicies { delayMillis, maxDelayBase); } + public static final RetryPolicy failoverOnNetworkException( + RetryPolicy fallbackPolicy, int maxFailovers, int maxRetries, + long delayMillis, long maxDelayBase) { + return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers, + maxRetries, delayMillis, maxDelayBase); + } + static class TryOnceThenFail implements RetryPolicy { @Override public RetryAction shouldRetry(Exception e, int retries, int failovers, @@ -516,18 +523,25 @@ public class RetryPolicies { private RetryPolicy fallbackPolicy; private int maxFailovers; + private int maxRetries; private long delayMillis; private long maxDelayBase; public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers) { - this(fallbackPolicy, maxFailovers, 0, 0); + this(fallbackPolicy, maxFailovers, 0, 0, 0); } public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers, long delayMillis, long maxDelayBase) { + this(fallbackPolicy, maxFailovers, 0, delayMillis, maxDelayBase); + } + + public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, + int maxFailovers, int maxRetries, long delayMillis, long maxDelayBase) { this.fallbackPolicy = fallbackPolicy; this.maxFailovers = maxFailovers; + this.maxRetries = maxRetries; this.delayMillis = delayMillis; this.maxDelayBase = maxDelayBase; } @@ -549,6 +563,10 @@ public class RetryPolicies { "failovers (" + failovers + ") exceeded maximum allowed (" + maxFailovers + ")"); } + if (retries - failovers > maxRetries) { + return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "retries (" + + retries + ") exceeded maximum allowed (" + maxRetries + ")"); + } if (e instanceof ConnectException || e instanceof NoRouteToHostException || diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bf8633747d1..c84166b863c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -863,6 +863,9 @@ Release 2.3.0 - UNRELEASED HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs on a secure cluster. (jing9) + HDFS-5399. Revisit SafeModeException and corresponding retry policies. + (Haohui Mai via todd) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index b522acb09a5..c1a1a73bf77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY; @@ -258,6 +260,7 @@ public class DFSClient implements java.io.Closeable { public static class Conf { final int hdfsTimeout; // timeout value for a DFS operation. final int maxFailoverAttempts; + final int maxRetryAttempts; final int failoverSleepBaseMillis; final int failoverSleepMaxMillis; final int maxBlockAcquireFailures; @@ -303,6 +306,9 @@ public class DFSClient implements java.io.Closeable { maxFailoverAttempts = conf.getInt( DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + maxRetryAttempts = conf.getInt( + DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); failoverSleepBaseMillis = conf.getInt( DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 38c40c21f38..fe1d3d1570e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -82,6 +82,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_DEFAULT = 0; public static final String DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY = "dfs.client.failover.connection.retries.on.timeouts"; public static final int DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0; + public static final String DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY = "dfs.client.retry.max.attempts"; + public static final int DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY = "dfs.client.socketcache.expiryMsec"; public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 2 * 60 * 1000; @@ -574,6 +576,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,... public static final String DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY = "dfs.http.client.failover.max.attempts"; public static final int DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT = 15; + public static final String DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY = "dfs.http.client.retry.max.attempts"; + public static final int DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY = "dfs.http.client.failover.sleep.base.millis"; public static final int DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT = 500; public static final String DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY = "dfs.http.client.failover.sleep.max.millis"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index bff0284c74f..1637bd03cd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -24,6 +24,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; import java.io.IOException; import java.lang.reflect.Constructor; @@ -144,9 +146,10 @@ public class NameNodeProxies { .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface, nameNodeUri); Conf config = new Conf(conf); - T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies - .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, - config.maxFailoverAttempts, config.failoverSleepBaseMillis, + T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, + RetryPolicies.failoverOnNetworkException( + RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts, + config.maxRetryAttempts, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis)); Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri); @@ -192,11 +195,14 @@ public class NameNodeProxies { int maxFailoverAttempts = config.getInt( DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + int maxRetryAttempts = config.getInt( + DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); InvocationHandler dummyHandler = new LossyRetryInvocationHandler( numResponseToDrop, failoverProxyProvider, RetryPolicies.failoverOnNetworkException( - RetryPolicies.TRY_ONCE_THEN_FAIL, - Math.max(numResponseToDrop + 1, maxFailoverAttempts), delay, + RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, + Math.max(numResponseToDrop + 1, maxRetryAttempts), delay, maxCap)); T proxy = (T) Proxy.newProxyInstance( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ef2c28622f9..4e209767dfc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1161,7 +1161,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (isInSafeMode()) { SafeModeException se = new SafeModeException(errorMsg, safeMode); if (haEnabled && haContext != null - && haContext.getState().getServiceState() == HAServiceState.ACTIVE) { + && haContext.getState().getServiceState() == HAServiceState.ACTIVE + && shouldRetrySafeMode(this.safeMode)) { throw new RetriableException(se); } else { throw se; @@ -1169,6 +1170,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } + /** + * We already know that the safemode is on. We will throw a RetriableException + * if the safemode is not manual or caused by low resource. + */ + private boolean shouldRetrySafeMode(SafeModeInfo safeMode) { + if (safeMode == null) { + return false; + } else { + return !safeMode.isManual() && !safeMode.areResourcesLow(); + } + } + public static Collection getNamespaceDirs(Configuration conf) { return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 148dc72d76c..efaf9637d7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -188,6 +188,9 @@ public class WebHdfsFileSystem extends FileSystem int maxFailoverAttempts = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + int maxRetryAttempts = conf.getInt( + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); int failoverSleepBaseMillis = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); @@ -197,7 +200,7 @@ public class WebHdfsFileSystem extends FileSystem this.retryPolicy = RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, - maxFailoverAttempts, failoverSleepBaseMillis, + maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 0c95764eba6..1c7b7f423cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.SafeModeInfo; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IOUtils; @@ -65,6 +66,7 @@ import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; import com.google.common.base.Supplier; import com.google.common.collect.Lists; @@ -124,6 +126,9 @@ public class TestHASafeMode { final Path test = new Path("/test"); // let nn0 enter safemode NameNodeAdapter.enterSafeMode(nn0, false); + SafeModeInfo safeMode = (SafeModeInfo) Whitebox.getInternalState( + nn0.getNamesystem(), "safeMode"); + Whitebox.setInternalState(safeMode, "extension", Integer.valueOf(30000)); LOG.info("enter safemode"); new Thread() { @Override From 747cce814cfc1fdb89569eeb85b4712f10beea0e Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 5 Feb 2014 04:20:48 +0000 Subject: [PATCH 23/54] Correct CHANGES.txt entry for HDFS-5399 (contributed by Jing, not Haohui) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564632 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c84166b863c..0b68cd35a0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -864,7 +864,7 @@ Release 2.3.0 - UNRELEASED on a secure cluster. (jing9) HDFS-5399. Revisit SafeModeException and corresponding retry policies. - (Haohui Mai via todd) + (Jing Zhao via todd) BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS From ebe0c17a95ae37d4768f2928ea193e89db34ead5 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 5 Feb 2014 04:55:00 +0000 Subject: [PATCH 24/54] YARN-1461. Added tags for YARN applications and changed RM to handle them. Contributed by Karthik Kambatla. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564633 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../ApplicationsRequestScope.java | 38 ++++++ .../GetApplicationsRequest.java | 117 +++++++++++++++++- .../yarn/api/records/ApplicationReport.java | 14 +++ .../records/ApplicationSubmissionContext.java | 26 +++- .../hadoop/yarn/conf/YarnConfiguration.java | 7 ++ .../src/main/proto/yarn_protos.proto | 2 + .../src/main/proto/yarn_service_protos.proto | 8 ++ .../impl/pb/GetApplicationsRequestPBImpl.java | 62 +++++++++- .../impl/pb/ApplicationReportPBImpl.java | 31 +++++ .../ApplicationSubmissionContextPBImpl.java | 60 ++++++++- .../yarn/api/records/impl/pb/ProtoUtils.java | 14 +++ .../yarn/server/utils/BuilderUtils.java | 4 +- .../resourcemanager/ClientRMService.java | 33 ++++- .../server/resourcemanager/RMAppManager.java | 3 +- .../server/resourcemanager/rmapp/RMApp.java | 9 +- .../resourcemanager/rmapp/RMAppImpl.java | 15 ++- .../resourcemanager/webapp/AppBlock.java | 1 + .../resourcemanager/webapp/RMWebServices.java | 13 +- .../resourcemanager/webapp/dao/AppInfo.java | 10 +- .../resourcemanager/TestClientRMService.java | 51 +++++++- .../applicationsmanager/MockAsm.java | 12 +- .../resourcemanager/rmapp/MockRMApp.java | 6 + .../rmapp/TestRMAppTransitions.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 6 +- .../webapp/TestRMWebServicesApps.java | 4 +- .../src/site/apt/ResourceManagerRest.apt.vm | 1 + 27 files changed, 528 insertions(+), 24 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a9af3e0121a..ee4ac79d57b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -93,6 +93,9 @@ Release 2.4.0 - UNRELEASED YARN-1634. Added a testable in-memory implementation of ApplicationTimelineStore. (Zhijie Shen via vinodkv) + YARN-1461. Added tags for YARN applications and changed RM to handle them. + (Karthik Kambatla via zjshen) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java new file mode 100644 index 00000000000..168223340c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Enumeration that controls the scope of applications fetched + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public enum ApplicationsRequestScope { + /** All jobs */ + ALL, + + /** Jobs viewable by current user */ + VIEWABLE, + + /** Jobs owned by current user */ + OWN +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 95254b2cb13..4cc0b70e4af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.protocolrecords; import java.util.EnumSet; import java.util.Set; -import org.apache.commons.collections.buffer.UnboundedFifoBuffer; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -49,6 +48,86 @@ public abstract class GetApplicationsRequest { return request; } + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + * + *

Setting any of the parameters to null, would just disable that + * filter

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @param users list of users to filter by + * @param queues list of scheduler queues to filter by + * @param applicationTypes types of applications + * @param applicationTags application tags to filter by + * @param applicationStates application states to filter by + * @param startRange range of application start times to filter by + * @param finishRange range of application finish times to filter by + * @param limit number of applications to limit to + * @return {@link GetApplicationsRequest} to be used with + * {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)} + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope, + Set users, + Set queues, + Set applicationTypes, + Set applicationTags, + EnumSet applicationStates, + LongRange startRange, + LongRange finishRange, + Long limit) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + if (scope != null) { + request.setScope(scope); + } + request.setUsers(users); + request.setQueues(queues); + request.setApplicationTypes(applicationTypes); + request.setApplicationTags(applicationTags); + request.setApplicationStates(applicationStates); + if (startRange != null) { + request.setStartRange( + startRange.getMinimumLong(), startRange.getMaximumLong()); + } + if (finishRange != null) { + request.setFinishRange( + finishRange.getMinimumLong(), finishRange.getMaximumLong()); + } + if (limit != null) { + request.setLimit(limit); + } + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setScope(scope); + return request; + } + /** *

* The request from clients to get a report of Applications matching the @@ -257,4 +336,40 @@ public abstract class GetApplicationsRequest { @Private @Unstable public abstract void setFinishRange(long begin, long end); + + /** + * Get the tags to filter applications on + * + * @return list of tags to filter on + */ + @Private + @Unstable + public abstract Set getApplicationTags(); + + /** + * Set the list of tags to filter applications on + * + * @param tags list of tags to filter on + */ + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + + /** + * Get the {@link ApplicationsRequestScope} of applications to be filtered. + * + * @return {@link ApplicationsRequestScope} of applications to return. + */ + @Private + @Unstable + public abstract ApplicationsRequestScope getScope(); + + /** + * Set the {@link ApplicationsRequestScope} of applications to filter. + * + * @param scope scope to use for filtering applications + */ + @Private + @Unstable + public abstract void setScope(ApplicationsRequestScope scope); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 0854fdb89c9..412c22b13db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationReport is a report of an application.

* @@ -321,6 +323,18 @@ public abstract class ApplicationReport { @Unstable public abstract void setApplicationType(String applicationType); + /** + * Get all tags corresponding to the application + * @return Application's tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + @Private @Stable public abstract void setAMRMToken(Token amRmToken); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index a320bb26972..529df113c27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -25,8 +25,11 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationSubmissionContext represents all of the * information needed by the ResourceManager to launch @@ -284,7 +287,6 @@ public abstract class ApplicationSubmissionContext { @Stable public abstract void setApplicationType(String applicationType); - /** * Get the flag which indicates whether to keep containers across application * attempts or not. @@ -314,4 +316,26 @@ public abstract class ApplicationSubmissionContext { @Stable public abstract void setKeepContainersAcrossApplicationAttempts( boolean keepContainers); + + /** + * Get tags for the application + * + * @return the application tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + /** + * Set tags for the application. A maximum of + * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed + * per application. Each tag can be at most + * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH} + * characters, and can contain only ASCII characters. + * + * @param tags tags to set + */ + @Public + @Stable + public abstract void setApplicationTags(Set tags); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 44f6e9b3724..d0df7d07f34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -50,6 +51,12 @@ public class YarnConfiguration extends Configuration { @Private public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + @Evolving + public static final int APPLICATION_MAX_TAGS = 10; + + @Evolving + public static final int APPLICATION_MAX_TAG_LENGTH = 100; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; static { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c86d97149c7..8f6cf4c783b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -190,6 +190,7 @@ message ApplicationReportProto { optional float progress = 17; optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; + repeated string applicationTags = 20; } message ApplicationAttemptReportProto { @@ -287,6 +288,7 @@ message ApplicationSubmissionContextProto { optional ResourceProto resource = 9; optional string applicationType = 10 [default = "YARN"]; optional bool keep_containers_across_application_attempts = 11 [default = false]; + repeated string applicationTags = 12; } enum ApplicationAccessTypeProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 139b3bb1394..eff5cd7ae82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -136,6 +136,12 @@ message MoveApplicationAcrossQueuesRequestProto { message MoveApplicationAcrossQueuesResponseProto { } +enum ApplicationsRequestScopeProto { + ALL = 0; + VIEWABLE = 1; + OWN = 2; +} + message GetApplicationsRequestProto { repeated string application_types = 1; repeated YarnApplicationStateProto application_states = 2; @@ -146,6 +152,8 @@ message GetApplicationsRequestProto { optional int64 start_end = 7; optional int64 finish_begin = 8; optional int64 finish_end = 9; + repeated string applicationTags = 10; + optional ApplicationsRequestScopeProto scope = 11 [default = ALL]; } message GetApplicationsResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 10513a3dc18..733f19f715d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; @@ -49,6 +50,8 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { Set queues = null; long limit = Long.MAX_VALUE; LongRange start = null, finish = null; + private Set applicationTags; + private ApplicationsRequestScope scope; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -112,6 +115,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { }; builder.addAllApplicationStates(iterable); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } + if (this.scope != null) { + builder.setScope(ProtoUtils.convertToProtoFormat(scope)); + } } private void addLocalApplicationTypesToProto() { @@ -187,12 +196,64 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { this.applicationTypes = applicationTypes; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public EnumSet getApplicationStates() { initApplicationStates(); return this.applicationStates; } + private void initScope() { + if (this.scope != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.scope = ProtoUtils.convertFromProtoFormat(p.getScope()); + } + + @Override + public ApplicationsRequestScope getScope() { + initScope(); + return this.scope; + } + + public void setScope(ApplicationsRequestScope scope) { + maybeInitBuilder(); + if (scope == null) { + builder.clearScope(); + } + this.scope = scope; + } + @Override public void setApplicationStates(EnumSet applicationStates) { maybeInitBuilder(); @@ -223,7 +284,6 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { return this.users; } - @Override public void setUsers(Set users) { maybeInitBuilder(); if (users == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 9716f74a681..7e19d8fa2f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -38,6 +38,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationReportPBImpl extends ApplicationReport { @@ -49,6 +52,7 @@ public class ApplicationReportPBImpl extends ApplicationReport { private ApplicationAttemptId currentApplicationAttemptId; private Token clientToAMToken = null; private Token amRmToken = null; + private Set applicationTags = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -245,6 +249,21 @@ public class ApplicationReportPBImpl extends ApplicationReport { return amRmToken; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); @@ -355,6 +374,15 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.setApplicationType((applicationType)); } + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + } + this.applicationTags = tags; + } + @Override public void setDiagnostics(String diagnostics) { maybeInitBuilder(); @@ -450,6 +478,9 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.getAmRmToken())) { builder.setAmRmToken(convertToProtoFormat(this.amRmToken)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index ffaaf3556b2..c4a3a721990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import com.google.common.base.CharMatcher; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -25,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -34,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationSubmissionContextPBImpl @@ -47,6 +52,7 @@ extends ApplicationSubmissionContext { private Priority priority = null; private ContainerLaunchContext amContainer = null; private Resource resource = null; + private Set applicationTags = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -100,6 +106,9 @@ extends ApplicationSubmissionContext { builder.getResource())) { builder.setResource(convertToProtoFormat(this.resource)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { @@ -196,7 +205,22 @@ extends ApplicationSubmissionContext { } return (p.getApplicationType()); } - + + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setQueue(String queue) { maybeInitBuilder(); @@ -217,6 +241,40 @@ extends ApplicationSubmissionContext { builder.setApplicationType((applicationType)); } + private void checkTags(Set tags) { + if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) { + throw new IllegalArgumentException("Too many applicationTags, a maximum of only " + + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!"); + } + for (String tag : tags) { + if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) { + throw new IllegalArgumentException("Tag " + tag + " is too long, " + + "maximum allowed length of a tag is " + + YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); + } + if (!CharMatcher.ASCII.matchesAllOf(tag)) { + throw new IllegalArgumentException("A tag can only have ASCII " + + "characters! Invalid tag - " + tag); + } + } + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + checkTags(tags); + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public ContainerLaunchContext getAMContainerSpec() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 8d737448567..4a3c13792ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.ByteString; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; @Private @Unstable @@ -113,6 +115,18 @@ public class ProtoUtils { YARN_APPLICATION_ATTEMPT_STATE_PREFIX, "")); } + /* + * ApplicationsRequestScope + */ + public static YarnServiceProtos.ApplicationsRequestScopeProto + convertToProtoFormat(ApplicationsRequestScope e) { + return YarnServiceProtos.ApplicationsRequestScopeProto.valueOf(e.name()); + } + public static ApplicationsRequestScope convertFromProtoFormat + (YarnServiceProtos.ApplicationsRequestScopeProto e) { + return ApplicationsRequestScope.valueOf(e.name()); + } + /* * ApplicationResourceUsageReport */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 1b2a03e5511..ac25c001248 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; @@ -312,7 +313,7 @@ public class BuilderUtils { String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String appType, Token amRmToken) { + float progress, String appType, Token amRmToken, Set tags) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -334,6 +335,7 @@ public class BuilderUtils { report.setProgress(progress); report.setApplicationType(appType); report.setAMRMToken(amRmToken); + report.setApplicationTags(tags); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 086a6d823e3..2f8526a7c71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -445,9 +446,11 @@ public class ClientRMService extends AbstractService implements request.getApplicationStates(); Set users = request.getUsers(); Set queues = request.getQueues(); + Set tags = request.getApplicationTags(); long limit = request.getLimit(); LongRange start = request.getStartRange(); LongRange finish = request.getFinishRange(); + ApplicationsRequestScope scope = request.getScope(); final Map apps = rmContext.getRMApps(); Iterator appsIter; @@ -494,6 +497,17 @@ public class ClientRMService extends AbstractService implements List reports = new ArrayList(); while (appsIter.hasNext() && reports.size() < limit) { RMApp application = appsIter.next(); + + // Check if current application falls under the specified scope + boolean allowAccess = checkAccess(callerUGI, application.getUser(), + ApplicationAccessType.VIEW_APP, application); + if (scope == ApplicationsRequestScope.OWN && + !callerUGI.getUserName().equals(application.getUser())) { + continue; + } else if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) { + continue; + } + if (applicationTypes != null && !applicationTypes.isEmpty()) { String appTypeToMatch = caseSensitive ? application.getApplicationType() @@ -523,8 +537,23 @@ public class ClientRMService extends AbstractService implements continue; } - boolean allowAccess = checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.VIEW_APP, application); + if (tags != null && !tags.isEmpty()) { + Set appTags = application.getApplicationTags(); + if (appTags == null || appTags.isEmpty()) { + continue; + } + boolean match = false; + for (String tag : tags) { + if (appTags.contains(tag)) { + match = true; + break; + } + } + if (!match) { + continue; + } + } + reports.add(application.createAndGetApplicationReport( callerUGI.getUserName(), allowAccess)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7dbc4cc2800..7855042db78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -320,7 +320,8 @@ public class RMAppManager implements EventHandler, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, - submitTime, submissionContext.getApplicationType()); + submitTime, submissionContext.getApplicationType(), + submissionContext.getApplicationTags()); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 1809a4bb470..1a7e74ef672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -194,7 +195,13 @@ public interface RMApp extends EventHandler { * Returns the application type * @return the application type. */ - String getApplicationType(); + String getApplicationType(); + + /** + * Get tags for the application + * @return tags corresponding to the application + */ + Set getApplicationTags(); /** * Check whether this application is safe to terminate. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index d4cf416dd33..edbe676badd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -104,6 +104,7 @@ public class RMAppImpl implements RMApp, Recoverable { private final long submitTime; private final Set updatedNodes = new HashSet(); private final String applicationType; + private final Set applicationTags; // Mutable fields private long startTime; @@ -302,9 +303,9 @@ public class RMAppImpl implements RMApp, Recoverable { public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, - ApplicationSubmissionContext submissionContext, - YarnScheduler scheduler, - ApplicationMasterService masterService, long submitTime, String applicationType) { + ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, + ApplicationMasterService masterService, long submitTime, + String applicationType, Set applicationTags) { this.applicationId = applicationId; this.name = name; @@ -320,6 +321,7 @@ public class RMAppImpl implements RMApp, Recoverable { this.submitTime = submitTime; this.startTime = System.currentTimeMillis(); this.applicationType = applicationType; + this.applicationTags = applicationTags; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -553,7 +555,7 @@ public class RMAppImpl implements RMApp, Recoverable { createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken); + amrmToken, applicationTags); } finally { this.readLock.unlock(); } @@ -1085,6 +1087,11 @@ public class RMAppImpl implements RMApp, Recoverable { return this.applicationType; } + @Override + public Set getApplicationTags() { + return this.applicationTags; + } + @Override public boolean isAppSafeToTerminate() { RMAppState state = getState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index ed147fbb733..445a5a2ca99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -110,6 +110,7 @@ public class AppBlock extends HtmlBlock { _("User:", app.getUser()). _("Name:", app.getName()). _("Application Type:", app.getApplicationType()). + _("Application Tags:", app.getApplicationTags()). _("State:", app.getState()). _("FinalStatus:", app.getFinalStatus()). _("Started:", Times.format(app.getStartTime())). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index e18c04740cb..e854e2ce525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -261,12 +261,14 @@ public class RMWebServices { @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, - @QueryParam("applicationTypes") Set applicationTypes) { + @QueryParam("applicationTypes") Set applicationTypes, + @QueryParam("applicationTags") Set applicationTags) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; + boolean checkAppTags = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -327,6 +329,11 @@ public class RMWebServices { checkAppTypes = true; } + Set appTags = parseQueries(applicationTags, false); + if (!appTags.isEmpty()) { + checkAppTags = true; + } + // stateQuery is deprecated. if (stateQuery != null && !stateQuery.isEmpty()) { statesQuery.add(stateQuery); @@ -354,6 +361,10 @@ public class RMWebServices { request.setApplicationTypes(appTypes); } + if (checkAppTags) { + request.setApplicationTags(appTags); + } + if (checkAppStates) { request.setApplicationStates(appStates); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 295e1f3a41a..a4f4873ed4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import com.google.common.base.Joiner; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; @@ -67,6 +67,7 @@ public class AppInfo { protected String diagnostics; protected long clusterId; protected String applicationType; + protected String applicationTags = ""; // these are only allowed if acls allow protected long startedTime; @@ -117,6 +118,9 @@ public class AppInfo { if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } + if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { + this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); + } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { @@ -239,6 +243,10 @@ public class AppInfo { public String getApplicationType() { return this.applicationType; } + + public String getApplicationTags() { + return this.applicationTags; + } public int getRunningContainers() { return this.runningContainers; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1894a115ac6..7c496815ab2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -41,8 +41,10 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import junit.framework.Assert; +import org.apache.commons.lang.math.LongRange; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +53,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; @@ -72,6 +75,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; @@ -465,6 +469,7 @@ public class TestClientRMService { {MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName()}; ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), getApplicationId(103)}; + List tags = Arrays.asList("Tag1", "Tag2", "Tag3"); // Submit applications for (int i = 0; i < appIds.length; i++) { @@ -472,7 +477,8 @@ public class TestClientRMService { when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); SubmitApplicationRequest submitRequest = mockSubmitAppRequest( - appId, appNames[i], queues[i % queues.length]); + appId, appNames[i], queues[i % queues.length], + new HashSet(tags.subList(0, i + 1))); rmService.submitApplication(submitRequest); } @@ -513,6 +519,41 @@ public class TestClientRMService { userSet.add(UserGroupInformation.getCurrentUser().getShortUserName()); assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + + // Check tags + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.ALL, null, null, null, null, null, null, + null, null); + Set tagSet = new HashSet(); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 6, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(0)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 3, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(1)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 2, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(2)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 1, + rmService.getApplications(request).getApplicationList().size()); + + // Check scope + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.VIEWABLE); + assertEquals("Incorrect number of applications for the scope", 6, + rmService.getApplications(request).getApplicationList().size()); + + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.OWN); + assertEquals("Incorrect number of applications for the scope", 3, + rmService.getApplications(request).getApplicationList().size()); } @Test(timeout=4000) @@ -583,6 +624,11 @@ public class TestClientRMService { private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { + return mockSubmitAppRequest(appId, name, queue, null); + } + + private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, + String name, String queue, Set tags) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( @@ -596,6 +642,7 @@ public class TestClientRMService { submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); + submissionContext.setApplicationTags(tags); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -664,7 +711,7 @@ public class TestClientRMService { when(asContext.getMaxAppAttempts()).thenReturn(1); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis(), "YARN")); + .currentTimeMillis(), "YARN", null)); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId3, 1); RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index aa116bf85b2..01e5eea96cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; @@ -139,6 +140,11 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } + @Override + public Set getApplicationTags() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public void setQueue(String name) { throw new UnsupportedOperationException("Not supported yet."); @@ -235,7 +241,11 @@ public abstract class MockAsm extends MockApps { public int getMaxAppAttempts() { return maxAppAttempts; } - + + @Override + public Set getApplicationTags() { + return null; + } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index debcffe97dc..ac3751a191c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -217,6 +218,11 @@ public class MockRMApp implements RMApp { return YarnConfiguration.DEFAULT_APPLICATION_TYPE; } + @Override + public Set getApplicationTags() { + return null; + } + @Override public boolean isAppSafeToTerminate() { return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index a8c2ab65c96..5ac9353928e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -230,7 +230,7 @@ public class TestRMAppTransitions { RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b251ce7dd19..892438902de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -621,7 +621,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -647,7 +647,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -1765,7 +1765,7 @@ public class TestFairScheduler { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 349bae4c384..18350fb30d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 19, info.length()); + // 20 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 20, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 73902912571..ac6b4466c27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1123,6 +1123,7 @@ ResourceManager REST API's. * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. + * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. ------ ** Elements of the (Applications) object From cec369149b4205af0e20dfbe7f2f541f46688fc5 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 5 Feb 2014 05:42:02 +0000 Subject: [PATCH 25/54] HADOOP-10273. Fix 'mvn site'. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564638 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 06d97b1428e..e3a39daa581 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -320,6 +320,8 @@ Release 2.4.0 - UNRELEASED HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) + HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/pom.xml b/pom.xml index 27db19ba353..a4f824102cc 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.apache.maven.plugins maven-site-plugin - 3.2 + 3.3 org.apache.maven.wagon @@ -329,7 +329,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs maven-site-plugin - 3.0 + 3.3 attach-descriptor From d6bd920bba0d7cb77ca76c3a79d1ba1e039da9e5 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 5 Feb 2014 06:48:00 +0000 Subject: [PATCH 26/54] HDFS-5709. Improve NameNode upgrade with existing reserved paths and path components. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 61 +++++- .../hadoop/hdfs/protocol/HdfsConstants.java | 13 +- .../server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 144 ++++++++------ .../hdfs/server/namenode/FSImageFormat.java | 179 +++++++++++++++++- .../hadoop/hdfs/server/namenode/NameNode.java | 39 +++- .../src/site/apt/HdfsUserGuide.apt.vm | 29 ++- .../src/site/xdoc/HdfsSnapshots.xml | 21 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 85 +++++++++ .../namenode/TestNameNodeOptionParsing.java | 88 +++++++++ .../src/test/resources/hadoop-2-reserved.tgz | Bin 0 -> 2838 bytes 12 files changed, 584 insertions(+), 81 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0b68cd35a0e..5b39abfeeaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding error during testing (Haohui Mai via brandonli) + HDFS-5709. Improve NameNode upgrade with existing reserved paths and path + components. (Andrew Wang via atm) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index b58f2732d0a..9274f505a4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -261,6 +261,47 @@ public class DFSUtil { return true; } + /** + * Checks if a string is a valid path component. For instance, components + * cannot contain a ":" or "/", and cannot be equal to a reserved component + * like ".snapshot". + *

+ * The primary use of this method is for validating paths when loading the + * FSImage. During normal NN operation, paths are sometimes allowed to + * contain reserved components. + * + * @return If component is valid + */ + public static boolean isValidNameForComponent(String component) { + if (component.equals(".") || + component.equals("..") || + component.indexOf(":") >= 0 || + component.indexOf("/") >= 0) { + return false; + } + return !isReservedPathComponent(component); + } + + + /** + * Returns if the component is reserved. + * + *

+ * Note that some components are only reserved under certain directories, e.g. + * "/.reserved" is reserved, while "/hadoop/.reserved" is not. + * + * @param component + * @return if the component is reserved + */ + public static boolean isReservedPathComponent(String component) { + for (String reserved : HdfsConstants.RESERVED_PATH_COMPONENTS) { + if (component.equals(reserved)) { + return true; + } + } + return false; + } + /** * Converts a byte array to a string using UTF8 encoding. */ @@ -312,7 +353,25 @@ public class DFSUtil { } return result.toString(); } - + + /** + * Converts a list of path components into a path using Path.SEPARATOR. + * + * @param components Path components + * @return Combined path as a UTF-8 string + */ + public static String strings2PathString(String[] components) { + if (components.length == 0) { + return ""; + } + if (components.length == 1) { + if (components[0] == null || components[0].isEmpty()) { + return Path.SEPARATOR; + } + } + return Joiner.on(Path.SEPARATOR).join(components); + } + /** * Given a list of path components returns a byte array */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index da934c36992..d1c7e143c90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; /************************************ * Some handy constants @@ -108,7 +109,17 @@ public class HdfsConstants { */ public static final int LAYOUT_VERSION = LayoutVersion .getCurrentLayoutVersion(); - + + /** + * Path components that are reserved in HDFS. + *

+ * .reserved is only reserved under root ("/"). + */ + public static final String[] RESERVED_PATH_COMPONENTS = new String[] { + HdfsConstants.DOT_SNAPSHOT_DIR, + FSDirectory.DOT_RESERVED_STRING + }; + /** * A special path component contained in the path for a snapshot file/dir */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 50f6e730408..77fe7c6dd06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -59,7 +59,8 @@ public final class HdfsServerConstants { INITIALIZESHAREDEDITS("-initializeSharedEdits"), RECOVER ("-recover"), FORCE("-force"), - NONINTERACTIVE("-nonInteractive"); + NONINTERACTIVE("-nonInteractive"), + RENAMERESERVED("-renameReserved"); private final String name; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index ce2cf2f7be7..7433b6b1094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormat.renameReservedPathsOnUpgrade; import static org.apache.hadoop.util.Time.now; import java.io.FilterInputStream; @@ -292,8 +293,10 @@ public class FSEditLogLoader { switch (op.opCode) { case OP_ADD: { AddCloseOp addCloseOp = (AddCloseOp)op; + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); @@ -304,9 +307,9 @@ public class FSEditLogLoader { // 3. OP_ADD to open file for append // See if the file already exists (persistBlocks call) - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); final INodeFile oldFile = INodeFile.valueOf( - iip.getINode(0), addCloseOp.path, true); + iip.getINode(0), path, true); INodeFile newFile = oldFile; if (oldFile == null) { // this is OP_ADD on a new file (case 1) // versions > 0 support per file replication @@ -319,10 +322,10 @@ public class FSEditLogLoader { inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId); newFile = fsDir.unprotectedAddFile(inodeId, - addCloseOp.path, addCloseOp.permissions, replication, + path, addCloseOp.permissions, replication, addCloseOp.mtime, addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, addCloseOp.clientMachine); - fsNamesys.leaseManager.addLease(addCloseOp.clientName, addCloseOp.path); + fsNamesys.leaseManager.addLease(addCloseOp.clientName, path); // add the op into retry cache if necessary if (toAddRetryCache) { @@ -338,11 +341,11 @@ public class FSEditLogLoader { FSNamesystem.LOG.debug("Reopening an already-closed file " + "for append"); } - LocatedBlock lb = fsNamesys.prepareFileForWrite(addCloseOp.path, + LocatedBlock lb = fsNamesys.prepareFileForWrite(path, oldFile, addCloseOp.clientName, addCloseOp.clientMachine, null, false, iip.getLatestSnapshotId(), false); - newFile = INodeFile.valueOf(fsDir.getINode(addCloseOp.path), - addCloseOp.path, true); + newFile = INodeFile.valueOf(fsDir.getINode(path), + path, true); // add the op into retry cache is necessary if (toAddRetryCache) { @@ -363,16 +366,17 @@ public class FSEditLogLoader { } case OP_CLOSE: { AddCloseOp addCloseOp = (AddCloseOp)op; - + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); } - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); - final INodeFile file = INodeFile.valueOf(iip.getINode(0), addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); + final INodeFile file = INodeFile.valueOf(iip.getINode(0), path); // Update the salient file attributes. file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID); @@ -386,24 +390,26 @@ public class FSEditLogLoader { // could show up twice in a row. But after that version, this // should be fixed, so we should treat it as an error. throw new IOException( - "File is not under construction: " + addCloseOp.path); + "File is not under construction: " + path); } // One might expect that you could use removeLease(holder, path) here, // but OP_CLOSE doesn't serialize the holder. So, remove by path. if (file.isUnderConstruction()) { - fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path); + fsNamesys.leaseManager.removeLeaseWithPrefixPath(path); file.toCompleteFile(file.getModificationTime()); } break; } case OP_UPDATE_BLOCKS: { UpdateBlocksOp updateOp = (UpdateBlocksOp)op; + final String path = + renameReservedPathsOnUpgrade(updateOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + updateOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + updateOp.blocks.length); } - INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(updateOp.path), - updateOp.path); + INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path), + path); // Update in-memory data structures updateBlocks(fsDir, updateOp, oldFile); @@ -414,7 +420,7 @@ public class FSEditLogLoader { } case OP_ADD_BLOCK: { AddBlockOp addBlockOp = (AddBlockOp) op; - String path = addBlockOp.getPath(); + String path = renameReservedPathsOnUpgrade(addBlockOp.getPath(), logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug(op.opCode + ": " + path + " new block id : " + addBlockOp.getLastBlock().getBlockId()); @@ -428,14 +434,20 @@ public class FSEditLogLoader { SetReplicationOp setReplicationOp = (SetReplicationOp)op; short replication = fsNamesys.getBlockManager().adjustReplication( setReplicationOp.replication); - fsDir.unprotectedSetReplication(setReplicationOp.path, + fsDir.unprotectedSetReplication( + renameReservedPathsOnUpgrade(setReplicationOp.path, logVersion), replication, null); break; } case OP_CONCAT_DELETE: { ConcatDeleteOp concatDeleteOp = (ConcatDeleteOp)op; - fsDir.unprotectedConcat(concatDeleteOp.trg, concatDeleteOp.srcs, - concatDeleteOp.timestamp); + String trg = renameReservedPathsOnUpgrade(concatDeleteOp.trg, logVersion); + String[] srcs = new String[concatDeleteOp.srcs.length]; + for (int i=0; i removedINodes = new ChunkedArrayList(); + final String snapshotRoot = + renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().deleteSnapshot( - deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName, + snapshotRoot, deleteSnapshotOp.snapshotName, collectedBlocks, removedINodes); fsNamesys.removeBlocksAndUpdateSafemodeTotal(collectedBlocks); collectedBlocks.clear(); @@ -617,8 +647,11 @@ public class FSEditLogLoader { } case OP_RENAME_SNAPSHOT: { RenameSnapshotOp renameSnapshotOp = (RenameSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().renameSnapshot( - renameSnapshotOp.snapshotRoot, renameSnapshotOp.snapshotOldName, + snapshotRoot, renameSnapshotOp.snapshotOldName, renameSnapshotOp.snapshotNewName); if (toAddRetryCache) { @@ -629,14 +662,19 @@ public class FSEditLogLoader { } case OP_ALLOW_SNAPSHOT: { AllowSnapshotOp allowSnapshotOp = (AllowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(allowSnapshotOp.snapshotRoot, logVersion); fsNamesys.getSnapshotManager().setSnapshottable( - allowSnapshotOp.snapshotRoot, false); + snapshotRoot, false); break; } case OP_DISALLOW_SNAPSHOT: { DisallowSnapshotOp disallowSnapshotOp = (DisallowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(disallowSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().resetSnapshottable( - disallowSnapshotOp.snapshotRoot); + snapshotRoot); break; } case OP_SET_GENSTAMP_V2: { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 385917e8e0e..3ad258a4512 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -32,12 +32,13 @@ import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; -import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -45,13 +46,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; -import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -67,6 +70,10 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * Contains inner classes for reading or writing the on-disk format for @@ -405,7 +412,8 @@ public class FSImageFormat { } /** - * load fsimage files assuming only local names are stored + * load fsimage files assuming only local names are stored. Used when + * snapshots are not supported by the layout version. * * @param numFiles number of files expected to be read * @param in image input stream @@ -521,6 +529,8 @@ public class FSImageFormat { */ private int loadDirectory(DataInput in, Counter counter) throws IOException { String parentPath = FSImageSerialization.readString(in); + // Rename .snapshot paths if we're doing an upgrade + parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion()); final INodeDirectory parent = INodeDirectory.valueOf( namesystem.dir.rootDir.getNode(parentPath, true), parentPath); return loadChildren(parent, in, counter); @@ -580,11 +590,9 @@ public class FSImageFormat { */ private void addToParent(INodeDirectory parent, INode child) { FSDirectory fsDir = namesystem.dir; - if (parent == fsDir.rootDir && FSDirectory.isReservedName(child)) { - throw new HadoopIllegalArgumentException("File name \"" - + child.getLocalName() + "\" is reserved. Please " - + " change the name of the existing file or directory to another " - + "name before upgrading to this release."); + if (parent == fsDir.rootDir) { + child.setLocalName(renameReservedRootComponentOnUpgrade( + child.getLocalNameBytes(), getLayoutVersion())); } // NOTE: This does not update space counts for parents if (!parent.addChild(child)) { @@ -621,7 +629,9 @@ public class FSImageFormat { public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in, boolean updateINodeMap, Counter counter) throws IOException { - final byte[] localName = FSImageSerialization.readLocalName(in); + byte[] localName = FSImageSerialization.readLocalName(in); + localName = + renameReservedComponentOnUpgrade(localName, getLayoutVersion()); INode inode = loadINode(localName, isSnapshotINode, in, counter); if (updateINodeMap && LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) { @@ -926,7 +936,156 @@ public class FSImageFormat { return snapshotMap.get(in.readInt()); } } - + + @VisibleForTesting + public static TreeMap renameReservedMap = + new TreeMap(); + + /** + * Use the default key-value pairs that will be used to determine how to + * rename reserved paths on upgrade. + */ + @VisibleForTesting + public static void useDefaultRenameReservedPairs() { + renameReservedMap.clear(); + for (String key: HdfsConstants.RESERVED_PATH_COMPONENTS) { + renameReservedMap.put( + key, + key + "." + LayoutVersion.getCurrentLayoutVersion() + "." + + "UPGRADE_RENAMED"); + } + } + + /** + * Set the key-value pairs that will be used to determine how to rename + * reserved paths on upgrade. + */ + @VisibleForTesting + public static void setRenameReservedPairs(String renameReserved) { + // Clear and set the default values + useDefaultRenameReservedPairs(); + // Overwrite with provided values + setRenameReservedMapInternal(renameReserved); + } + + private static void setRenameReservedMapInternal(String renameReserved) { + Collection pairs = + StringUtils.getTrimmedStringCollection(renameReserved); + for (String p : pairs) { + String[] pair = StringUtils.split(p, '/', '='); + Preconditions.checkArgument(pair.length == 2, + "Could not parse key-value pair " + p); + String key = pair[0]; + String value = pair[1]; + Preconditions.checkArgument(DFSUtil.isReservedPathComponent(key), + "Unknown reserved path " + key); + Preconditions.checkArgument(DFSUtil.isValidNameForComponent(value), + "Invalid rename path for " + key + ": " + value); + LOG.info("Will rename reserved path " + key + " to " + value); + renameReservedMap.put(key, value); + } + } + + /** + * When upgrading from an old version, the filesystem could contain paths + * that are now reserved in the new version (e.g. .snapshot). This renames + * these new reserved paths to a user-specified value to avoid collisions + * with the reserved name. + * + * @param path Old path potentially containing a reserved path + * @return New path with reserved path components renamed to user value + */ + static String renameReservedPathsOnUpgrade(String path, + final int layoutVersion) { + final String oldPath = path; + // If any known LVs aren't supported, we're doing an upgrade + if (!LayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Only need to worry about the root directory + if (components.length > 1) { + components[1] = DFSUtil.bytes2String( + renameReservedRootComponentOnUpgrade( + DFSUtil.string2Bytes(components[1]), + layoutVersion)); + path = DFSUtil.strings2PathString(components); + } + } + if (!LayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Special case the root path + if (components.length == 0) { + return path; + } + for (int i=0; i] ] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + StartupOption.IMPORT.getName() + "] | [" @@ -1056,7 +1058,8 @@ public class NameNode implements NameNodeStatusMXBean { out.println(USAGE + "\n"); } - private static StartupOption parseArguments(String args[]) { + @VisibleForTesting + static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for(int i=0; i < argsLen; i++) { @@ -1103,11 +1106,33 @@ public class NameNode implements NameNodeStatusMXBean { startOpt = StartupOption.CHECKPOINT; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.UPGRADE; - // might be followed by two args - if (i + 2 < argsLen - && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { - i += 2; - startOpt.setClusterId(args[i]); + /* Can be followed by CLUSTERID with a required parameter or + * RENAMERESERVED with an optional parameter + */ + while (i + 1 < argsLen) { + String flag = args[i + 1]; + if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { + if (i + 2 < argsLen) { + i += 2; + startOpt.setClusterId(args[i]); + } else { + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED + .getName())) { + if (i + 2 < argsLen) { + FSImageFormat.setRenameReservedPairs(args[i + 2]); + i += 2; + } else { + FSImageFormat.useDefaultRenameReservedPairs(); + i += 1; + } + } else { + LOG.fatal("Unknown upgrade flag " + flag); + return null; + } } } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLBACK; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm index b84da5991b3..9d6aeb97fb5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm @@ -435,7 +435,7 @@ HDFS Users Guide state it was in before the upgrade. HDFS upgrade is described in more detail in {{{http://wiki.apache.org/hadoop/Hadoop_Upgrade}Hadoop Upgrade}} Wiki page. HDFS can have one such backup at a time. Before upgrading, - administrators need to remove existing backupusing bin/hadoop dfsadmin + administrators need to remove existing backup using bin/hadoop dfsadmin <<<-finalizeUpgrade>>> command. The following briefly describes the typical upgrade procedure: @@ -459,6 +459,33 @@ HDFS Users Guide * start the cluster with rollback option. (<<>>). + When upgrading to a new version of HDFS, it is necessary to rename or + delete any paths that are reserved in the new version of HDFS. If the + NameNode encounters a reserved path during upgrade, it will print an + error like the following: + + <<< /.reserved is a reserved path and .snapshot is a + reserved path component in this version of HDFS. Please rollback and delete + or rename this path, or upgrade with the -renameReserved [key-value pairs] + option to automatically rename these paths during upgrade.>>> + + Specifying <<<-upgrade -renameReserved [optional key-value pairs]>>> causes + the NameNode to automatically rename any reserved paths found during + startup. For example, to rename all paths named <<<.snapshot>>> to + <<<.my-snapshot>>> and <<<.reserved>>> to <<<.my-reserved>>>, a user would + specify <<<-upgrade -renameReserved + .snapshot=.my-snapshot,.reserved=.my-reserved>>>. + + If no key-value pairs are specified with <<<-renameReserved>>>, the + NameNode will then suffix reserved paths with + <<<..UPGRADE_RENAMED>>>, e.g. + <<<.snapshot.-51.UPGRADE_RENAMED>>>. + + There are some caveats to this renaming process. It's recommended, + if possible, to first <<>> before upgrading. + This is because data inconsistency can result if an edit log operation + refers to the destination of an automatically renamed file. + * File Permissions and Security The file permissions are designed to be similar to file permissions on diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index 9aecf9cd55a..bd499c79c80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd"> - HFDS Snapshots + HDFS Snapshots @@ -99,15 +99,22 @@

  • Copying a file from snapshot s0: hdfs dfs -cp /foo/.snapshot/s0/bar /tmp
    • -

    - Note that the name ".snapshot" is now a reserved file name in HDFS - so that users cannot create a file/directory with ".snapshot" as the name. - If ".snapshot" is used in a previous version of HDFS, it must be renamed before upgrade; - otherwise, upgrade will fail. -

    +
    + +

    + The HDFS snapshot feature introduces a new reserved path name used to + interact with snapshots: .snapshot. When upgrading from an + older version of HDFS, existing paths named .snapshot need + to first be renamed or deleted to avoid conflicting with the reserved path. + See the upgrade section in + the HDFS user guide + for more information.

    + +
    +

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java index c686e4ebe7c..fda4e835304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.TreeMap; @@ -43,7 +44,9 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -67,6 +70,7 @@ public class TestDFSUpgradeFromImage { private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt"; private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz"; private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz"; + private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz"; private static class ReferenceFileInfo { String path; @@ -320,6 +324,87 @@ public class TestDFSUpgradeFromImage { assertEquals("Upgrade did not fail with bad MD5", 1, md5failures); } } + + /** + * Test upgrade from 2.0 image with a variety of .snapshot and .reserved + * paths to test renaming on upgrade + */ + @Test + public void testUpgradeFromRel2ReservedImage() throws IOException { + unpackStorage(HADOOP2_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + try { + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "reserved path component in this version", + e); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + // Make sure the paths were renamed as expected + DistributedFileSystem dfs = cluster.getFileSystem(); + ArrayList toList = new ArrayList(); + ArrayList found = new ArrayList(); + toList.add(new Path("/")); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + String[] expected = new String[] { + "/edits", + "/edits/.reserved", + "/edits/.user-snapshot", + "/edits/.user-snapshot/editsdir", + "/edits/.user-snapshot/editsdir/editscontents", + "/edits/.user-snapshot/editsdir/editsdir2", + "/image", + "/image/.reserved", + "/image/.user-snapshot", + "/image/.user-snapshot/imagedir", + "/image/.user-snapshot/imagedir/imagecontents", + "/image/.user-snapshot/imagedir/imagedir2", + "/.my-reserved", + "/.my-reserved/edits-touch", + "/.my-reserved/image-touch" + }; + + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } static void recoverAllLeases(DFSClient dfs, Path path) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java new file mode 100644 index 00000000000..d43eda0cefd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java @@ -0,0 +1,88 @@ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.junit.Test; + +public class TestNameNodeOptionParsing { + + @Test(timeout = 10000) + public void testUpgrade() { + StartupOption opt = null; + // UPGRADE is set, but nothing else + opt = NameNode.parseArguments(new String[] {"-upgrade"}); + assertEquals(opt, StartupOption.UPGRADE); + assertNull(opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // cluster ID is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // Everything is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid", "-renameReserved", + ".snapshot=.my-snapshot,.reserved=.my-reserved" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Reset the map + FSImageFormat.renameReservedMap.clear(); + // Everything is set, but in a different order + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.my-snapshot", "-clusterid", + "mycid"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Try the default renameReserved + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals( + ".snapshot." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals( + ".reserved." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".reserved")); + + // Try some error conditions + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.not-reserved=.my-not-reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Unknown reserved path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.snapshot" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".snapshot=.reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + opt = NameNode.parseArguments(new String[] { "-upgrade", "-cid"}); + assertNull(opt); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3cb2ee63f0e2624968dcd97836a608b88e1a36c2 GIT binary patch literal 2838 zcmeH}`&-h79>+bM<<2pUo|(tI!Etlt(J8e|ElM`+pw5)E(o_`FO{XAeGR@FfD{RG- z@{)-;miHT8P*hYTC3R+sN`i_4MqU9|+|i2L1|o8sJzD6$S+xhuK;10RkLUaSD(1-s5?Gay#(gf;`He z#5OH<6JZN1y}Z7Tubim%Xx45EweO5B#fwOlgicHJKwXr4i&~aZst>oVBW>=jCOuxv z&3GJMO`=@@7)_UfbyQhkNqj_>BI>g(~37=EaW!aI(3KbS9 z;=b=_d#}s#{6Ds|<{V}8$Y#h-c?>N|^X-^)WMi&E6UkX6QInvCV(*q`4~;vlmU|QZ zHaI@N-F&ckF1{_DDS2*K)DV6)2Q% zUsw<$|JC}%iZJU=#iycMXGCWB1uY&P&7$QAj<;lM@2UlumO(^QQ?rM2n?7$PuRl%! zclzkctYmqq9$*{=blfhTHjB<(4esF*S0~ko*!iI4(-`Ehzt2R%R~y0C*T~9eR$^i{ z*%Ug9c|i|Lcf}CN*8tvIyU?E-S#GAoo`f88SJRDSJyJT06TPb)gCdwjfe7Xg4D#_r zg&zA7b_*7w`fVdxgZOR{O;L4Iuou-zcoO#AOM*^0#Pi+{>D*rSGSxxA^hdu;xYPY6 zjur+WdcRr%h0psC-u%HP5!&Z^8%&liK~VTeSfg&Zo%v6RjWV>1KI(z-xT+9`iZQkA zXMT7$j;+PN3D{yQ1QGby3z-|Q7Me1k8H=3$)a;X-n1YiwIqQMW^xLYMh%al95TY71 zT`sx$z*vID8xlzbUiP#*cid)4s^1I(da#e%4PoZ;G;O%{$ZZIzX$p6}mF#JWxbhf% zYKF>(YvNsJ#KISH!Pep$KV+DzVwD*he7Fw;J`ngnBd}Z%gvlAr4+tq6|DVsZg)~8x z-;vt9`Ky7au6E{vTygr>F20N6>iZxl7G@NGpg-9*AjLNqPl7s>$2%V~avK+AjhqOt z7#KT$o>t>hi>%jkSRP!@g1j)xrEdy6FZ;`*8HXlwo11HDTJsl;77`j<2%`Ts-09`R zi286x5sCHXTtY*A>LHWujy=N3K}!WZXMJsA3vx%(8nufjtC3G(U0sd38=V_zQq@~9 z$_xvMxV)p<9_07zjbbHr$&>S%2Hk}`=nI5h);*& ze+aPA6NPG{wV9XVv^d*_{kSl2X$qpM#U7(b-GKwcZmUC&>Pg-rM zWuup1%iZ>(xG>e^7#0-wr0mskO5%}zvb!UUbgUx=xEUj_suaf|!5&w4#yH*(+7z(5 z^Xzp}KJ;vIZe`3L-ySqR`~IoG0+dKVf@w4+nCd15Oo=$oU6v$cJ#N| zbZN|c6*Q1E+LegVmp7x52<3r^!B*4|$CA92E+-7J1ZBA+m(-u1YXf+R?SPIS`lP`m z)fu6*bDgc_7a(JK@YR9nE<}X7KBzF&Keyu%C1ov_*-c0xV1ge)8;b_I6#nvs=Iw@& ztc-}X-yy2g8+}NoEt=bL!=v~)4hqU(PT};Q8s~YUUYHyCA*{~jkH(hmK7V6zx+9Te zVaH`J%r@Yj+OWmju6ufZ4A$%2>L!bZgrF+N!g}fnI>4J{E&&-HzPoxnxmsom{%cDz R(m2VWuLLw?@YMl>KLII!>~8=7 literal 0 HcmV?d00001 From 484faadffec97f8e454cf18b821f6ef12a27616c Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 5 Feb 2014 17:02:26 +0000 Subject: [PATCH 27/54] YARN-1636. Augmented Application-history server's web-services to also expose new APIs for retrieving and storing timeline information. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564829 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 9 + .../webapp/YarnJacksonJaxbJsonProvider.java | 58 ++++ .../src/main/resources/yarn-default.xml | 8 + .../ApplicationHistoryServer.java | 18 +- .../webapp/AHSWebApp.java | 11 +- .../webapp/ATSWebServices.java | 297 ++++++++++++++++++ .../TestApplicationHistoryServer.java | 2 +- .../webapp/TestATSWebServices.java | 212 +++++++++++++ 9 files changed, 615 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ee4ac79d57b..b2f70e41afd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,10 @@ Release 2.4.0 - UNRELEASED YARN-1461. Added tags for YARN applications and changed RM to handle them. (Karthik Kambatla via zjshen) + YARN-1636. Augmented Application-history server's web-services to also expose + new APIs for retrieving and storing timeline information. (Zhijie Shen via + vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d0df7d07f34..9e3619e7f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1028,6 +1028,15 @@ public class YarnConfiguration extends Configuration { public static final String AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = AHS_PREFIX + "webapp.spnego-keytab-file"; + //////////////////////////////// + // ATS Configs + //////////////////////////////// + + public static final String ATS_PREFIX = YARN_PREFIX + "ats."; + + /** ATS store class */ + public static final String ATS_STORE = ATS_PREFIX + "store.class"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java new file mode 100644 index 00000000000..100e91f7c8c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.webapp; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Provider; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; + +import com.google.inject.Singleton; + +/** + * YARN's implementation of JAX-RS abstractions based on + * {@link JacksonJaxbJsonProvider} needed for deserialize JSON content to or + * serialize it from POJO objects. + */ +@Singleton +@Provider +@Unstable +@Private +public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider { + + public YarnJacksonJaxbJsonProvider() { + super(); + } + + @Override + public ObjectMapper locateMapper(Class type, MediaType mediaType) { + ObjectMapper mapper = super.locateMapper(type, mediaType); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig() + .setSerializationInclusion(Inclusion.NON_NULL); + return mapper; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a5906986665..6c8c1a77e14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1137,6 +1137,14 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + + + + Store class name for application timeline store + yarn.ats.store.class + org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore + + The interval that the yarn client library uses to poll the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 3a864c80698..4ec986065b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -27,11 +27,14 @@ import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -51,6 +54,7 @@ public class ApplicationHistoryServer extends CompositeService { ApplicationHistoryClientService ahsClientService; ApplicationHistoryManager historyManager; + ApplicationTimelineStore timelineStore; private WebApp webApp; public ApplicationHistoryServer() { @@ -63,6 +67,8 @@ public class ApplicationHistoryServer extends CompositeService { ahsClientService = createApplicationHistoryClientService(historyManager); addService(ahsClientService); addService((Service) historyManager); + timelineStore = createApplicationTimelineStore(conf); + addIfService(timelineStore); super.serviceInit(conf); } @@ -135,6 +141,15 @@ public class ApplicationHistoryServer extends CompositeService { return new ApplicationHistoryManagerImpl(); } + protected ApplicationTimelineStore createApplicationTimelineStore( + Configuration conf) { + // TODO: need to replace the MemoryApplicationTimelineStore.class with the + // LevelDB implementation + return ReflectionUtils.newInstance(conf.getClass( + YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class, + ApplicationTimelineStore.class), conf); + } + protected void startWebApp() { String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); @@ -148,7 +163,8 @@ public class ApplicationHistoryServer extends CompositeService { YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress).start(new AHSWebApp(historyManager)); + .at(bindAddress) + .start(new AHSWebApp(historyManager, timelineStore)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 81f838396d7..d2cfc32dab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -21,24 +21,31 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; public class AHSWebApp extends WebApp implements YarnWebParams { private final ApplicationHistoryManager applicationHistoryManager; + private final ApplicationTimelineStore applicationTimelineStore; - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager) { + public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, + ApplicationTimelineStore applicationTimelineStore) { this.applicationHistoryManager = applicationHistoryManager; + this.applicationTimelineStore = applicationTimelineStore; } @Override public void setup() { - bind(JAXBContextResolver.class); + bind(YarnJacksonJaxbJsonProvider.class); bind(AHSWebServices.class); + bind(ATSWebServices.class); bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); + bind(ApplicationTimelineStore.class).toInstance(applicationTimelineStore); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java new file mode 100644 index 00000000000..4ea501d89a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineReader.Field; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.NameValuePair; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +@Path("/ws/v1/apptimeline") +//TODO: support XML serialization/deserialization +public class ATSWebServices { + + private ApplicationTimelineStore store; + + @Inject + public ATSWebServices(ApplicationTimelineStore store) { + this.store = store; + } + + @XmlRootElement(name = "about") + @XmlAccessorType(XmlAccessType.NONE) + @Public + @Unstable + public static class AboutInfo { + + private String about; + + public AboutInfo() { + + } + + public AboutInfo(String about) { + this.about = about; + } + + @XmlElement(name = "About") + public String getAbout() { + return about; + } + + public void setAbout(String about) { + this.about = about; + } + + } + + /** + * Return the description of the application timeline web services. + */ + @GET + @Path("/") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public AboutInfo about( + @Context HttpServletRequest req, + @Context HttpServletResponse res) { + init(res); + return new AboutInfo("Application Timeline API"); + } + + /** + * Return a list of entities that match the given parameters. + */ + @GET + @Path("/{entityType}") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEntities getEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @QueryParam("primaryFilter") String primaryFilter, + @QueryParam("secondaryFilter") String secondaryFilter, + @QueryParam("windowStart") String windowStart, + @QueryParam("windowEnd") String windowEnd, + @QueryParam("limit") String limit, + @QueryParam("fields") String fields) { + init(res); + ATSEntities entities = null; + try { + entities = store.getEntities( + parseStr(entityType), + parseLongStr(limit), + parseLongStr(windowStart), + parseLongStr(windowEnd), + parsePairStr(primaryFilter, ":"), + parsePairsStr(secondaryFilter, ",", ":"), + parseFieldsStr(fields, ",")); + } catch (NumberFormatException e) { + throw new BadRequestException( + "windowStart, windowEnd or limit is not a numeric value."); + } catch (IllegalArgumentException e) { + throw new BadRequestException("requested invalid field."); + } + if (entities == null) { + return new ATSEntities(); + } + return entities; + } + + /** + * Return a single entity of the given entity type and Id. + */ + @GET + @Path("/{entityType}/{entityId}") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEntity getEntity( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @PathParam("entityId") String entityId, + @QueryParam("fields") String fields) { + init(res); + ATSEntity entity = null; + try { + entity = + store.getEntity(parseStr(entityId), parseStr(entityType), + parseFieldsStr(fields, ",")); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + "requested invalid field."); + } + if (entity == null) { + throw new WebApplicationException(Response.Status.NOT_FOUND); + } + return entity; + } + + /** + * Return the events that match the given parameters. + */ + @GET + @Path("/{entityType}/events") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEvents getEvents( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @QueryParam("entityId") String entityId, + @QueryParam("eventType") String eventType, + @QueryParam("windowStart") String windowStart, + @QueryParam("windowEnd") String windowEnd, + @QueryParam("limit") String limit) { + init(res); + ATSEvents events = null; + try { + events = store.getEntityTimelines( + parseStr(entityType), + parseArrayStr(entityId, ","), + parseLongStr(limit), + parseLongStr(windowStart), + parseLongStr(windowEnd), + parseArrayStr(eventType, ",")); + } catch (NumberFormatException e) { + throw new BadRequestException( + "windowStart, windowEnd or limit is not a numeric value."); + } + if (events == null) { + return new ATSEvents(); + } + return events; + } + + /** + * Store the given entities into the timeline store, and return the errors + * that happen during storing. + */ + @POST + @Path("/") + @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSPutErrors postEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + ATSEntities entities) { + init(res); + if (entities == null) { + return new ATSPutErrors(); + } + return store.put(entities); + } + + private void init(HttpServletResponse response) { + response.setContentType(null); + } + + private static SortedSet parseArrayStr(String str, String delimiter) { + if (str == null) { + return null; + } + SortedSet strSet = new TreeSet(); + String[] strs = str.split(delimiter); + for (String aStr : strs) { + strSet.add(aStr.trim()); + } + return strSet; + } + + private static NameValuePair parsePairStr(String str, String delimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(delimiter, 2); + return new NameValuePair(strs[0].trim(), strs[1].trim()); + } + + private static Collection parsePairsStr( + String str, String aDelimiter, String pDelimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(aDelimiter); + Set pairs = new HashSet(); + for (String aStr : strs) { + pairs.add(parsePairStr(aStr, pDelimiter)); + } + return pairs; + } + + private static EnumSet parseFieldsStr(String str, String delimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(delimiter); + List fieldList = new ArrayList(); + for (String s : strs) { + fieldList.add(Field.valueOf(s.toUpperCase())); + } + if (fieldList.size() == 0) + return null; + Field f1 = fieldList.remove(fieldList.size() - 1); + if (fieldList.size() == 0) + return EnumSet.of(f1); + else + return EnumSet.of(f1, fieldList.toArray(new Field[fieldList.size()])); + } + + private static Long parseLongStr(String str) { + return str == null ? null : Long.parseLong(str.trim()); + } + + private static String parseStr(String str) { + return str == null ? null : str.trim(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 8bd515b6abe..d6d20af189c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -40,7 +40,7 @@ public class TestApplicationHistoryServer { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(2, historyServer.getServices().size()); + assertEquals(3, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java new file mode 100644 index 00000000000..1ff73ff35a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import static org.junit.Assert.assertEquals; + +import javax.ws.rs.core.MediaType; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.TestMemoryApplicationTimelineStore; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + + +public class TestATSWebServices extends JerseyTest { + + private static ApplicationTimelineStore store; + + private Injector injector = Guice.createInjector(new ServletModule() { + + @Override + protected void configureServlets() { + bind(YarnJacksonJaxbJsonProvider.class); + bind(ATSWebServices.class); + bind(GenericExceptionHandler.class); + try{ + store = mockApplicationTimelineStore(); + } catch (Exception e) { + Assert.fail(); + } + bind(ApplicationTimelineStore.class).toInstance(store); + serve("/*").with(GuiceContainer.class); + } + + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + private ApplicationTimelineStore mockApplicationTimelineStore() + throws Exception { + TestMemoryApplicationTimelineStore store = + new TestMemoryApplicationTimelineStore(); + store.setup(); + return store.getApplicationTimelineStore(); + } + + public TestATSWebServices() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter") + .servletPath("/") + .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class)) + .build()); + } + + @Test + public void testAbout() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSWebServices.AboutInfo about = + response.getEntity(ATSWebServices.AboutInfo.class); + Assert.assertNotNull(about); + Assert.assertEquals("Application Timeline API", about.getAbout()); + } + + @Test + public void testGetEntities() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntities entities = response.getEntity(ATSEntities.class); + Assert.assertNotNull(entities); + Assert.assertEquals(2, entities.getEntities().size()); + ATSEntity entity1 = entities.getEntities().get(0); + Assert.assertNotNull(entity1); + Assert.assertEquals("id_1", entity1.getEntityId()); + Assert.assertEquals("type_1", entity1.getEntityType()); + Assert.assertEquals(123l, entity1.getStartTime().longValue()); + Assert.assertEquals(2, entity1.getEvents().size()); + Assert.assertEquals(2, entity1.getPrimaryFilters().size()); + Assert.assertEquals(4, entity1.getOtherInfo().size()); + ATSEntity entity2 = entities.getEntities().get(1); + Assert.assertNotNull(entity2); + Assert.assertEquals("id_2", entity2.getEntityId()); + Assert.assertEquals("type_1", entity2.getEntityType()); + Assert.assertEquals(123l, entity2.getStartTime().longValue()); + Assert.assertEquals(2, entity2.getEvents().size()); + Assert.assertEquals(2, entity2.getPrimaryFilters().size()); + Assert.assertEquals(4, entity2.getOtherInfo().size()); + } + + @Test + public void testGetEntity() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("id_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntity entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("id_1", entity.getEntityId()); + Assert.assertEquals("type_1", entity.getEntityType()); + Assert.assertEquals(123l, entity.getStartTime().longValue()); + Assert.assertEquals(2, entity.getEvents().size()); + Assert.assertEquals(2, entity.getPrimaryFilters().size()); + Assert.assertEquals(4, entity.getOtherInfo().size()); + } + + @Test + public void testGetEvents() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("events") + .queryParam("entityId", "id_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEvents events = response.getEntity(ATSEvents.class); + Assert.assertNotNull(events); + Assert.assertEquals(1, events.getAllEvents().size()); + ATSEvents.ATSEventsOfOneEntity partEvents = events.getAllEvents().get(0); + Assert.assertEquals(2, partEvents.getEvents().size()); + ATSEvent event1 = partEvents.getEvents().get(0); + Assert.assertEquals(456l, event1.getTimestamp()); + Assert.assertEquals("end_event", event1.getEventType()); + Assert.assertEquals(1, event1.getEventInfo().size()); + ATSEvent event2 = partEvents.getEvents().get(1); + Assert.assertEquals(123l, event2.getTimestamp()); + Assert.assertEquals("start_event", event2.getEventType()); + Assert.assertEquals(0, event2.getEventInfo().size()); + } + + @Test + public void testPostEntities() throws Exception { + ATSEntities entities = new ATSEntities(); + ATSEntity entity = new ATSEntity(); + entity.setEntityId("test id"); + entity.setEntityType("test type"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSPutErrors errors = response.getEntity(ATSPutErrors.class); + Assert.assertNotNull(errors); + Assert.assertEquals(0, errors.getErrors().size()); + // verify the entity exists in the store + response = r.path("ws").path("v1").path("apptimeline") + .path("test type").path("test id") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("test id", entity.getEntityId()); + Assert.assertEquals("test type", entity.getEntityType()); + } + +} From e6eccf20defcaf84a9566482371c7be196779c0d Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Wed, 5 Feb 2014 18:09:07 +0000 Subject: [PATCH 28/54] YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564856 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/SchedulerApplication.java | 6 +- .../fair/AllocationConfiguration.java | 3 +- .../scheduler/fair/FairScheduler.java | 119 ++++++++++++++- .../fair/MaxRunningAppsEnforcer.java | 51 ++++--- .../scheduler/fair/TestFairScheduler.java | 137 ++++++++++++++++++ .../fair/TestMaxRunningAppsEnforcer.java | 3 +- .../src/site/apt/FairScheduler.apt.vm | 34 ++++- 8 files changed, 326 insertions(+), 30 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b2f70e41afd..0cdfc6fa567 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -14,6 +14,9 @@ Trunk - Unreleased YARN-1504. RM changes for moving apps between queues (Sandy Ryza) + YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy + Ryza) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 1c4a5a638c5..4d6ca0eedbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @Unstable public class SchedulerApplication { - private final Queue queue; + private Queue queue; private final String user; private SchedulerApplicationAttempt currentAttempt; @@ -37,6 +37,10 @@ public class SchedulerApplication { public Queue getQueue() { return queue; } + + public void setQueue(Queue queue) { + this.queue = queue; + } public String getUser() { return user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 5a33dcb1829..6fc90f472c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -39,7 +39,8 @@ public class AllocationConfiguration { // Minimum resource allocation for each queue private final Map minQueueResources; // Maximum amount of resources per queue - private final Map maxQueueResources; + @VisibleForTesting + final Map maxQueueResources; // Sharing weights for each queue private final Map queueWeights; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b88ad503dd1..e057e740fb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -766,7 +766,9 @@ public class FairScheduler extends AbstractYarnScheduler { boolean wasRunnable = queue.removeApp(attempt); if (wasRunnable) { - maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt); + maxRunningEnforcer.untrackRunnableApp(attempt); + maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, + attempt.getQueue()); } else { maxRunningEnforcer.untrackNonRunnableApp(attempt); } @@ -1355,4 +1357,119 @@ public class FairScheduler extends AbstractYarnScheduler { queue.collectSchedulerApplications(apps); return apps; } + + @Override + public synchronized String moveApplication(ApplicationId appId, + String queueName) throws YarnException { + SchedulerApplication app = applications.get(appId); + if (app == null) { + throw new YarnException("App to be moved " + appId + " not found."); + } + FSSchedulerApp attempt = (FSSchedulerApp) app.getCurrentAppAttempt(); + + FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); + FSLeafQueue targetQueue = queueMgr.getLeafQueue(queueName, false); + if (targetQueue == null) { + throw new YarnException("Target queue " + queueName + + " not found or is not a leaf queue."); + } + if (targetQueue == oldQueue) { + return oldQueue.getQueueName(); + } + + if (oldQueue.getRunnableAppSchedulables().contains( + attempt.getAppSchedulable())) { + verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); + } + + executeMove(app, attempt, oldQueue, targetQueue); + return targetQueue.getQueueName(); + } + + private void verifyMoveDoesNotViolateConstraints(FSSchedulerApp app, + FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException { + String queueName = targetQueue.getQueueName(); + ApplicationAttemptId appAttId = app.getApplicationAttemptId(); + // When checking maxResources and maxRunningApps, only need to consider + // queues before the lowest common ancestor of the two queues because the + // total running apps in queues above will not be changed. + FSQueue lowestCommonAncestor = findLowestCommonAncestorQueue(oldQueue, + targetQueue); + Resource consumption = app.getCurrentConsumption(); + + // Check whether the move would go over maxRunningApps or maxShare + FSQueue cur = targetQueue; + while (cur != lowestCommonAncestor) { + // maxRunningApps + if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) { + throw new YarnException("Moving app attempt " + appAttId + " to queue " + + queueName + " would violate queue maxRunningApps constraints on" + + " queue " + cur.getQueueName()); + } + + // maxShare + if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption), + cur.getMaxShare())) { + throw new YarnException("Moving app attempt " + appAttId + " to queue " + + queueName + " would violate queue maxShare constraints on" + + " queue " + cur.getQueueName()); + } + + cur = cur.getParent(); + } + } + + /** + * Helper for moveApplication, which is synchronized, so all operations will + * be atomic. + */ + private void executeMove(SchedulerApplication app, FSSchedulerApp attempt, + FSLeafQueue oldQueue, FSLeafQueue newQueue) { + boolean wasRunnable = oldQueue.removeApp(attempt); + // if app was not runnable before, it may be runnable now + boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, + attempt.getUser()); + if (wasRunnable && !nowRunnable) { + throw new IllegalStateException("Should have already verified that app " + + attempt.getApplicationId() + " would be runnable in new queue"); + } + + if (wasRunnable) { + maxRunningEnforcer.untrackRunnableApp(attempt); + } else if (nowRunnable) { + // App has changed from non-runnable to runnable + maxRunningEnforcer.untrackNonRunnableApp(attempt); + } + + attempt.move(newQueue); // This updates all the metrics + app.setQueue(newQueue); + newQueue.addApp(attempt, nowRunnable); + + if (nowRunnable) { + maxRunningEnforcer.trackRunnableApp(attempt); + } + if (wasRunnable) { + maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, oldQueue); + } + } + + private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { + // Because queue names include ancestors, separated by periods, we can find + // the lowest common ancestors by going from the start of the names until + // there's a character that doesn't match. + String name1 = queue1.getName(); + String name2 = queue2.getName(); + // We keep track of the last period we encounter to avoid returning root.apple + // when the queues are root.applepie and root.appletart + int lastPeriodIndex = -1; + for (int i = 0; i < Math.max(name1.length(), name2.length()); i++) { + if (name1.length() <= i || name2.length() <= i || + name1.charAt(i) != name2.charAt(i)) { + return queueMgr.getQueue(name1.substring(lastPeriodIndex)); + } else if (name1.charAt(i) == '.') { + lastPeriodIndex = i; + } + } + return queue1; // names are identical + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java index 862e44d0b98..359519a2f29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java @@ -105,26 +105,15 @@ public class MaxRunningAppsEnforcer { } /** - * Updates the relevant tracking variables after a runnable app with the given - * queue and user has been removed. Checks to see whether any other applications - * are now runnable and makes them so. + * Checks to see whether any other applications runnable now that the given + * application has been removed from the given queue. And makes them so. * * Runs in O(n log(n)) where n is the number of queues that are under the * highest queue that went from having no slack to having slack. */ - public void updateRunnabilityOnAppRemoval(FSSchedulerApp app) { + public void updateRunnabilityOnAppRemoval(FSSchedulerApp app, FSLeafQueue queue) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); - // Update usersRunnableApps - String user = app.getUser(); - int newUserNumRunning = usersNumRunnableApps.get(user) - 1; - if (newUserNumRunning == 0) { - usersNumRunnableApps.remove(user); - } else { - usersNumRunnableApps.put(user, newUserNumRunning); - } - - // Update runnable app bookkeeping for queues: // childqueueX might have no pending apps itself, but if a queue higher up // in the hierarchy parentqueueY has a maxRunningApps set, an app completion // in childqueueX could allow an app in some other distant child of @@ -133,16 +122,14 @@ public class MaxRunningAppsEnforcer { // the queue was already at its max before the removal. // Thus we find the ancestor queue highest in the tree for which the app // that was at its maxRunningApps before the removal. - FSLeafQueue queue = app.getQueue(); FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() == allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null; FSParentQueue parent = queue.getParent(); while (parent != null) { if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent - .getName())) { + .getName()) - 1) { highestQueueWithAppsNowRunnable = parent; } - parent.decrementRunnableApps(); parent = parent.getParent(); } @@ -157,7 +144,12 @@ public class MaxRunningAppsEnforcer { gatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable); } - if (newUserNumRunning == allocConf.getUserMaxApps(user) - 1) { + String user = app.getUser(); + Integer userNumRunning = usersNumRunnableApps.get(user); + if (userNumRunning == null) { + userNumRunning = 0; + } + if (userNumRunning == allocConf.getUserMaxApps(user) - 1) { List userWaitingApps = usersNonRunnableApps.get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.add(userWaitingApps); @@ -208,6 +200,29 @@ public class MaxRunningAppsEnforcer { } } + /** + * Updates the relevant tracking variables after a runnable app with the given + * queue and user has been removed. + */ + public void untrackRunnableApp(FSSchedulerApp app) { + // Update usersRunnableApps + String user = app.getUser(); + int newUserNumRunning = usersNumRunnableApps.get(user) - 1; + if (newUserNumRunning == 0) { + usersNumRunnableApps.remove(user); + } else { + usersNumRunnableApps.put(user, newUserNumRunning); + } + + // Update runnable app bookkeeping for queues + FSLeafQueue queue = app.getQueue(); + FSParentQueue parent = queue.getParent(); + while (parent != null) { + parent.decrementRunnableApps(); + parent = parent.getParent(); + } + } + /** * Stops tracking the given non-runnable app */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 892438902de..a84db75c2d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -2547,4 +2550,138 @@ public class TestFairScheduler { TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } + + @Test + public void testMoveRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + ApplicationId appId = appAttId.getApplicationId(); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage()); + scheduler.update(); + assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand()); + + scheduler.moveApplication(appId, "queue2"); + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertSame(targetQueue, app.getQueue()); + assertFalse(oldQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage()); + assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage()); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + + scheduler.update(); + assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand()); + assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand()); + } + + @Test + public void testMoveNonRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + assertEquals(0, oldQueue.getNumRunnableApps()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(0, targetQueue.getNumRunnableApps()); + assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test + public void testMoveMakesAppRunnable() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertTrue(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertFalse(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertFalse(targetQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxAppsConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", + Resource.newInstance(1024, 1)); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveToNonexistentQueue() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + scheduler.getQueueManager().getLeafQueue("queue1", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index 51daeec42e3..c1866f01cd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer { private void removeApp(FSSchedulerApp app) { app.getQueue().removeApp(app); - maxAppsEnforcer.updateRunnabilityOnAppRemoval(app); + maxAppsEnforcer.untrackRunnableApp(app); + maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 655de70c1d2..97436fb8e54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs) * {Administration} - The fair scheduler provides support for administration at runtime through two mechanisms: + The fair scheduler provides support for administration at runtime through a few mechanisms: - * It is possible to modify minimum shares, limits, weights, preemption timeouts - and queue scheduling policies at runtime by editing the allocation file. The - scheduler will reload this file 10-15 seconds after it sees that it was - modified. +Modifying configuration at runtime - * Current applications, queues, and fair shares can be examined through the - ResourceManager's web interface, at - http:///cluster/scheduler. + It is possible to modify minimum shares, limits, weights, preemption timeouts + and queue scheduling policies at runtime by editing the allocation file. The + scheduler will reload this file 10-15 seconds after it sees that it was + modified. + +Monitoring through web UI + + Current applications, queues, and fair shares can be examined through the + ResourceManager's web interface, at + http:///cluster/scheduler. The following fields can be seen for each queue on the web interface: @@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs) In addition to the information that the ResourceManager normally displays about each application, the web interface includes the application's fair share. +Moving applications between queues + + The Fair Scheduler supports moving a running application to a different queue. + This can be useful for moving an important application to a higher priority + queue, or for moving an unimportant application to a lower priority queue. + Apps can be moved by running "yarn application -movetoqueue appID -queue + targetQueueName". + + When an application is moved to a queue, its existing allocations become + counted with the new queue's allocations instead of the old for purposes + of determining fairness. An attempt to move an application to a queue will + fail if the addition of the app's resources to that queue would violate the + its maxRunningApps or maxResources constraints. + From dcaaeefeaf8e15998a7471fd463f3783f9cf9e5b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 5 Feb 2014 19:17:20 +0000 Subject: [PATCH 29/54] Addendum patch for HDFS-5709, add missing license header. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564885 13f79535-47bb-0310-9956-ffa450edef68 --- .../namenode/TestNameNodeOptionParsing.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java index d43eda0cefd..5f47cdabc30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; From 69dbf0b22587e4f8fe2651ebd53f873920f29c91 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 5 Feb 2014 19:43:56 +0000 Subject: [PATCH 30/54] HDFS-5876. SecureDataNodeStarter does not pick up configuration in hdfs-site.xml. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564897 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/datanode/DataNode.java | 11 ++++++----- .../hdfs/server/datanode/SecureDataNodeStarter.java | 5 ++++- .../hdfs/server/namenode/TestNameNodeHttpServer.java | 1 + 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5b39abfeeaf..d42457d6ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -869,6 +869,9 @@ Release 2.3.0 - UNRELEASED HDFS-5399. Revisit SafeModeException and corresponding retry policies. (Jing Zhao via todd) + HDFS-5876. SecureDataNodeStarter does not pick up configuration in + hdfs-site.xml. (Haohui Mai via jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 1dabd4a94c6..ad580a53d1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -362,13 +362,13 @@ public class DataNode extends Configured .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))); HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); - InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); - String infoHost = infoSocAddr.getHostName(); if (policy.isHttpEnabled()) { if (secureResources == null) { + InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); int port = infoSocAddr.getPort(); - builder.addEndpoint(URI.create("http://" + infoHost + ":" + port)); + builder.addEndpoint(URI.create("http://" + + NetUtils.getHostPortString(infoSocAddr))); if (port == 0) { builder.setFindPort(true); } @@ -381,7 +381,7 @@ public class DataNode extends Configured if (policy.isHttpsEnabled()) { InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get( - DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 0)); + DFS_DATANODE_HTTPS_ADDRESS_KEY, DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)); Configuration sslConf = DFSUtil.loadSslConfiguration(conf); DFSUtil.loadSslConfToHttpServerBuilder(builder, sslConf); @@ -390,7 +390,8 @@ public class DataNode extends Configured if (port == 0) { builder.setFindPort(true); } - builder.addEndpoint(URI.create("https://" + infoHost + ":" + port)); + builder.addEndpoint(URI.create("https://" + + NetUtils.getHostPortString(secInfoSocAddr))); } this.infoServer = builder.build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java index 000f7aedd2d..477b7f66558 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java @@ -25,6 +25,7 @@ import org.apache.commons.daemon.DaemonContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; @@ -62,7 +63,9 @@ public class SecureDataNodeStarter implements Daemon { @Override public void init(DaemonContext context) throws Exception { System.err.println("Initializing secure datanode resources"); - Configuration conf = new Configuration(); + // Create a new HdfsConfiguration object to ensure that the configuration in + // hdfs-site.xml is picked up. + Configuration conf = new HdfsConfiguration(); // Stash command-line arguments for regular datanode args = context.getArguments(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index be32b5b2728..39983b10ed2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -85,6 +85,7 @@ public class TestNameNodeHttpServer { @Test public void testHttpPolicy() throws Exception { conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); InetSocketAddress addr = InetSocketAddress.createUnresolved("localhost", 0); NameNodeHttpServer server = null; From d598b6ef9f10ae011fecbe198360cde63a4e4d50 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 5 Feb 2014 22:48:06 +0000 Subject: [PATCH 31/54] HDFS-5873. dfs.http.policy should have higher precedence over dfs.https.enable. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564973 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/http/HttpConfig.java | 11 ++-- .../src/site/apt/SecureMode.apt.vm | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 39 +++++++------- .../apache/hadoop/hdfs/TestHttpPolicy.java | 54 +++++++++++++++++++ .../namenode/TestNameNodeHttpServer.java | 4 +- 6 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index c533fedf65e..d323f764359 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -34,13 +34,14 @@ public class HttpConfig { HTTPS_ONLY, HTTP_AND_HTTPS; + private static final Policy[] VALUES = values(); public static Policy fromString(String value) { - if (HTTPS_ONLY.name().equalsIgnoreCase(value)) { - return HTTPS_ONLY; - } else if (HTTP_AND_HTTPS.name().equalsIgnoreCase(value)) { - return HTTP_AND_HTTPS; + for (Policy p : VALUES) { + if (p.name().equalsIgnoreCase(value)) { + return p; + } } - return HTTP_ONLY; + return null; } public boolean isHttpEnabled() { diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm index 9bd55a67fff..68ca4b0da68 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm @@ -352,7 +352,8 @@ Configuration for <<>> | | | This value is deprecated. Use dfs.http.policy | *-------------------------+-------------------------+------------------------+ | <<>> | or or | | -| | | HTTPS_ONLY turns off http access | +| | | HTTPS_ONLY turns off http access. This option takes precedence over | +| | | the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. | *-------------------------+-------------------------+------------------------+ | <<>> | | | *-------------------------+-------------------------+------------------------+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d42457d6ea6..0ad51b75692 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -872,6 +872,9 @@ Release 2.3.0 - UNRELEASED HDFS-5876. SecureDataNodeStarter does not pick up configuration in hdfs-site.xml. (Haohui Mai via jing9) + HDFS-5873. dfs.http.policy should have higher precedence over dfs.https.enable. + (Haohui Mai via jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 9274f505a4a..8dccff0d32d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1567,31 +1567,34 @@ public class DFSUtil { * configuration settings. */ public static HttpConfig.Policy getHttpPolicy(Configuration conf) { - String httpPolicy = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY, - DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); - - HttpConfig.Policy policy = HttpConfig.Policy.fromString(httpPolicy); - - if (policy == HttpConfig.Policy.HTTP_ONLY) { - boolean httpsEnabled = conf.getBoolean( - DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, + String policyStr = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY); + if (policyStr == null) { + boolean https = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT); - boolean hadoopSslEnabled = conf.getBoolean( + boolean hadoopSsl = conf.getBoolean( CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY, CommonConfigurationKeys.HADOOP_SSL_ENABLED_DEFAULT); - if (hadoopSslEnabled) { + if (hadoopSsl) { LOG.warn(CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY - + " is deprecated. Please use " - + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); - policy = HttpConfig.Policy.HTTPS_ONLY; - } else if (httpsEnabled) { - LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY - + " is deprecated. Please use " - + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); - policy = HttpConfig.Policy.HTTP_AND_HTTPS; + + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY + + "."); } + if (https) { + LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY + + "."); + } + + return (hadoopSsl || https) ? HttpConfig.Policy.HTTP_AND_HTTPS + : HttpConfig.Policy.HTTP_ONLY; + } + + HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr); + if (policy == null) { + throw new HadoopIllegalArgumentException("Unregonized value '" + + policyStr + "' for " + DFSConfigKeys.DFS_HTTP_POLICY_KEY); } conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java new file mode 100644 index 00000000000..e448e151cb6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.apache.hadoop.http.HttpConfig.Policy.HTTP_AND_HTTPS; +import static org.apache.hadoop.http.HttpConfig.Policy.HTTP_ONLY; + +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +public final class TestHttpPolicy { + + @Test(expected = HadoopIllegalArgumentException.class) + public void testInvalidPolicyValue() { + Configuration conf = new Configuration(); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, "invalid"); + DFSUtil.getHttpPolicy(conf); + } + + @Test + public void testDeprecatedConfiguration() { + Configuration conf = new Configuration(false); + Assert.assertSame(HTTP_ONLY, DFSUtil.getHttpPolicy(conf)); + + conf.setBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, true); + Assert.assertSame(HTTP_AND_HTTPS, DFSUtil.getHttpPolicy(conf)); + + conf = new Configuration(false); + conf.setBoolean(DFSConfigKeys.HADOOP_SSL_ENABLED_KEY, true); + Assert.assertSame(HTTP_AND_HTTPS, DFSUtil.getHttpPolicy(conf)); + + conf = new Configuration(false); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HTTP_ONLY.name()); + conf.setBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, true); + Assert.assertSame(HTTP_ONLY, DFSUtil.getHttpPolicy(conf)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index 39983b10ed2..975a6649e60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -104,7 +104,9 @@ public class TestNameNodeHttpServer { server.getHttpsAddress() == null)); } finally { - server.stop(); + if (server != null) { + server.stop(); + } } } From 996e25c3d11228b1c30a811a5df2b3024fa4640b Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 6 Feb 2014 00:13:34 +0000 Subject: [PATCH 32/54] HADOOP-10325. Improve Jenkins Javadoc warnings from test-patch.sh (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565010 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 49 +++++++++++++------ .../hadoop-common/CHANGES.txt | 2 + 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 7143b514060..ed671a64ebb 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -300,6 +300,17 @@ prebuildWithoutPatch () { {color:red}-1 patch{color}. Trunk compilation may be broken." return 1 fi + + echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1" + $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Trunk javadoc compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. Trunk compilation may be broken." + return 1 + fi + return 0 } @@ -401,6 +412,11 @@ applyPatch () { } ############################################################################### +calculateJavadocWarnings() { + WARNING_FILE="$1" + RET=$(egrep "^[0-9]+ warnings$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') +} + ### Check there are no javadoc warnings checkJavadocWarnings () { echo "" @@ -420,24 +436,29 @@ checkJavadocWarnings () { (cd hadoop-common-project/hadoop-annotations; $MVN install > /dev/null 2>&1) fi $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "" - echo "" - echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." + calculateJavadocWarnings "$PATCH_DIR/trunkJavadocWarnings.txt" + numTrunkJavadocWarnings=$RET + calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" + numPatchJavadocWarnings=$RET + grep -i warning "$PATCH_DIR/trunkJavadocWarnings.txt" > "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" + grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + diff -u "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" \ + "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ + "$PATCH_DIR/diffJavadocWarnings.txt" + rm -f "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + echo "There appear to be $numTrunkJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." + if [[ $numTrunkJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then + if [[ $numPatchJavadocWarnings -gt $numTrunkJavadocWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT - #There are 12 warnings that are caused by things that are caused by using sun internal APIs. - #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. - OK_JAVADOC_WARNINGS=14; - ### if current warnings greater than OK_JAVADOC_WARNINGS - if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages." - return 1 + {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$numTrunkJavadocWarnings))` warning messages. + See $BUILD_URL/artifact/trunk/patchprocess/diffJavadocWarnings.txt for details." + return 1 + fi fi JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 javadoc{color}. The javadoc tool did not generate any warning messages." + {color:green}+1 javadoc{color}. There were no new javadoc warning messages." return 0 } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e3a39daa581..31e673157eb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -116,6 +116,8 @@ Trunk (Unreleased) HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry McCay via omalley) + HADOOP-10325. Improve jenkins javadoc warnings from test-patch.sh (cmccabe) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. From b36cc29073a22ec3bb14f1f7b34dc5518fd8c6a2 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 06:20:41 +0000 Subject: [PATCH 33/54] MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565087 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/app/TestRMContainerAllocator.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fae27e4929a..23171ccc1bd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -347,6 +347,9 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5723. MR AM container log can be truncated or empty. (Mohammad Kamrul Islam via kasha) + MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. + (Vinod Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 3eb5222865c..c62fa39c2ea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1652,8 +1652,16 @@ public class TestRMContainerAllocator { RMApp app = rm.submitApp(1024); dispatcher.await(); + // Make a node to register so as to launch the AM. + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + amNodeManager.nodeHeartbeat(true); + dispatcher.await(); + ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); + rm.sendAMLaunched(appAttemptId); + dispatcher.await(); + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job job = mock(Job.class); when(job.getReport()).thenReturn( From 24775c10efbff28a323ef9128120f0cbf06d17a6 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 07:21:42 +0000 Subject: [PATCH 34/54] YARN-1628. Fixed the test failure in TestContainerManagerSecurity. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565094 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/test/resources/core-site.xml | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0cdfc6fa567..7f7b795c0eb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -627,6 +627,9 @@ Release 2.3.0 - UNRELEASED YARN-1629. IndexOutOfBoundsException in MaxRunningAppsEnforcer (Sandy Ryza) + YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml new file mode 100644 index 00000000000..f0d3085ef85 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + + + hadoop.security.token.service.use_ip + false + + + From ab96a0838dafbfea77382135914feadbfd03cf53 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 6 Feb 2014 15:45:47 +0000 Subject: [PATCH 35/54] HDFS-5881. Fix skip() of the short-circuit local reader(legacy). Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565310 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java | 6 +++--- .../main/java/org/apache/hadoop/hdfs/DFSInputStream.java | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0ad51b75692..2720c7ddd1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -337,6 +337,8 @@ Release 2.4.0 - UNRELEASED HDFS-5709. Improve NameNode upgrade with existing reserved paths and path components. (Andrew Wang via atm) + HDFS-5881. Fix skip() of the short-circuit local reader(legacy). (kihwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java index 2f661933619..ffc4eb9f8ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java @@ -629,7 +629,7 @@ class BlockReaderLocalLegacy implements BlockReader { skipBuf = new byte[bytesPerChecksum]; } int ret = read(skipBuf, 0, (int)(n - remaining)); - return ret; + return (remaining + ret); } // optimize for big gap: discard the current buffer, skip to @@ -660,9 +660,9 @@ class BlockReaderLocalLegacy implements BlockReader { int ret = read(skipBuf, 0, myOffsetFromChunkBoundary); if (ret == -1) { // EOS - return toskip; + return (toskip + remaining); } else { - return (toskip + ret); + return (toskip + remaining + ret); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 73861bc8ade..438030eaa97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -1345,6 +1345,14 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, pos += blockReader.skip(diff); if (pos == targetPos) { done = true; + } else { + // The range was already checked. If the block reader returns + // something unexpected instead of throwing an exception, it is + // most likely a bug. + String errMsg = "BlockReader failed to seek to " + + targetPos + ". Instead, it seeked to " + pos + "."; + DFSClient.LOG.warn(errMsg); + throw new IOException(errMsg); } } catch (IOException e) {//make following read to retry if(DFSClient.LOG.isDebugEnabled()) { From 4594b74b8536aea308854493a0e82c8b2919174b Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 6 Feb 2014 18:29:30 +0000 Subject: [PATCH 36/54] MAPREDUCE-5699. Allow setting tags on MR jobs (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565384 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 ++ .../src/main/resources/mapred-default.xml | 10 +++++++++- .../main/java/org/apache/hadoop/mapred/YARNRunner.java | 7 +++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 23171ccc1bd..d3eac8d94b6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -153,6 +153,8 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5732. Report proper queue when job has been automatically placed (Sandy Ryza) + MAPREDUCE-5699. Allow setting tags on MR jobs (kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index f1a3e453fb3..ffb95433513 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -60,6 +60,8 @@ public interface MRJobConfig { public static final String QUEUE_NAME = "mapreduce.job.queuename"; + public static final String JOB_TAGS = "mapreduce.job.tags"; + public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String SPLIT_FILE = "mapreduce.job.splitfile"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index dea2adf1350..cc663923eaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1,4 +1,5 @@ + - @@ -727,6 +727,14 @@ + + mapreduce.job.tags + + Tags for the job that will be passed to YARN at submission + time. Queries to YARN for applications can filter on these tags. + + + mapreduce.cluster.local.dir ${hadoop.tmp.dir}/mapred/local diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index fc23c6541f9..ce475c197c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -21,7 +21,9 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Vector; @@ -467,6 +469,8 @@ public class YARNRunner implements ClientProtocol { ContainerLaunchContext.newInstance(localResources, environment, vargsFinal, null, securityTokens, acls); + Collection tagsFromConf = + jobConf.getTrimmedStringCollection(MRJobConfig.JOB_TAGS); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = @@ -486,6 +490,9 @@ public class YARNRunner implements ClientProtocol { MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); appContext.setResource(capability); appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE); + if (tagsFromConf != null && !tagsFromConf.isEmpty()) { + appContext.setApplicationTags(new HashSet(tagsFromConf)); + } return appContext; } From c4b0ce0ace8381be299a7fa013e17bfe3294e0bf Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 6 Feb 2014 18:40:23 +0000 Subject: [PATCH 37/54] HADOOP-10327. Trunk windows build broken after HDFS-5746. Contributed by Vinay. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565389 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/io/nativeio/NativeIO.c | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 31e673157eb..2698cc2a787 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -324,6 +324,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + HADOOP-10327. Trunk windows build broken after HDFS-5746. + (Vinay via cnauroth) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 2b5c0a4ab0a..79c9b9d4697 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -671,6 +671,7 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mmap( JNIEnv *env, jclass clazz, jobject jfd, jint jprot, jboolean jshared, jlong length) { +#ifdef UNIX void *addr = 0; int prot, flags, fd; @@ -684,18 +685,33 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mmap( throw_ioe(env, errno); } return (jlong)(intptr_t)addr; +#endif // UNIX + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.mmap() is not supported on Windows"); + return NULL; +#endif } JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_munmap( JNIEnv *env, jclass clazz, jlong jaddr, jlong length) { +#ifdef UNIX void *addr; addr = (void*)(intptr_t)jaddr; if (munmap(addr, length) < 0) { throw_ioe(env, errno); } +#endif // UNIX + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.munmap() is not supported on Windows"); + return NULL; +#endif } @@ -1050,4 +1066,3 @@ JNIEnv *env, jclass clazz) /** * vim: sw=2: ts=2: et: */ - From 6182e7592d7255792ae6a04a5551296d755f6a37 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 18:56:46 +0000 Subject: [PATCH 38/54] YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565394 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../distributedshell/ApplicationMaster.java | 15 ++++++++------- .../ContainerLaunchFailAppMaster.java | 10 +++++----- .../distributedshell/TestDSFailedAppMaster.java | 13 +++++++------ 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7f7b795c0eb..31103413074 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -630,6 +630,9 @@ Release 2.3.0 - UNRELEASED YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod Kumar Vavilapalli via zjshen) + YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 9b6e788c3cb..8d869a225cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -232,7 +232,6 @@ public class ApplicationMaster { private static final String shellArgsPath = "shellArgs"; private volatile boolean done; - private volatile boolean success; private ByteBuffer allTokens; @@ -254,8 +253,8 @@ public class ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); @@ -480,7 +479,7 @@ public class ApplicationMaster { * @throws IOException */ @SuppressWarnings({ "unchecked" }) - public boolean run() throws YarnException, IOException { + public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); Credentials credentials = @@ -561,7 +560,6 @@ public class ApplicationMaster { amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); - return success; } @VisibleForTesting @@ -569,7 +567,8 @@ public class ApplicationMaster { return new NMCallbackHandler(this); } - protected void finish() { + @VisibleForTesting + protected boolean finish() { // wait for completion. while (!done && (numCompletedContainers.get() != numTotalContainers)) { @@ -600,7 +599,7 @@ public class ApplicationMaster { FinalApplicationStatus appStatus; String appMessage = null; - success = true; + boolean success = true; if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { appStatus = FinalApplicationStatus.SUCCEEDED; @@ -621,6 +620,8 @@ public class ApplicationMaster { } amRMClient.stop(); + + return success; } private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java index e845490e35b..8e561c67362 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java @@ -18,13 +18,13 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.nio.ByteBuffer; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ContainerId; -import java.nio.ByteBuffer; -import java.util.Map; - public class ContainerLaunchFailAppMaster extends ApplicationMaster { private static final Log LOG = @@ -66,8 +66,8 @@ public class ContainerLaunchFailAppMaster extends ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java index 644f66799b2..db7419bc8e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java @@ -29,8 +29,8 @@ public class TestDSFailedAppMaster extends ApplicationMaster { private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class); @Override - public boolean run() throws YarnException, IOException { - boolean res = super.run(); + public void run() throws YarnException, IOException { + super.run(); // for the 2nd attempt. if (appAttemptID.getAttemptId() == 2) { @@ -39,11 +39,12 @@ public class TestDSFailedAppMaster extends ApplicationMaster { // numRequestedContainers should be set to 0. if (numAllocatedContainers.get() != 1 || numRequestedContainers.get() != 0) { - LOG.info("Application Master failed. exiting"); + LOG.info("NumAllocatedContainers is " + numAllocatedContainers.get() + + " and NumRequestedContainers is " + numAllocatedContainers.get() + + ".Application Master failed. exiting"); System.exit(200); } } - return res; } public static void main(String[] args) { @@ -54,7 +55,7 @@ public class TestDSFailedAppMaster extends ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); + appMaster.run(); if (appMaster.appAttemptID.getAttemptId() == 1) { try { // sleep some time, wait for the AM to launch a container. @@ -63,7 +64,7 @@ public class TestDSFailedAppMaster extends ApplicationMaster { // fail the first am. System.exit(100); } - appMaster.finish(); + result = appMaster.finish(); } catch (Throwable t) { System.exit(1); } From b527a975a4899e2b04c94eba46879625de1111e6 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 6 Feb 2014 19:07:18 +0000 Subject: [PATCH 39/54] MAPREDUCE-5743. Fixed CHANGES.txt to give credit to Ted too. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565401 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d3eac8d94b6..33db5bf461f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -350,7 +350,7 @@ Release 2.3.0 - UNRELEASED (Mohammad Kamrul Islam via kasha) MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. - (Vinod Kumar Vavilapalli via zjshen) + (Ted Yu and Vinod Kumar Vavilapalli via zjshen) Release 2.2.0 - 2013-10-13 From e0cda4895948c500a7bbc0a1a553d3698be3e317 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 6 Feb 2014 21:08:09 +0000 Subject: [PATCH 40/54] HDFS-4911. Reduce PeerCache timeout to be commensurate with dfs.datanode.socket.reuse.keepalive (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565435 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../org/apache/hadoop/hdfs/PeerCache.java | 20 ++- .../hdfs/TestDataTransferKeepalive.java | 126 ++++++++++++------ 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2720c7ddd1f..b0cc1fdebae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -308,6 +308,9 @@ Release 2.4.0 - UNRELEASED HDFS-5746. Add ShortCircuitSharedMemorySegment (cmccabe) + HDFS-4911. Reduce PeerCache timeout to be commensurate with + dfs.datanode.socket.reuse.keepalive (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fe1d3d1570e..02297a64fd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -86,7 +86,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY = "dfs.client.socketcache.expiryMsec"; - public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 2 * 60 * 1000; + public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 3000; public static final String DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL = "dfs.client.write.exclude.nodes.cache.expiry.interval.millis"; public static final long DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10 * 60 * 1000; // 10 minutes, in ms public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address"; @@ -215,7 +215,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose"; public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false; public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive"; - public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000; + public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 4000; public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java index 424b641c8c3..ba6736a9db4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; @@ -25,6 +26,7 @@ import java.util.Map.Entry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.LinkedListMultimap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -118,6 +120,11 @@ class PeerCache { return instance; } + @VisibleForTesting + public static synchronized void setInstance(int c, long e) { + instance = new PeerCache(c, e); + } + private boolean isDaemonStarted() { return (daemon == null)? false: true; } @@ -171,8 +178,17 @@ class PeerCache { while (iter.hasNext()) { Value candidate = iter.next(); iter.remove(); - if (!candidate.getPeer().isClosed()) { - return candidate.getPeer(); + long ageMs = Time.monotonicNow() - candidate.getTime(); + Peer peer = candidate.getPeer(); + if (ageMs >= expiryPeriod) { + try { + peer.close(); + } catch (IOException e) { + LOG.warn("got IOException closing stale peer " + peer + + ", which is " + ageMs + " ms old"); + } + } else if (!peer.isClosed()) { + return peer; } } return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java index bf4e13bd027..bdfc62d5fde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java @@ -19,16 +19,19 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Supplier; + import java.io.InputStream; import java.io.PrintWriter; -import java.net.InetSocketAddress; -import java.net.Socket; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -37,10 +40,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ReflectionUtils; import org.junit.After; import org.junit.Before; @@ -51,10 +52,7 @@ import com.google.common.io.NullOutputStream; public class TestDataTransferKeepalive { Configuration conf = new HdfsConfiguration(); private MiniDFSCluster cluster; - private FileSystem fs; - private InetSocketAddress dnAddr; private DataNode dn; - private DFSClient dfsClient; private static Path TEST_FILE = new Path("/test"); private static final int KEEPALIVE_TIMEOUT = 1000; @@ -69,15 +67,7 @@ public class TestDataTransferKeepalive { cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1).build(); - fs = cluster.getFileSystem(); - dfsClient = ((DistributedFileSystem)fs).dfs; - dfsClient.peerCache.clear(); - - String poolId = cluster.getNamesystem().getBlockPoolId(); dn = cluster.getDataNodes().get(0); - DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP( - dn, poolId); - dnAddr = NetUtils.createSocketAddr(dnReg.getXferAddr()); } @After @@ -90,34 +80,86 @@ public class TestDataTransferKeepalive { * its configured keepalive timeout. */ @Test(timeout=30000) - public void testKeepaliveTimeouts() throws Exception { + public void testDatanodeRespectsKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 60000L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Clients that write aren't currently re-used. - assertEquals(0, dfsClient.peerCache.size()); + assertEquals(0, fs.dfs.peerCache.size()); assertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.readFile(fs, TEST_FILE); - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); assertXceiverCount(1); // Sleep for a bit longer than the keepalive timeout // and make sure the xceiver died. - Thread.sleep(KEEPALIVE_TIMEOUT * 2); + Thread.sleep(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT + 1); assertXceiverCount(0); // The socket is still in the cache, because we don't // notice that it's closed until we try to read // from it again. - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); // Take it out of the cache - reading should // give an EOF. - Peer peer = dfsClient.peerCache.get(dn.getDatanodeId(), false); + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); assertNotNull(peer); assertEquals(-1, peer.getInputStream().read()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); + } + + /** + * Test that the client respects its keepalive timeout. + */ + @Test(timeout=30000) + public void testClientResponsesKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much shorter than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 10L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); + + // Clients that write aren't currently re-used. + assertEquals(0, fs.dfs.peerCache.size()); + assertXceiverCount(0); + + // Reads the file, so we should get a + // cached socket, and should have an xceiver on the other side. + DFSTestUtil.readFile(fs, TEST_FILE); + assertEquals(1, fs.dfs.peerCache.size()); + assertXceiverCount(1); + + // Sleep for a bit longer than the client keepalive timeout. + Thread.sleep(CLIENT_EXPIRY_MS + 1); + + // Taking out a peer which is expired should give a null. + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); + assertTrue(peer == null); + + // The socket cache is now empty. + assertEquals(0, fs.dfs.peerCache.size()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); } /** @@ -125,8 +167,17 @@ public class TestDataTransferKeepalive { * read bytes off the stream quickly. The datanode should time out sending the * chunks and the transceiver should die, even if it has a long keepalive. */ - @Test(timeout=30000) + @Test(timeout=300000) public void testSlowReader() throws Exception { + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 600000L; + Configuration clientConf = new Configuration(conf); + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); // Restart the DN with a shorter write timeout. DataNodeProperties props = cluster.stopDataNode(0); props.conf.setInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY, @@ -134,38 +185,31 @@ public class TestDataTransferKeepalive { props.conf.setInt(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY, 120000); assertTrue(cluster.restartDataNode(props, true)); + dn = cluster.getDataNodes().get(0); // Wait for heartbeats to avoid a startup race where we // try to write the block while the DN is still starting. cluster.triggerHeartbeats(); - dn = cluster.getDataNodes().get(0); - DFSTestUtil.createFile(fs, TEST_FILE, 1024*1024*8L, (short)1, 0L); FSDataInputStream stm = fs.open(TEST_FILE); - try { - stm.read(); - assertXceiverCount(1); + stm.read(); + assertXceiverCount(1); - // Poll for 0 running xceivers. Allow up to 5 seconds for some slack. - long totalSleepTime = 0; - long sleepTime = WRITE_TIMEOUT + 100; - while (getXceiverCountWithoutServer() > 0 && totalSleepTime < 5000) { - Thread.sleep(sleepTime); - totalSleepTime += sleepTime; - sleepTime = 100; + GenericTestUtils.waitFor(new Supplier() { + public Boolean get() { + // DN should time out in sendChunks, and this should force + // the xceiver to exit. + return getXceiverCountWithoutServer() == 0; } + }, 500, 50000); - // DN should time out in sendChunks, and this should force - // the xceiver to exit. - assertXceiverCount(0); - } finally { - IOUtils.closeStream(stm); - } + IOUtils.closeStream(stm); } @Test(timeout=30000) public void testManyClosedSocketsInCache() throws Exception { // Make a small file + DistributedFileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Insert a bunch of dead sockets in the cache, by opening From 12c2582c1f845b6ce344ec3181628d31739c8ec8 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 6 Feb 2014 21:16:01 +0000 Subject: [PATCH 41/54] HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command returns 0 result. Contributed by Tassapol Athiapinya. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565440 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b0cc1fdebae..ce457bd51b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -342,6 +342,9 @@ Release 2.4.0 - UNRELEASED HDFS-5881. Fix skip() of the short-circuit local reader(legacy). (kihwal) + HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command + returns 0 result. (Tassapol Athiapinya via cnauroth) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index b3538daf525..290e60087f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -962,9 +962,8 @@ public class CacheAdmin extends Configured implements Tool { if (numResults > 0) { System.out.print(listing); } - // If there are no results, we return 1 (failure exit code); - // otherwise we return 0 (success exit code). - return (numResults == 0) ? 1 : 0; + // If list pools succeed, we return 0 (success exit code) + return 0; } } From 0001e39cf0a913d2acb88ba177f72bf30d1ca655 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 6 Feb 2014 21:56:00 +0000 Subject: [PATCH 42/54] HADOOP-10273. Update CHANGES.txt to reflect new target version is 2.3 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565453 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2698cc2a787..b5b8e73445c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -322,8 +322,6 @@ Release 2.4.0 - UNRELEASED HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) - HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) - HADOOP-10327. Trunk windows build broken after HDFS-5746. (Vinay via cnauroth) @@ -698,6 +696,8 @@ Release 2.3.0 - UNRELEASED HADOOP-10311. Cleanup vendor names from the code base. (tucu) + HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From 914a9709b91dbbedf251e8c135019741e6ed01b2 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 6 Feb 2014 23:27:57 +0000 Subject: [PATCH 43/54] MAPREDUCE-5744. Job hangs because RMContainerAllocator.preemptReduce() violates the comparator contract (Gera Shegalov via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565478 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 33db5bf461f..3b72f402223 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -230,6 +230,10 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5725. Make explicit that TestNetworkedJob relies on the Capacity Scheduler (Sandy Ryza) + MAPREDUCE-5744. Job hangs because + RMContainerAllocator$AssignedRequests.preemptReduce() violates the + comparator contract (Gera Shegalov via kasha) + OPTIMIZATIONS MAPREDUCE-4680. Job history cleaner should only check timestamps of files in diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index a8ee06b3b48..a0c690c51ea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -1143,9 +1143,9 @@ public class RMContainerAllocator extends RMContainerRequestor new Comparator() { @Override public int compare(TaskAttemptId o1, TaskAttemptId o2) { - float p = getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress() - - getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress(); - return p >= 0 ? 1 : -1; + return Float.compare( + getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress(), + getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress()); } }); From 943b2190d72f930a76f15558fca0dbb128b2d592 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Fri, 7 Feb 2014 00:18:46 +0000 Subject: [PATCH 44/54] YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565497 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/rmapp/RMAppImpl.java | 6 +- .../yarn/server/resourcemanager/MockRM.java | 7 ++ .../yarn/server/resourcemanager/TestRM.java | 101 +++++++++++++++++- .../applicationsmanager/TestAMRestart.java | 2 +- .../rmapp/TestRMAppTransitions.java | 7 ++ .../webapp/TestRMWebServicesApps.java | 4 +- 7 files changed, 119 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 31103413074..6afd4bdbfed 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -633,6 +633,9 @@ Release 2.3.0 - UNRELEASED YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod Kumar Vavilapalli via zjshen) + YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index edbe676badd..196e89d32c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -196,10 +196,8 @@ public class RMAppImpl implements RMApp, Recoverable { // waiting for the previous AM to exit. RMAppEventType.ATTEMPT_FAILED, new AttemptFailedTransition(RMAppState.ACCEPTED)) - .addTransition(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING, - RMAppEventType.KILL, - new FinalSavingTransition( - new AppKilledTransition(), RMAppState.KILLED)) + .addTransition(RMAppState.ACCEPTED, RMAppState.KILLING, + RMAppEventType.KILL, new KillAttemptTransition()) // ACCECPTED state can once again receive APP_ACCEPTED event, because on // recovery the app returns ACCEPTED state and the app once again go // through the scheduler and triggers one more APP_ACCEPTED event at diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 935820e66b3..31035b420a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -482,6 +482,13 @@ public class MockRM extends ResourceManager { RMAppAttempt attempt = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); + rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); + return am; + } + + public static MockAM launchAndRegisterAM(RMApp app, MockRM rm, MockNM nm) + throws Exception { + MockAM am = launchAM(app, rm, nm); am.registerAppAttempt(); rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); return am; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index c7e6d7fb979..b899ea708bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.spy; + import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -33,6 +37,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; @@ -44,9 +49,17 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -54,7 +67,9 @@ import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Test; +import org.mockito.ArgumentMatcher; +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestRM { private static final Log LOG = LogFactory.getLog(TestRM.class); @@ -397,19 +412,19 @@ public class TestRM { MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishApplicationMaster(app1, rm1, nm1, am1); // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); // a killed app RMApp app3 = rm1.submitApp(200); - MockAM am3 = MockRM.launchAM(app3, rm1, nm1); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED); @@ -449,7 +464,7 @@ public class TestRM { // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); @@ -466,10 +481,88 @@ public class TestRM { Assert.assertEquals(-1, report1.getRpcPort()); } + /** + * Validate killing an application when it is at accepted state. + * @throws Exception exception + */ + @Test (timeout = 60000) + public void testApplicationKillAtAcceptedState() throws Exception { + + YarnConfiguration conf = new YarnConfiguration(); + final Dispatcher dispatcher = new AsyncDispatcher() { + @Override + public EventHandler getEventHandler() { + + class EventArgMatcher extends ArgumentMatcher { + @Override + public boolean matches(Object argument) { + if (argument instanceof RMAppAttemptEvent) { + if (((RMAppAttemptEvent) argument).getType().equals( + RMAppAttemptEventType.KILL)) { + return true; + } + } + return false; + } + } + + EventHandler handler = spy(super.getEventHandler()); + doNothing().when(handler).handle(argThat(new EventArgMatcher())); + return handler; + } + }; + + MockRM rm = new MockRM(conf) { + @Override + protected Dispatcher createDispatcher() { + return dispatcher; + } + }; + + rm.start(); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); + nm1.registerNode(); + + // a failed app + RMApp application = rm.submitApp(200); + MockAM am = MockRM.launchAM(application, rm, nm1); + am.waitForState(RMAppAttemptState.LAUNCHED); + nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); + rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED); + + // Now kill the application before new attempt is launched, the app report + // returns the invalid AM host and port. + KillApplicationRequest request = + KillApplicationRequest.newInstance(application.getApplicationId()); + rm.getClientRMService().forceKillApplication(request); + + // Specific test for YARN-1689 follows + // Now let's say a race causes AM to register now. This should not crash RM. + am.registerAppAttempt(false); + + // We explicitly intercepted the kill-event to RMAppAttempt, so app should + // still be in KILLING state. + rm.waitForState(application.getApplicationId(), RMAppState.KILLING); + // AM should now be in running + rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING); + + // Simulate that appAttempt is killed. + rm.getRMContext().getDispatcher().getEventHandler().handle( + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED)); + rm.waitForState(application.getApplicationId(), RMAppState.KILLED); + } + public static void main(String[] args) throws Exception { TestRM t = new TestRM(); t.testGetNewAppId(); t.testAppWithNoContainers(); t.testAppOnMultiNode(); + t.testNMToken(); + t.testActivatingApplicationAfterAddingNM(); + t.testInvalidateAMHostPortWhenAMFailedOrKilled(); + t.testInvalidatedAMHostPortOnAMRestart(); + t.testApplicationKillAtAcceptedState(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index f8329d68a75..ca9befd599c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -69,7 +69,7 @@ public class TestAMRestart { new MockNM("127.0.0.1:2351", 4089, rm1.getResourceTrackerService()); nm2.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); int NUM_CONTAINERS = 3; // allocate NUM_CONTAINERS containers am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 5ac9353928e..58482ee38ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -639,6 +639,13 @@ public class TestRMAppTransitions { RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + + assertAppState(RMAppState.KILLING, application); + RMAppEvent appAttemptKilled = + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED); + application.handle(appAttemptKilled); + assertAppState(RMAppState.FINAL_SAVING, application); sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 18350fb30d2..cfdf9283acf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am = MockRM.launchAM(app1, rm, amNodeManager); + MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps extends JerseyTest { } // wait for app to start a new attempt. rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); - am = MockRM.launchAM(app1, rm, amNodeManager); + am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); numAttempt++; } assertEquals("incorrect number of attempts", maxAppAttempts, From fe2fb24a61a8dd0fc3bbee0b3434a503f4c55ad2 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Fri, 7 Feb 2014 00:43:45 +0000 Subject: [PATCH 45/54] HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails intermittently. Contributed by Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565505 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ce457bd51b5..09ec9867b13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -345,6 +345,9 @@ Release 2.4.0 - UNRELEASED HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command returns 0 result. (Tassapol Athiapinya via cnauroth) + HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails + intermittently. (Chen He via kihwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java index ff9ea0728c9..eefb6202540 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java @@ -65,7 +65,7 @@ public class TestBalancerWithNodeGroup { ClientProtocol client; - static final long TIMEOUT = 20000L; //msec + static final long TIMEOUT = 40000L; //msec static final double CAPACITY_ALLOWED_VARIANCE = 0.005; // 0.5% static final double BALANCE_ALLOWED_VARIANCE = 0.11; // 10%+delta static final int DEFAULT_BLOCK_SIZE = 10; From 0bf97bda18fc84f5687edc9744812f565d44116b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 7 Feb 2014 01:12:52 +0000 Subject: [PATCH 46/54] HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565507 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/oncrpc/TestFrameDecoder.java | 32 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b5b8e73445c..7a7106197d9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,6 +325,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10327. Trunk windows build broken after HDFS-5746. (Vinay via cnauroth) + HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. + (Arpit Agarwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java index cdeaa3f2bed..9f951c3c1d9 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; +import java.util.Random; import org.apache.hadoop.oncrpc.RpcUtil.RpcFrameDecoder; import org.apache.hadoop.oncrpc.security.CredentialsNone; @@ -31,17 +32,17 @@ import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelException; import org.jboss.netty.channel.ChannelHandlerContext; import org.junit.Test; import org.mockito.Mockito; public class TestFrameDecoder { - private static int port = 12345; // some random server port private static int resultSize; - static void testRequest(XDR request) { - SimpleTcpClient tcpClient = new SimpleTcpClient("localhost", port, request, + static void testRequest(XDR request, int serverPort) { + SimpleTcpClient tcpClient = new SimpleTcpClient("localhost", serverPort, request, true); tcpClient.run(); } @@ -148,10 +149,25 @@ public class TestFrameDecoder { @Test public void testFrames() { - RpcProgram program = new TestFrameDecoder.TestRpcProgram("TestRpcProgram", - "localhost", port, 100000, 1, 2); - SimpleTcpServer tcpServer = new SimpleTcpServer(port, program, 1); - tcpServer.run(); + Random rand = new Random(); + int serverPort = 30000 + rand.nextInt(10000); + int retries = 10; // A few retries in case initial choice is in use. + + while (true) { + try { + RpcProgram program = new TestFrameDecoder.TestRpcProgram("TestRpcProgram", + "localhost", serverPort, 100000, 1, 2); + SimpleTcpServer tcpServer = new SimpleTcpServer(serverPort, program, 1); + tcpServer.run(); + break; // Successfully bound a port, break out. + } catch (ChannelException ce) { + if (retries-- > 0) { + serverPort += rand.nextInt(20); // Port in use? Try another. + } else { + throw ce; // Out of retries. + } + } + } XDR xdrOut = createGetportMount(); int headerSize = xdrOut.size(); @@ -161,7 +177,7 @@ public class TestFrameDecoder { int requestSize = xdrOut.size() - headerSize; // Send the request to the server - testRequest(xdrOut); + testRequest(xdrOut, serverPort); // Verify the server got the request with right size assertEquals(requestSize, resultSize); From 30294a2196691503637276aa66ddd6fadad3979f Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 7 Feb 2014 02:39:32 +0000 Subject: [PATCH 47/54] YARN-1665. Simplify the configuration of RM HA by having better default values. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565517 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/conf/YarnConfiguration.java | 4 ++-- .../apache/hadoop/yarn/client/TestRMFailover.java | 4 ++-- .../src/main/resources/yarn-default.xml | 15 +++++++++------ .../yarn/server/resourcemanager/TestRMHA.java | 3 ++- .../recovery/TestZKRMStateStore.java | 2 ++ .../yarn/server/TestMiniYARNClusterForHA.java | 2 +- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6afd4bdbfed..11a3f42d001 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -139,6 +139,9 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1665. Simplify the configuration of RM HA by having better default + values. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9e3619e7f0e..b8b891f1e6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -373,11 +373,11 @@ public class YarnConfiguration extends Configuration { public static final String AUTO_FAILOVER_ENABLED = AUTO_FAILOVER_PREFIX + "enabled"; - public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = false; + public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true; public static final String AUTO_FAILOVER_EMBEDDED = AUTO_FAILOVER_PREFIX + "embedded"; - public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = false; + public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true; public static final String AUTO_FAILOVER_ZK_BASE_PATH = AUTO_FAILOVER_PREFIX + "zk-base-path"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index 8900b160dc0..a57d507b5b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -172,8 +172,6 @@ public class TestRMFailover extends ClientBaseWithFixes { @Test public void testAutomaticFailover() throws YarnException, InterruptedException, IOException { - conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); - conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true); conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster"); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000); @@ -193,6 +191,7 @@ public class TestRMFailover extends ClientBaseWithFixes { @Test public void testWebAppProxyInStandAloneMode() throws YarnException, InterruptedException, IOException { + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); WebAppProxyServer webAppProxyServer = new WebAppProxyServer(); try { conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099"); @@ -227,6 +226,7 @@ public class TestRMFailover extends ClientBaseWithFixes { @Test public void testEmbeddedWebAppProxy() throws YarnException, InterruptedException, IOException { + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster.init(conf); cluster.start(); getAdminService(0).transitionToActive(req); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6c8c1a77e14..c50ea7b7087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -405,17 +405,20 @@ - Enable automatic failover. + Enable automatic failover. + By default, it is enabled only when HA is enabled yarn.resourcemanager.ha.automatic-failover.enabled - false + true - Enable embedded automatic failover. The embedded elector - relies on the RM state store to handle fencing, and is primarily intended - to be used in conjunction with ZKRMStateStore. + Enable embedded automatic failover. + By default, it is enabled only when HA is enabled. + The embedded elector relies on the RM state store to handle fencing, + and is primarily intended to be used in conjunction with ZKRMStateStore. + yarn.resourcemanager.ha.automatic-failover.embedded - false + true diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 5b4f5709d75..9bb64642061 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -119,6 +119,7 @@ public class TestRMHA { */ @Test (timeout = 30000) public void testStartAndTransitions() throws IOException { + configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.init(conf); @@ -178,7 +179,6 @@ public class TestRMHA { "automatic failover is enabled"; Configuration conf = new YarnConfiguration(configuration); - conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); rm = new MockRM(conf); rm.init(conf); @@ -236,6 +236,7 @@ public class TestRMHA { String errorMessageForEventHandler = "Expect to get the same number of handlers"; String errorMessageForService = "Expect to get the same number of services"; + configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf) { @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 417f34a6f27..41fdca24aef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -159,6 +159,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { HAServiceProtocol.RequestSource.REQUEST_BY_USER); Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234); + conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); ResourceManager rm1 = new ResourceManager(); rm1.init(conf1); rm1.start(); @@ -170,6 +171,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { rm1.getRMContext().getRMAdminService().getServiceStatus().getState()); Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678); + conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); ResourceManager rm2 = new ResourceManager(); rm2.init(conf2); rm2.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java index 05266858a22..437acdf5171 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java @@ -42,7 +42,7 @@ public class TestMiniYARNClusterForHA { @Before public void setup() throws IOException, InterruptedException { Configuration conf = new YarnConfiguration(); - + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(), 2, 1, 1, 1); cluster.init(conf); From 8b2336fcefa906a5bfe7f6dcf36c18fb34f377f5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 7 Feb 2014 03:18:39 +0000 Subject: [PATCH 48/54] YARN-1660. Simplified the RM HA configuration to accept and be able to simply depend just on configuration properties of the form yarn.resourcemanager.hostname.RMID and use the default ports for all service addresses. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565523 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 5 ++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 51 ++++++++++++------ .../hadoop/yarn/conf/YarnConfiguration.java | 26 ++++++++++ .../apache/hadoop/yarn/conf/TestHAUtil.java | 5 +- .../yarn/server/resourcemanager/TestRMHA.java | 52 +++++++++++++++++++ 5 files changed, 120 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 11a3f42d001..382255da0c3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -142,6 +142,11 @@ Release 2.4.0 - UNRELEASED YARN-1665. Simplify the configuration of RM HA by having better default values. (Xuan Gong via vinodkv) + YARN-1660. Simplified the RM HA configuration to accept and be able to simply + depend just on configuration properties of the form + yarn.resourcemanager.hostname.RMID and use the default ports for all service + addresses. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index ec2c64b5c2b..b5a0b1a2077 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -101,20 +101,7 @@ public class HAUtil { for (String id: ids) { // verify the RM service addresses configurations for every RMIds for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { - String confKey = null; - try { - confKey = addSuffix(prefix, id); - if (conf.getTrimmed(confKey) == null) { - throwBadConfigurationException(getNeedToSetValueMessage(confKey)); - } - } catch (IllegalArgumentException iae) { - String errmsg = iae.getMessage(); - if (confKey == null) { - // Error at addSuffix - errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id); - } - throwBadConfigurationException(errmsg); - } + checkAndSetRMRPCAddress(prefix, id, conf); } setValue.append(id); setValue.append(","); @@ -249,9 +236,13 @@ public class HAUtil { @InterfaceAudience.Private @VisibleForTesting static String getConfKeyForRMInstance(String prefix, Configuration conf) { - return YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix) - ? addSuffix(prefix, getRMHAId(conf)) - : prefix; + if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) { + return prefix; + } else { + String RMId = getRMHAId(conf); + checkAndSetRMRPCAddress(prefix, RMId, conf); + return addSuffix(prefix, RMId); + } } public static String getConfValueForRMInstance(String prefix, @@ -284,4 +275,30 @@ public class HAUtil { } return key + "." + suffix; } + + private static void checkAndSetRMRPCAddress(String prefix, String RMId, + Configuration conf) { + String rpcAddressConfKey = null; + try { + rpcAddressConfKey = addSuffix(prefix, RMId); + if (conf.getTrimmed(rpcAddressConfKey) == null) { + String hostNameConfKey = addSuffix(YarnConfiguration.RM_HOSTNAME, RMId); + String confVal = conf.getTrimmed(hostNameConfKey); + if (confVal == null) { + throwBadConfigurationException(getNeedToSetValueMessage( + hostNameConfKey + " or " + addSuffix(prefix, RMId))); + } else { + conf.set(addSuffix(prefix, RMId), confVal + ":" + + YarnConfiguration.getRMDefaultPortNumber(prefix)); + } + } + } catch (IllegalArgumentException iae) { + String errmsg = iae.getMessage(); + if (rpcAddressConfKey == null) { + // Error at addSuffix + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, RMId); + } + throwBadConfigurationException(errmsg); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b8b891f1e6e..269809dda33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; @@ -108,6 +109,8 @@ public class YarnConfiguration extends Configuration { public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id"; + public static final String RM_HOSTNAME = RM_PREFIX + "hostname"; + /** The address of the applications manager interface in the RM.*/ public static final String RM_ADDRESS = RM_PREFIX + "address"; @@ -1139,4 +1142,27 @@ public class YarnConfiguration extends Configuration { } return super.updateConnectAddr(prefix, addr); } + + @Private + public static int getRMDefaultPortNumber(String addressPrefix) { + if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_PORT; + } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT; + } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT; + } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT; + } else if (addressPrefix + .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT; + } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) { + return YarnConfiguration.DEFAULT_RM_ADMIN_PORT; + } else { + throw new HadoopIllegalArgumentException( + "Invalid RM RPC address Prefix: " + addressPrefix + + ". The valid value should be one of " + + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index 1908b6b75ec..891b434262f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -159,8 +159,9 @@ public class TestHAUtil { String confKey = HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID); assertEquals("YarnRuntimeException by Configuration#set()", - HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(confKey), - e.getMessage()); + HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage( + HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID) + + " or " + confKey), e.getMessage()); } // simulate the case YarnConfiguration.RM_HA_IDS doesn't contain diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 9bb64642061..3c9f92a621d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.junit.Before; import org.junit.Test; @@ -314,6 +315,57 @@ public class TestRMHA { } } + @Test + public void testHAWithRMHostName() { + //test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set + //We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration + configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, + RM1_NODE_ID), "1.1.1.1"); + configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, + RM2_NODE_ID), "0.0.0.0"); + configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, + RM3_NODE_ID), "2.2.2.2"); + try { + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + assertEquals("RPC address not set for " + confKey, + RM1_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM1_NODE_ID))); + assertEquals("RPC address not set for " + confKey, + RM2_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM2_NODE_ID))); + assertEquals("RPC address not set for " + confKey, + RM3_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM3_NODE_ID))); + } + } catch (YarnRuntimeException e) { + fail("Should not throw any exceptions."); + } + + //test if only RM_HOSTBANE_{rm_id} is set + configuration.clear(); + configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + + RM2_NODE_ID); + configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, + RM1_NODE_ID), "1.1.1.1"); + configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, + RM2_NODE_ID), "0.0.0.0"); + try { + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS, + "1.1.1.1:8032", + conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID))); + assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS, + "0.0.0.0:8032", + conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID))); + + } catch (YarnRuntimeException e) { + fail("Should not throw any exceptions."); + } + } + @SuppressWarnings("rawtypes") class MyCountingDispatcher extends AbstractService implements Dispatcher { From c79cc3a3140e390eb40eb4b3ba89fff5d4ecd5ee Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 7 Feb 2014 18:35:44 +0000 Subject: [PATCH 49/54] YARN-1497. Command line additions for moving apps between queues (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565754 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapred/ResourceMgrDelegate.java | 6 ++ hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/client/api/YarnClient.java | 15 ++++ .../yarn/client/api/impl/YarnClientImpl.java | 9 ++ .../yarn/client/cli/ApplicationCLI.java | 36 ++++++++ .../hadoop/yarn/client/cli/YarnCLI.java | 1 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 90 +++++++++++++++---- 7 files changed, 143 insertions(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index fb3e47de353..e65744bd95d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -400,4 +400,10 @@ public class ResourceMgrDelegate extends YarnClient { IOException { return client.getContainers(applicationAttemptId); } + + @Override + public void moveApplicationAcrossQueues(ApplicationId appId, String queue) + throws YarnException, IOException { + client.moveApplicationAcrossQueues(appId, queue); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 382255da0c3..8ac0af93cc3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -17,6 +17,9 @@ Trunk - Unreleased YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy Ryza) + YARN-1497. Command line additions for moving apps between queues (Sandy + Ryza) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index dd27a0209da..c6db3abb5de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -436,4 +436,19 @@ public abstract class YarnClient extends AbstractService { public abstract List getContainers( ApplicationAttemptId applicationAttemptId) throws YarnException, IOException; + + /** + *

    + * Attempts to move the given application to the given queue. + *

    + * + * @param appId + * Application to move. + * @param queue + * Queue to place it in to. + * @throws YarnException + * @throws IOException + */ + public abstract void moveApplicationAcrossQueues(ApplicationId appId, + String queue) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index ac80240d8ab..feb3bb7ffa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; @@ -478,4 +479,12 @@ public class YarnClientImpl extends YarnClient { } throw new YarnException("History service is not enabled."); } + + @Override + public void moveApplicationAcrossQueues(ApplicationId appId, + String queue) throws YarnException, IOException { + MoveApplicationAcrossQueuesRequest request = + MoveApplicationAcrossQueuesRequest.newInstance(appId, queue); + rmClient.moveApplicationAcrossQueues(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index d520866e51c..80e548d26e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -61,6 +61,7 @@ public class ApplicationCLI extends YarnCLI { private static final String APP_TYPE_CMD = "appTypes"; private static final String APP_STATE_CMD = "appStates"; private static final String ALLSTATES_OPTION = "ALL"; + private static final String QUEUE_CMD = "queue"; public static final String APPLICATION = "application"; public static final String APPLICATION_ATTEMPT = "applicationattempt"; public static final String CONTAINER = "container"; @@ -96,6 +97,10 @@ public class ApplicationCLI extends YarnCLI { + "and -appStates to filter applications based on application state"); } opts.addOption(KILL_CMD, true, "Kills the application."); + opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a " + + "different queue."); + opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to" + + " specify which queue to move an application to."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + "filter applications based on " @@ -112,6 +117,8 @@ public class ApplicationCLI extends YarnCLI { appStateOpt.setArgName("States"); opts.addOption(appStateOpt); opts.getOption(KILL_CMD).setArgName("Application ID"); + opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID"); + opts.getOption(QUEUE_CMD).setArgName("Queue Name"); opts.getOption(STATUS_CMD).setArgName("Application ID"); int exitCode = -1; @@ -202,6 +209,13 @@ public class ApplicationCLI extends YarnCLI { return exitCode; } killApplication(cliParser.getOptionValue(KILL_CMD)); + } else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) { + if (!cliParser.hasOption(QUEUE_CMD)) { + printUsage(opts); + return exitCode; + } + moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), + cliParser.getOptionValue(QUEUE_CMD)); } else if (cliParser.hasOption(HELP_CMD)) { printUsage(opts); return 0; @@ -366,6 +380,28 @@ public class ApplicationCLI extends YarnCLI { client.killApplication(appId); } } + + /** + * Kills the application with the application id as appId + * + * @param applicationId + * @throws YarnException + * @throws IOException + */ + private void moveApplicationAcrossQueues(String applicationId, String queue) + throws YarnException, IOException { + ApplicationId appId = ConverterUtils.toApplicationId(applicationId); + ApplicationReport appReport = client.getApplicationReport(appId); + if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED + || appReport.getYarnApplicationState() == YarnApplicationState.KILLED + || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { + sysout.println("Application " + applicationId + " has already finished "); + } else { + sysout.println("Moving application " + applicationId + " to queue " + queue); + client.moveApplicationAcrossQueues(appId, queue); + sysout.println("Successfully completed move."); + } + } /** * Prints the application report for an application id. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java index 921c1355b1d..26349fa81a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java @@ -33,6 +33,7 @@ public abstract class YarnCLI extends Configured implements Tool { public static final String STATUS_CMD = "status"; public static final String LIST_CMD = "list"; public static final String KILL_CMD = "kill"; + public static final String MOVE_TO_QUEUE_CMD = "movetoqueue"; public static final String HELP_CMD = "help"; protected PrintStream sysout; protected PrintStream syserr; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 48ac5484893..12bc6be7316 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -675,6 +675,7 @@ public class TestYarnCLI { int result = spyCli.run(new String[] { "-help" }); Assert.assertTrue(result == 0); verify(spyCli).printUsage(any(Options.class)); + System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl Assert.assertEquals(createApplicationCLIHelpMessage(), sysOutStream.toString()); @@ -748,6 +749,56 @@ public class TestYarnCLI { "' doesn't exist in RM.", ex.getMessage()); } } + + @Test + public void testMoveApplicationAcrossQueues() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, 1), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( + newApplicationReport2); + int result = cli.run(new String[] { "-movetoqueue", applicationId.toString(), + "-queue", "targetqueue"}); + assertEquals(0, result); + verify(client, times(0)).moveApplicationAcrossQueues( + any(ApplicationId.class), any(String.class)); + verify(sysOut).println( + "Application " + applicationId + " has already finished "); + + ApplicationReport newApplicationReport = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, 1), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( + newApplicationReport); + result = cli.run(new String[] { "-movetoqueue", applicationId.toString(), + "-queue", "targetqueue"}); + assertEquals(0, result); + verify(client).moveApplicationAcrossQueues(any(ApplicationId.class), + any(String.class)); + verify(sysOut).println("Moving application application_1234_0005 to queue targetqueue"); + verify(sysOut).println("Successfully completed move."); + + doThrow(new ApplicationNotFoundException("Application with id '" + + applicationId + "' doesn't exist in RM.")).when(client) + .moveApplicationAcrossQueues(applicationId, "targetqueue"); + cli = createAndGetAppCLI(); + try { + result = cli.run(new String[] { "-movetoqueue", applicationId.toString(), + "-queue", "targetqueue"}); + Assert.fail(); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ApplicationNotFoundException); + Assert.assertEquals("Application with id '" + applicationId + + "' doesn't exist in RM.", ex.getMessage()); + } + } @Test public void testListClusterNodes() throws Exception { @@ -1087,23 +1138,28 @@ public class TestYarnCLI { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("usage: application"); - pw.println(" -appStates Works with -list to filter applications based"); - pw.println(" on input comma-separated list of application"); - pw.println(" states. The valid application state can be one"); - pw.println(" of the following:"); - pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,"); - pw.println(" FINISHED,FAILED,KILLED"); - pw.println(" -appTypes Works with -list to filter applications based"); - pw.println(" on input comma-separated list of application"); - pw.println(" types."); - pw.println(" -help Displays help for all commands."); - pw.println(" -kill Kills the application."); - pw.println(" -list List applications from the RM. Supports"); - pw.println(" optional use of -appTypes to filter"); - pw.println(" applications based on application type, and"); - pw.println(" -appStates to filter applications based on"); - pw.println(" application state"); - pw.println(" -status Prints the status of the application."); + pw.println(" -appStates Works with -list to filter applications"); + pw.println(" based on input comma-separated list of"); + pw.println(" application states. The valid application"); + pw.println(" state can be one of the following:"); + pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN"); + pw.println(" NING,FINISHED,FAILED,KILLED"); + pw.println(" -appTypes Works with -list to filter applications"); + pw.println(" based on input comma-separated list of"); + pw.println(" application types."); + pw.println(" -help Displays help for all commands."); + pw.println(" -kill Kills the application."); + pw.println(" -list List applications from the RM. Supports"); + pw.println(" optional use of -appTypes to filter"); + pw.println(" applications based on application type,"); + pw.println(" and -appStates to filter applications"); + pw.println(" based on application state"); + pw.println(" -movetoqueue Moves the application to a different"); + pw.println(" queue."); + pw.println(" -queue Works with the movetoqueue command to"); + pw.println(" specify which queue to move an"); + pw.println(" application to."); + pw.println(" -status Prints the status of the application."); pw.close(); String appsHelpStr = baos.toString("UTF-8"); return appsHelpStr; From 704521441b60f3e1c37206d046dfb851628b6c20 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 7 Feb 2014 20:40:24 +0000 Subject: [PATCH 50/54] YARN-1493,YARN-1490,YARN-1041, YARN-1166,YARN-1566,YARN-1689,YARN-1661 are reverted from branch-2.3. Updating YARN's CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565805 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8ac0af93cc3..daa2f68b4c3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -106,6 +106,16 @@ Release 2.4.0 - UNRELEASED new APIs for retrieving and storing timeline information. (Zhijie Shen via vinodkv) + YARN-1490. Introduced the ability to make ResourceManager optionally not kill + all containers when an ApplicationMaster exits. (Jian He via vinodkv) + + YARN-1041. Added the ApplicationMasterProtocol API for applications to use the + ability in ResourceManager to optionally not kill containers when the + ApplicationMaster exits. (Jian He via vinodkv) + + YARN-1566. Changed Distributed Shell to retain containers across application + attempts. (Jian He via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via @@ -150,6 +160,9 @@ Release 2.4.0 - UNRELEASED yarn.resourcemanager.hostname.RMID and use the default ports for all service addresses. (Xuan Gong via vinodkv) + YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize + app-attempts separately from apps. (Jian He via vinodkv) + OPTIMIZATIONS BUG FIXES @@ -205,6 +218,15 @@ Release 2.4.0 - UNRELEASED YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi via zjshen) + YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be + triggered by accordingly app event and attempt event. + + YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod + Kumar Vavilapalli via zjshen) + + YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -248,18 +270,8 @@ Release 2.3.0 - UNRELEASED YARN-1029. Added embedded leader election in the ResourceManager. (Karthik Kambatla via vinodkv) - YARN-1490. Introduced the ability to make ResourceManager optionally not kill - all containers when an ApplicationMaster exits. (Jian He via vinodkv) - YARN-1033. Expose RM active/standby state to Web UI and REST API (kasha) - YARN-1041. Added the ApplicationMasterProtocol API for applications to use the - ability in ResourceManager to optionally not kill containers when the - ApplicationMaster exits. (Jian He via vinodkv) - - YARN-1566. Changed Distributed Shell to retain containers across application - attempts. (Jian He via vinodkv) - IMPROVEMENTS YARN-305. Fair scheduler logs too many "Node offered to app" messages. @@ -423,9 +435,6 @@ Release 2.3.0 - UNRELEASED YARN-1541. Changed ResourceManager to invalidate ApplicationMaster host/port information once an AM crashes. (Jian He via vinodkv) - YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize - app-attempts separately from apps. (Jian He via vinodkv) - YARN-1482. Modified WebApplicationProxy to make it work across ResourceManager fail-over. (Xuan Gong via vinodkv) @@ -611,9 +620,6 @@ Release 2.3.0 - UNRELEASED YARN-1574. RMDispatcher should be reset on transition to standby. (Xuan Gong via kasha) - YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be - triggered by accordingly app event and attempt event. - YARN-1598. HA-related rmadmin commands don't work on a secure cluster (kasha) YARN-1603. Remove two *.orig files which were unexpectedly committed. @@ -641,12 +647,6 @@ Release 2.3.0 - UNRELEASED YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod Kumar Vavilapalli via zjshen) - YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod - Kumar Vavilapalli via zjshen) - - YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod - Kumar Vavilapalli via zjshen) - Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From d01158a49879222825767639e1325da6729262c3 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 7 Feb 2014 22:55:27 +0000 Subject: [PATCH 51/54] HDFS-5882. TestAuditLogs is flaky (jxiang via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565840 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/namenode/TestAuditLogs.java | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 09ec9867b13..fcc02d27c50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -348,6 +348,8 @@ Release 2.4.0 - UNRELEASED HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails intermittently. (Chen He via kihwal) + HDFS-5882. TestAuditLogs is flaky (jxiang via cmccabe) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java index 0757355e8e2..2591aee7933 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Enumeration; import java.util.List; import java.util.regex.Pattern; @@ -301,11 +302,18 @@ public class TestAuditLogs { // Turn off the logs Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger(); logger.setLevel(Level.OFF); - + + // Close the appenders and force all logs to be flushed + Enumeration appenders = logger.getAllAppenders(); + while (appenders.hasMoreElements()) { + Appender appender = (Appender)appenders.nextElement(); + appender.close(); + } + BufferedReader reader = new BufferedReader(new FileReader(auditLogFile)); String line = null; boolean ret = true; - + try { for (int i = 0; i < ndupe; i++) { line = reader.readLine(); From 60eca33e83869bd8171cdf735d8f0a1de3b4b90d Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 7 Feb 2014 22:58:23 +0000 Subject: [PATCH 52/54] HDFS-5900. Cannot set cache pool limit of unlimited via CacheAdmin. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565841 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/tools/CacheAdmin.java | 19 +++++++++--- .../src/test/resources/testCacheAdminConf.xml | 30 +++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fcc02d27c50..47f931f2215 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -350,6 +350,9 @@ Release 2.4.0 - UNRELEASED HDFS-5882. TestAuditLogs is flaky (jxiang via cmccabe) + HDFS-5900. Cannot set cache pool limit of "unlimited" via CacheAdmin. + (wang) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index 290e60087f0..b674d09d509 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -140,6 +140,18 @@ public class CacheAdmin extends Configured implements Tool { return maxTtl; } + private static Long parseLimitString(String limitString) { + Long limit = null; + if (limitString != null) { + if (limitString.equalsIgnoreCase("unlimited")) { + limit = CachePoolInfo.LIMIT_UNLIMITED; + } else { + limit = Long.parseLong(limitString); + } + } + return limit; + } + private static Expiration parseExpirationString(String ttlString) throws IOException { Expiration ex = null; @@ -650,8 +662,8 @@ public class CacheAdmin extends Configured implements Tool { info.setMode(new FsPermission(mode)); } String limitString = StringUtils.popOptionWithArgument("-limit", args); - if (limitString != null) { - long limit = Long.parseLong(limitString); + Long limit = parseLimitString(limitString); + if (limit != null) { info.setLimit(limit); } String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); @@ -726,8 +738,7 @@ public class CacheAdmin extends Configured implements Tool { Integer mode = (modeString == null) ? null : Integer.parseInt(modeString, 8); String limitString = StringUtils.popOptionWithArgument("-limit", args); - Long limit = (limitString == null) ? - null : Long.parseLong(limitString); + Long limit = parseLimitString(limitString); String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); Long maxTtl = null; try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml index 64de6bf87c4..13f1b9ae14d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml @@ -469,6 +469,8 @@ -removePool pool1 + -removePool pool2 + -removePool pool3 @@ -489,5 +491,33 @@ + + + Testing setting pool unlimited limits + + -addPool pool1 -limit unlimited -owner andrew -group andrew + -addPool pool2 -limit 10 -owner andrew -group andrew + -modifyPool pool2 -limit unlimited + -listPools + + + -removePool pool1 + -removePool pool2 + + + + SubstringComparator + Found 2 results + + + SubstringComparator + pool1 andrew andrew rwxr-xr-x unlimited never + + + SubstringComparator + pool2 andrew andrew rwxr-xr-x unlimited never + + + From d57c6e0fe76b36884cbe07f43604f00ba19743a5 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 8 Feb 2014 01:55:33 +0000 Subject: [PATCH 53/54] YARN-1672. YarnConfiguration is missing a default for yarn.nodemanager.log.retain-seconds (Naren Koneru via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565866 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 1 + .../containermanager/loghandler/NonAggregatingLogHandler.java | 3 ++- .../loghandler/TestNonAggregatingLogHandler.java | 3 ++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index daa2f68b4c3..c3027f0bf84 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -227,6 +227,9 @@ Release 2.4.0 - UNRELEASED YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod Kumar Vavilapalli via zjshen) + YARN-1672. YarnConfiguration is missing a default for + yarn.nodemanager.log.retain-seconds (Naren Koneru via kasha) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 269809dda33..5322ccd5de6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -629,6 +629,7 @@ public class YarnConfiguration extends Configuration { */ public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX + "log.retain-seconds"; + public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; /** * Number of threads used in log cleanup. Only applicable if Log aggregation diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java index a9aacb58c2a..40173e1be21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java @@ -73,7 +73,8 @@ public class NonAggregatingLogHandler extends AbstractService implements protected void serviceInit(Configuration conf) throws Exception { // Default 3 hours. this.deleteDelaySeconds = - conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 3 * 60 * 60); + conf.getLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, + YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS); sched = createScheduledThreadPoolExecutor(conf); super.serviceInit(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java index 298157bcd4a..300ca286bab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java @@ -145,7 +145,8 @@ public class TestNonAggregatingLogHandler { conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false); - conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 10800l); + conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, + YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS); DrainDispatcher dispatcher = createDispatcher(conf); EventHandler appEventHandler = mock(EventHandler.class); From 23b2e43f5d678517e33590d15dec73225b9c5682 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Sat, 8 Feb 2014 02:15:46 +0000 Subject: [PATCH 54/54] YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. Contributed by Billie Rinaldi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565868 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-project/pom.xml | 6 + hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/apptimeline/ATSPutErrors.java | 18 +- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../src/main/resources/yarn-default.xml | 8 +- .../TestApplicationTimelineRecords.java | 4 +- .../pom.xml | 19 + .../ApplicationTimelineReader.java | 16 +- .../ApplicationTimelineWriter.java | 5 +- .../{EntityId.java => EntityIdentifier.java} | 8 +- .../apptimeline/GenericObjectMapper.java | 208 +++++ .../LeveldbApplicationTimelineStore.java | 854 ++++++++++++++++++ .../MemoryApplicationTimelineStore.java | 26 +- .../webapp/ATSWebServices.java | 37 +- .../ApplicationTimelineStoreTestUtils.java | 24 +- .../apptimeline/TestGenericObjectMapper.java | 89 ++ .../TestLeveldbApplicationTimelineStore.java | 95 ++ .../TestMemoryApplicationTimelineStore.java | 11 +- .../webapp/TestATSWebServices.java | 37 + 19 files changed, 1431 insertions(+), 41 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/{EntityId.java => EntityIdentifier.java} (91%) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3a6519c17c2..b5337f12619 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -784,6 +784,12 @@ grizzly-http-servlet 2.1.2 + + + org.fusesource.leveldbjni + leveldbjni-all + 1.8 + diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c3027f0bf84..3039c6f9a07 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -116,6 +116,9 @@ Release 2.4.0 - UNRELEASED YARN-1566. Changed Distributed Shell to retain containers across application attempts. (Jian He via vinodkv) + YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. (Billie + Rinaldi via zjshen) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java index 91458e1419f..d330eb41dff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java @@ -94,9 +94,21 @@ public class ATSPutErrors { @Public @Unstable public static class ATSPutError { + /** + * Error code returned when no start time can be found when putting an + * entity. This occurs when the entity does not already exist in the + * store and it is put with no start time or events specified. + */ + public static final int NO_START_TIME = 1; + /** + * Error code returned if an IOException is encountered when putting an + * entity. + */ + public static final int IO_EXCEPTION = 2; + private String entityId; private String entityType; - private Integer errorCode; + private int errorCode; /** * Get the entity Id @@ -144,7 +156,7 @@ public class ATSPutErrors { * @return an error code */ @XmlElement(name = "errorcode") - public Integer getErrorCode() { + public int getErrorCode() { return errorCode; } @@ -154,7 +166,7 @@ public class ATSPutErrors { * @param errorCode * an error code */ - public void setErrorCode(Integer errorCode) { + public void setErrorCode(int errorCode) { this.errorCode = errorCode; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 5322ccd5de6..8c8ad16e8e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1041,6 +1041,10 @@ public class YarnConfiguration extends Configuration { /** ATS store class */ public static final String ATS_STORE = ATS_PREFIX + "store.class"; + /** ATS leveldb path */ + public static final String ATS_LEVELDB_PATH_PROPERTY = + ATS_PREFIX + "leveldb-apptimeline-store.path"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c50ea7b7087..cc8b12437ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1145,7 +1145,13 @@ Store class name for application timeline store yarn.ats.store.class - org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore + org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore + + + + Store file name for leveldb application timeline store + yarn.ats.leveldb-apptimeline-store.path + ${yarn.log.dir}/ats diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java index f2a6d3ef461..24d1ce91e62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java @@ -117,14 +117,14 @@ public class TestApplicationTimelineRecords { ATSPutError error1 = new ATSPutError(); error1.setEntityId("entity id 1"); error1.setEntityId("entity type 1"); - error1.setErrorCode(1); + error1.setErrorCode(ATSPutError.NO_START_TIME); atsPutErrors.addError(error1); List errors = new ArrayList(); errors.add(error1); ATSPutError error2 = new ATSPutError(); error2.setEntityId("entity id 2"); error2.setEntityId("entity type 2"); - error2.setErrorCode(2); + error2.setErrorCode(ATSPutError.IO_EXCEPTION); errors.add(error2); atsPutErrors.addErrors(errors); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index d314d026e99..59be859c7c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -167,6 +167,25 @@ jersey-test-framework-grizzly2 test + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + + commons-collections + commons-collections + + + + org.fusesource.leveldbjni + leveldbjni-all + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java index 97a217dc98a..e448ba8bcad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; +import java.io.IOException; import java.util.Collection; import java.util.EnumSet; import java.util.Set; @@ -78,13 +79,15 @@ public interface ApplicationTimelineReader { * retrieve (see {@link Field}). If the set of fields * contains {@link Field#LAST_EVENT_ONLY} and not * {@link Field#EVENTS}, the most recent event for - * each entity is retrieved. + * each entity is retrieved. If null, retrieves all + * fields. * @return An {@link ATSEntities} object. + * @throws IOException */ ATSEntities getEntities(String entityType, Long limit, Long windowStart, Long windowEnd, NameValuePair primaryFilter, Collection secondaryFilters, - EnumSet fieldsToRetrieve); + EnumSet fieldsToRetrieve) throws IOException; /** * This method retrieves the entity information for a given entity. @@ -95,11 +98,13 @@ public interface ApplicationTimelineReader { * retrieve (see {@link Field}). If the set of * fields contains {@link Field#LAST_EVENT_ONLY} and * not {@link Field#EVENTS}, the most recent event - * for each entity is retrieved. + * for each entity is retrieved. If null, retrieves + * all fields. * @return An {@link ATSEntity} object. + * @throws IOException */ ATSEntity getEntity(String entity, String entityType, EnumSet - fieldsToRetrieve); + fieldsToRetrieve) throws IOException; /** * This method retrieves the events for a list of entities all of the same @@ -118,8 +123,9 @@ public interface ApplicationTimelineReader { * @param eventTypes Restricts the events returned to the given types. If * null, events of all types will be returned. * @return An {@link ATSEvents} object. + * @throws IOException */ ATSEvents getEntityTimelines(String entityType, SortedSet entityIds, Long limit, Long windowStart, - Long windowEnd, Set eventTypes); + Long windowEnd, Set eventTypes) throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java index b7bd0708e43..2a16833d980 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java @@ -23,6 +23,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import java.io.IOException; + /** * This interface is for storing application timeline information. */ @@ -37,7 +39,8 @@ public interface ApplicationTimelineWriter { * * @param data An {@link ATSEntities} object. * @return An {@link ATSPutErrors} object. + * @throws IOException */ - ATSPutErrors put(ATSEntities data); + ATSPutErrors put(ATSEntities data) throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java similarity index 91% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java index 26431f87569..d22e616fd1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityIdentifier.java @@ -26,12 +26,12 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; */ @Private @Unstable -public class EntityId implements Comparable { +public class EntityIdentifier implements Comparable { private String id; private String type; - public EntityId(String id, String type) { + public EntityIdentifier(String id, String type) { this.id = id; this.type = type; } @@ -53,7 +53,7 @@ public class EntityId implements Comparable { } @Override - public int compareTo(EntityId other) { + public int compareTo(EntityIdentifier other) { int c = type.compareTo(other.type); if (c != 0) return c; return id.compareTo(other.id); @@ -78,7 +78,7 @@ public class EntityId implements Comparable { return false; if (getClass() != obj.getClass()) return false; - EntityId other = (EntityId) obj; + EntityIdentifier other = (EntityIdentifier) obj; if (id == null) { if (other.id != null) return false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java new file mode 100644 index 00000000000..38ceb30c7d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/GenericObjectMapper.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.WritableUtils; +import org.codehaus.jackson.map.ObjectMapper; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * A utility class providing methods for serializing and deserializing + * objects. The {@link #write(Object)}, {@link #read(byte[])} and {@link + * #write(java.io.DataOutputStream, Object)}, {@link + * #read(java.io.DataInputStream)} methods are used by the + * {@link LeveldbApplicationTimelineStore} to store and retrieve arbitrary + * JSON, while the {@link #writeReverseOrderedLong} and {@link + * #readReverseOrderedLong} methods are used to sort entities in descending + * start time order. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GenericObjectMapper { + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final byte LONG = 0x1; + private static final byte INTEGER = 0x2; + private static final byte DOUBLE = 0x3; + private static final byte STRING = 0x4; + private static final byte BOOLEAN = 0x5; + private static final byte LIST = 0x6; + private static final byte MAP = 0x7; + + /** + * Serializes an Object into a byte array. Along with {@link #read(byte[]) }, + * can be used to serialize an Object and deserialize it into an Object of + * the same type without needing to specify the Object's type, + * as long as it is one of the JSON-compatible objects Long, Integer, + * Double, String, Boolean, List, or Map. The current implementation uses + * ObjectMapper to serialize complex objects (List and Map) while using + * Writable to serialize simpler objects, to produce fewer bytes. + * + * @param o An Object + * @return A byte array representation of the Object + * @throws IOException + */ + public static byte[] write(Object o) throws IOException { + if (o == null) + return EMPTY_BYTES; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + write(new DataOutputStream(baos), o); + return baos.toByteArray(); + } + + /** + * Serializes an Object and writes it to a DataOutputStream. Along with + * {@link #read(java.io.DataInputStream)}, can be used to serialize an Object + * and deserialize it into an Object of the same type without needing to + * specify the Object's type, as long as it is one of the JSON-compatible + * objects Long, Integer, Double, String, Boolean, List, or Map. The current + * implementation uses ObjectMapper to serialize complex objects (List and + * Map) while using Writable to serialize simpler objects, to produce fewer + * bytes. + * + * @param dos A DataOutputStream + * @param o An Object + * @throws IOException + */ + public static void write(DataOutputStream dos, Object o) + throws IOException { + if (o == null) + return; + if (o instanceof Long) { + dos.write(LONG); + WritableUtils.writeVLong(dos, (Long) o); + } else if(o instanceof Integer) { + dos.write(INTEGER); + WritableUtils.writeVInt(dos, (Integer) o); + } else if(o instanceof Double) { + dos.write(DOUBLE); + dos.writeDouble((Double) o); + } else if (o instanceof String) { + dos.write(STRING); + WritableUtils.writeString(dos, (String) o); + } else if (o instanceof Boolean) { + dos.write(BOOLEAN); + dos.writeBoolean((Boolean) o); + } else if (o instanceof List) { + dos.write(LIST); + ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(dos, o); + } else if (o instanceof Map) { + dos.write(MAP); + ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(dos, o); + } else { + throw new IOException("Couldn't serialize object"); + } + } + + /** + * Deserializes an Object from a byte array created with + * {@link #write(Object)}. + * + * @param b A byte array + * @return An Object + * @throws IOException + */ + public static Object read(byte[] b) throws IOException { + if (b == null || b.length == 0) + return null; + ByteArrayInputStream bais = new ByteArrayInputStream(b); + return read(new DataInputStream(bais)); + } + + /** + * Reads an Object from a DataInputStream whose data has been written with + * {@link #write(java.io.DataOutputStream, Object)}. + * + * @param dis A DataInputStream + * @return An Object, null if an unrecognized type + * @throws IOException + */ + public static Object read(DataInputStream dis) throws IOException { + byte code = (byte)dis.read(); + ObjectMapper mapper; + switch (code) { + case LONG: + return WritableUtils.readVLong(dis); + case INTEGER: + return WritableUtils.readVInt(dis); + case DOUBLE: + return dis.readDouble(); + case STRING: + return WritableUtils.readString(dis); + case BOOLEAN: + return dis.readBoolean(); + case LIST: + mapper = new ObjectMapper(); + return mapper.readValue(dis, ArrayList.class); + case MAP: + mapper = new ObjectMapper(); + return mapper.readValue(dis, HashMap.class); + default: + return null; + } + } + + /** + * Converts a long to a 8-byte array so that lexicographic ordering of the + * produced byte arrays sort the longs in descending order. + * + * @param l A long + * @return A byte array + */ + public static byte[] writeReverseOrderedLong(long l) { + byte[] b = new byte[8]; + b[0] = (byte)(0x7f ^ ((l >> 56) & 0xff)); + for (int i = 1; i < 7; i++) + b[i] = (byte)(0xff ^ ((l >> 8*(7-i)) & 0xff)); + b[7] = (byte)(0xff ^ (l & 0xff)); + return b; + } + + /** + * Reads 8 bytes from an array starting at the specified offset and + * converts them to a long. The bytes are assumed to have been created + * with {@link #writeReverseOrderedLong}. + * + * @param b A byte array + * @param offset An offset into the byte array + * @return A long + */ + public static long readReverseOrderedLong(byte[] b, int offset) { + long l = b[offset] & 0xff; + for (int i = 1; i < 8; i++) { + l = l << 8; + l = l | (b[offset+i]&0xff); + } + return l ^ 0x7fffffffffffffffl; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java new file mode 100644 index 00000000000..c2e93cab948 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/LeveldbApplicationTimelineStore.java @@ -0,0 +1,854 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.collections.map.LRUMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents.ATSEventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.fusesource.leveldbjni.JniDBFactory; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBIterator; +import org.iq80.leveldb.Options; +import org.iq80.leveldb.WriteBatch; + +import static org.apache.hadoop.yarn.server.applicationhistoryservice + .apptimeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.applicationhistoryservice + .apptimeline.GenericObjectMapper.writeReverseOrderedLong; + +/** + * An implementation of an application timeline store backed by leveldb. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class LeveldbApplicationTimelineStore extends AbstractService + implements ApplicationTimelineStore { + private static final Log LOG = LogFactory + .getLog(LeveldbApplicationTimelineStore.class); + + private static final String FILENAME = "leveldb-apptimeline-store.ldb"; + + private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(); + private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(); + private static final byte[] INDEXED_ENTRY_PREFIX = "i".getBytes(); + + private static final byte[] PRIMARY_FILTER_COLUMN = "f".getBytes(); + private static final byte[] OTHER_INFO_COLUMN = "i".getBytes(); + private static final byte[] RELATED_COLUMN = "r".getBytes(); + private static final byte[] TIME_COLUMN = "t".getBytes(); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final int START_TIME_CACHE_SIZE = 10000; + + @SuppressWarnings("unchecked") + private final Map startTimeCache = + Collections.synchronizedMap(new LRUMap(START_TIME_CACHE_SIZE)); + + private DB db; + + public LeveldbApplicationTimelineStore() { + super(LeveldbApplicationTimelineStore.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + Options options = new Options(); + options.createIfMissing(true); + JniDBFactory factory = new JniDBFactory(); + String path = conf.get(YarnConfiguration.ATS_LEVELDB_PATH_PROPERTY); + File p = new File(path); + if (!p.exists()) + if (!p.mkdirs()) + throw new IOException("Couldn't create directory for leveldb " + + "application timeline store " + path); + LOG.info("Using leveldb path " + path); + db = factory.open(new File(path, FILENAME), options); + super.serviceInit(conf); + } + + @Override + protected void serviceStop() throws Exception { + IOUtils.cleanup(LOG, db); + super.serviceStop(); + } + + private static class KeyBuilder { + private static final int MAX_NUMBER_OF_KEY_ELEMENTS = 10; + private byte[][] b; + private boolean[] useSeparator; + private int index; + private int length; + + public KeyBuilder(int size) { + b = new byte[size][]; + useSeparator = new boolean[size]; + index = 0; + length = 0; + } + + public static KeyBuilder newInstance() { + return new KeyBuilder(MAX_NUMBER_OF_KEY_ELEMENTS); + } + + public KeyBuilder add(String s) { + return add(s.getBytes(), true); + } + + public KeyBuilder add(byte[] t) { + return add(t, false); + } + + public KeyBuilder add(byte[] t, boolean sep) { + b[index] = t; + useSeparator[index] = sep; + length += t.length; + if (sep) + length++; + index++; + return this; + } + + public byte[] getBytes() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(length); + for (int i = 0; i < index; i++) { + baos.write(b[i]); + if (i < index-1 && useSeparator[i]) + baos.write(0x0); + } + return baos.toByteArray(); + } + + public byte[] getBytesForLookup() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(length); + for (int i = 0; i < index; i++) { + baos.write(b[i]); + if (useSeparator[i]) + baos.write(0x0); + } + return baos.toByteArray(); + } + } + + private static class KeyParser { + private final byte[] b; + private int offset; + + public KeyParser(byte[] b, int offset) { + this.b = b; + this.offset = offset; + } + + public String getNextString() throws IOException { + if (offset >= b.length) + throw new IOException( + "tried to read nonexistent string from byte array"); + int i = 0; + while (offset+i < b.length && b[offset+i] != 0x0) + i++; + String s = new String(b, offset, i); + offset = offset + i + 1; + return s; + } + + public long getNextLong() throws IOException { + if (offset+8 >= b.length) + throw new IOException("byte array ran out when trying to read long"); + long l = readReverseOrderedLong(b, offset); + offset += 8; + return l; + } + + public int getOffset() { + return offset; + } + } + + @Override + public ATSEntity getEntity(String entity, String entityType, + EnumSet fields) throws IOException { + DBIterator iterator = null; + try { + byte[] revStartTime = getStartTime(entity, entityType, null, null, null); + if (revStartTime == null) + return null; + byte[] prefix = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX) + .add(entityType).add(revStartTime).add(entity).getBytesForLookup(); + + iterator = db.iterator(); + iterator.seek(prefix); + + return getEntity(entity, entityType, + readReverseOrderedLong(revStartTime, 0), fields, iterator, prefix, + prefix.length); + } finally { + IOUtils.cleanup(LOG, iterator); + } + } + + /** + * Read entity from a db iterator. If no information is found in the + * specified fields for this entity, return null. + */ + private static ATSEntity getEntity(String entity, String entityType, + Long startTime, EnumSet fields, DBIterator iterator, + byte[] prefix, int prefixlen) throws IOException { + if (fields == null) + fields = EnumSet.allOf(Field.class); + + ATSEntity atsEntity = new ATSEntity(); + boolean events = false; + boolean lastEvent = false; + if (fields.contains(Field.EVENTS)) { + events = true; + atsEntity.setEvents(new ArrayList()); + } else if (fields.contains(Field.LAST_EVENT_ONLY)) { + lastEvent = true; + atsEntity.setEvents(new ArrayList()); + } + else { + atsEntity.setEvents(null); + } + boolean relatedEntities = false; + if (fields.contains(Field.RELATED_ENTITIES)) { + relatedEntities = true; + atsEntity.setRelatedEntities(new HashMap>()); + } else { + atsEntity.setRelatedEntities(null); + } + boolean primaryFilters = false; + if (fields.contains(Field.PRIMARY_FILTERS)) { + primaryFilters = true; + atsEntity.setPrimaryFilters(new HashMap()); + } else { + atsEntity.setPrimaryFilters(null); + } + boolean otherInfo = false; + if (fields.contains(Field.OTHER_INFO)) { + otherInfo = true; + atsEntity.setOtherInfo(new HashMap()); + } else { + atsEntity.setOtherInfo(null); + } + + // iterate through the entity's entry, parsing information if it is part + // of a requested field + for (; iterator.hasNext(); iterator.next()) { + byte[] key = iterator.peekNext().getKey(); + if (!prefixMatches(prefix, prefixlen, key)) + break; + if (key[prefixlen] == PRIMARY_FILTER_COLUMN[0]) { + if (primaryFilters) { + atsEntity.addPrimaryFilter(parseRemainingKey(key, + prefixlen + PRIMARY_FILTER_COLUMN.length), + GenericObjectMapper.read(iterator.peekNext().getValue())); + } + } else if (key[prefixlen] == OTHER_INFO_COLUMN[0]) { + if (otherInfo) { + atsEntity.addOtherInfo(parseRemainingKey(key, + prefixlen + OTHER_INFO_COLUMN.length), + GenericObjectMapper.read(iterator.peekNext().getValue())); + } + } else if (key[prefixlen] == RELATED_COLUMN[0]) { + if (relatedEntities) { + addRelatedEntity(atsEntity, key, + prefixlen + RELATED_COLUMN.length); + } + } else if (key[prefixlen] == TIME_COLUMN[0]) { + if (events || (lastEvent && atsEntity.getEvents().size() == 0)) { + ATSEvent event = getEntityEvent(null, key, prefixlen + + TIME_COLUMN.length, iterator.peekNext().getValue()); + if (event != null) { + atsEntity.addEvent(event); + } + } + } else { + LOG.warn(String.format("Found unexpected column for entity %s of " + + "type %s (0x%02x)", entity, entityType, key[prefixlen])); + } + } + + atsEntity.setEntityId(entity); + atsEntity.setEntityType(entityType); + atsEntity.setStartTime(startTime); + + return atsEntity; + } + + @Override + public ATSEvents getEntityTimelines(String entityType, + SortedSet entityIds, Long limit, Long windowStart, + Long windowEnd, Set eventType) throws IOException { + ATSEvents atsEvents = new ATSEvents(); + if (entityIds == null || entityIds.isEmpty()) + return atsEvents; + // create a lexicographically-ordered map from start time to entities + Map> startTimeMap = new TreeMap>(new Comparator() { + @Override + public int compare(byte[] o1, byte[] o2) { + return WritableComparator.compareBytes(o1, 0, o1.length, o2, 0, + o2.length); + } + }); + DBIterator iterator = null; + try { + // look up start times for the specified entities + // skip entities with no start time + for (String entity : entityIds) { + byte[] startTime = getStartTime(entity, entityType, null, null, null); + if (startTime != null) { + List entities = startTimeMap.get(startTime); + if (entities == null) { + entities = new ArrayList(); + startTimeMap.put(startTime, entities); + } + entities.add(new EntityIdentifier(entity, entityType)); + } + } + for (Entry> entry : + startTimeMap.entrySet()) { + // look up the events matching the given parameters (limit, + // start time, end time, event types) for entities whose start times + // were found and add the entities to the return list + byte[] revStartTime = entry.getKey(); + for (EntityIdentifier entity : entry.getValue()) { + ATSEventsOfOneEntity atsEntity = new ATSEventsOfOneEntity(); + atsEntity.setEntityId(entity.getId()); + atsEntity.setEntityType(entityType); + atsEvents.addEvent(atsEntity); + KeyBuilder kb = KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX) + .add(entityType).add(revStartTime).add(entity.getId()) + .add(TIME_COLUMN); + byte[] prefix = kb.getBytesForLookup(); + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + byte[] revts = writeReverseOrderedLong(windowEnd); + kb.add(revts); + byte[] first = kb.getBytesForLookup(); + byte[] last = null; + if (windowStart != null) { + last = KeyBuilder.newInstance().add(prefix) + .add(writeReverseOrderedLong(windowStart)).getBytesForLookup(); + } + if (limit == null) { + limit = DEFAULT_LIMIT; + } + iterator = db.iterator(); + for (iterator.seek(first); atsEntity.getEvents().size() < limit && + iterator.hasNext(); iterator.next()) { + byte[] key = iterator.peekNext().getKey(); + if (!prefixMatches(prefix, prefix.length, key) || (last != null && + WritableComparator.compareBytes(key, 0, key.length, last, 0, + last.length) > 0)) + break; + ATSEvent event = getEntityEvent(eventType, key, prefix.length, + iterator.peekNext().getValue()); + if (event != null) + atsEntity.addEvent(event); + } + } + } + } finally { + IOUtils.cleanup(LOG, iterator); + } + return atsEvents; + } + + /** + * Returns true if the byte array begins with the specified prefix. + */ + private static boolean prefixMatches(byte[] prefix, int prefixlen, + byte[] b) { + if (b.length < prefixlen) + return false; + return WritableComparator.compareBytes(prefix, 0, prefixlen, b, 0, + prefixlen) == 0; + } + + @Override + public ATSEntities getEntities(String entityType, + Long limit, Long windowStart, Long windowEnd, + NameValuePair primaryFilter, Collection secondaryFilters, + EnumSet fields) throws IOException { + if (primaryFilter == null) { + // if no primary filter is specified, prefix the lookup with + // ENTITY_ENTRY_PREFIX + return getEntityByTime(ENTITY_ENTRY_PREFIX, entityType, limit, + windowStart, windowEnd, secondaryFilters, fields); + } else { + // if a primary filter is specified, prefix the lookup with + // INDEXED_ENTRY_PREFIX + primaryFilterName + primaryFilterValue + + // ENTITY_ENTRY_PREFIX + byte[] base = KeyBuilder.newInstance().add(INDEXED_ENTRY_PREFIX) + .add(primaryFilter.getName()) + .add(GenericObjectMapper.write(primaryFilter.getValue()), true) + .add(ENTITY_ENTRY_PREFIX).getBytesForLookup(); + return getEntityByTime(base, entityType, limit, windowStart, windowEnd, + secondaryFilters, fields); + } + } + + /** + * Retrieves a list of entities satisfying given parameters. + * + * @param base A byte array prefix for the lookup + * @param entityType The type of the entity + * @param limit A limit on the number of entities to return + * @param starttime The earliest entity start time to retrieve (exclusive) + * @param endtime The latest entity start time to retrieve (inclusive) + * @param secondaryFilters Filter pairs that the entities should match + * @param fields The set of fields to retrieve + * @return A list of entities + * @throws IOException + */ + private ATSEntities getEntityByTime(byte[] base, + String entityType, Long limit, Long starttime, Long endtime, + Collection secondaryFilters, EnumSet fields) + throws IOException { + DBIterator iterator = null; + try { + KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType); + // only db keys matching the prefix (base + entity type) will be parsed + byte[] prefix = kb.getBytesForLookup(); + if (endtime == null) { + // if end time is null, place no restriction on end time + endtime = Long.MAX_VALUE; + } + // using end time, construct a first key that will be seeked to + byte[] revts = writeReverseOrderedLong(endtime); + kb.add(revts); + byte[] first = kb.getBytesForLookup(); + byte[] last = null; + if (starttime != null) { + // if start time is not null, set a last key that will not be + // iterated past + last = KeyBuilder.newInstance().add(base).add(entityType) + .add(writeReverseOrderedLong(starttime)).getBytesForLookup(); + } + if (limit == null) { + // if limit is not specified, use the default + limit = DEFAULT_LIMIT; + } + + ATSEntities atsEntities = new ATSEntities(); + iterator = db.iterator(); + iterator.seek(first); + // iterate until one of the following conditions is met: limit is + // reached, there are no more keys, the key prefix no longer matches, + // or a start time has been specified and reached/exceeded + while (atsEntities.getEntities().size() < limit && iterator.hasNext()) { + byte[] key = iterator.peekNext().getKey(); + if (!prefixMatches(prefix, prefix.length, key) || (last != null && + WritableComparator.compareBytes(key, 0, key.length, last, 0, + last.length) > 0)) + break; + // read the start time and entity from the current key + KeyParser kp = new KeyParser(key, prefix.length); + Long startTime = kp.getNextLong(); + String entity = kp.getNextString(); + // parse the entity that owns this key, iterating over all keys for + // the entity + ATSEntity atsEntity = getEntity(entity, entityType, startTime, + fields, iterator, key, kp.getOffset()); + if (atsEntity == null) + continue; + // determine if the retrieved entity matches the provided secondary + // filters, and if so add it to the list of entities to return + boolean filterPassed = true; + if (secondaryFilters != null) { + for (NameValuePair filter : secondaryFilters) { + Object v = atsEntity.getOtherInfo().get(filter.getName()); + if (v == null) + v = atsEntity.getPrimaryFilters().get(filter.getName()); + if (v == null || !v.equals(filter.getValue())) { + filterPassed = false; + break; + } + } + } + if (filterPassed) + atsEntities.addEntity(atsEntity); + } + return atsEntities; + } finally { + IOUtils.cleanup(LOG, iterator); + } + } + + /** + * Put a single entity. If there is an error, add a PutError to the given + * response. + */ + private void put(ATSEntity atsEntity, ATSPutErrors response) { + WriteBatch writeBatch = null; + try { + writeBatch = db.createWriteBatch(); + List events = atsEntity.getEvents(); + // look up the start time for the entity + byte[] revStartTime = getStartTime(atsEntity.getEntityId(), + atsEntity.getEntityType(), atsEntity.getStartTime(), events, + writeBatch); + if (revStartTime == null) { + // if no start time is found, add an error and return + ATSPutError error = new ATSPutError(); + error.setEntityId(atsEntity.getEntityId()); + error.setEntityType(atsEntity.getEntityType()); + error.setErrorCode(ATSPutError.NO_START_TIME); + response.addError(error); + return; + } + Long revStartTimeLong = readReverseOrderedLong(revStartTime, 0); + Map primaryFilters = atsEntity.getPrimaryFilters(); + + // write event entries + if (events != null && !events.isEmpty()) { + for (ATSEvent event : events) { + byte[] revts = writeReverseOrderedLong(event.getTimestamp()); + byte[] key = createEntityEventKey(atsEntity.getEntityId(), + atsEntity.getEntityType(), revStartTime, revts, + event.getEventType()); + byte[] value = GenericObjectMapper.write(event.getEventInfo()); + writeBatch.put(key, value); + writePrimaryFilterEntries(writeBatch, primaryFilters, key, value); + } + } + + // write related entity entries + Map> relatedEntities = + atsEntity.getRelatedEntities(); + if (relatedEntities != null && !relatedEntities.isEmpty()) { + for (Entry> relatedEntityList : + relatedEntities.entrySet()) { + String relatedEntityType = relatedEntityList.getKey(); + for (String relatedEntityId : relatedEntityList.getValue()) { + // look up start time of related entity + byte[] relatedEntityStartTime = getStartTime(relatedEntityId, + relatedEntityType, null, null, writeBatch); + if (relatedEntityStartTime == null) { + // if start time is not found, set start time of the related + // entity to the start time of this entity, and write it to the + // db and the cache + relatedEntityStartTime = revStartTime; + writeBatch.put(createStartTimeLookupKey(relatedEntityId, + relatedEntityType), relatedEntityStartTime); + startTimeCache.put(new EntityIdentifier(relatedEntityId, + relatedEntityType), revStartTimeLong); + } + // write reverse entry (related entity -> entity) + byte[] key = createReleatedEntityKey(relatedEntityId, + relatedEntityType, relatedEntityStartTime, + atsEntity.getEntityId(), atsEntity.getEntityType()); + writeBatch.put(key, EMPTY_BYTES); + // TODO: write forward entry (entity -> related entity)? + } + } + } + + // write primary filter entries + if (primaryFilters != null && !primaryFilters.isEmpty()) { + for (Entry primaryFilter : primaryFilters.entrySet()) { + byte[] key = createPrimaryFilterKey(atsEntity.getEntityId(), + atsEntity.getEntityType(), revStartTime, primaryFilter.getKey()); + byte[] value = GenericObjectMapper.write(primaryFilter.getValue()); + writeBatch.put(key, value); + writePrimaryFilterEntries(writeBatch, primaryFilters, key, value); + } + } + + // write other info entries + Map otherInfo = atsEntity.getOtherInfo(); + if (otherInfo != null && !otherInfo.isEmpty()) { + for (Entry i : otherInfo.entrySet()) { + byte[] key = createOtherInfoKey(atsEntity.getEntityId(), + atsEntity.getEntityType(), revStartTime, i.getKey()); + byte[] value = GenericObjectMapper.write(i.getValue()); + writeBatch.put(key, value); + writePrimaryFilterEntries(writeBatch, primaryFilters, key, value); + } + } + db.write(writeBatch); + } catch (IOException e) { + LOG.error("Error putting entity " + atsEntity.getEntityId() + + " of type " + atsEntity.getEntityType(), e); + ATSPutError error = new ATSPutError(); + error.setEntityId(atsEntity.getEntityId()); + error.setEntityType(atsEntity.getEntityType()); + error.setErrorCode(ATSPutError.IO_EXCEPTION); + response.addError(error); + } finally { + IOUtils.cleanup(LOG, writeBatch); + } + } + + /** + * For a given key / value pair that has been written to the db, + * write additional entries to the db for each primary filter. + */ + private static void writePrimaryFilterEntries(WriteBatch writeBatch, + Map primaryFilters, byte[] key, byte[] value) + throws IOException { + if (primaryFilters != null && !primaryFilters.isEmpty()) { + for (Entry p : primaryFilters.entrySet()) { + writeBatch.put(addPrimaryFilterToKey(p.getKey(), p.getValue(), + key), value); + } + } + } + + @Override + public ATSPutErrors put(ATSEntities atsEntities) { + ATSPutErrors response = new ATSPutErrors(); + for (ATSEntity atsEntity : atsEntities.getEntities()) { + put(atsEntity, response); + } + return response; + } + + /** + * Get the unique start time for a given entity as a byte array that sorts + * the timestamps in reverse order (see {@link + * GenericObjectMapper#writeReverseOrderedLong(long)}). + * + * @param entityId The id of the entity + * @param entityType The type of the entity + * @param startTime The start time of the entity, or null + * @param events A list of events for the entity, or null + * @param writeBatch A leveldb write batch, if the method is called by a + * put as opposed to a get + * @return A byte array + * @throws IOException + */ + private byte[] getStartTime(String entityId, String entityType, + Long startTime, List events, WriteBatch writeBatch) + throws IOException { + EntityIdentifier entity = new EntityIdentifier(entityId, entityType); + if (startTime == null) { + // start time is not provided, so try to look it up + if (startTimeCache.containsKey(entity)) { + // found the start time in the cache + startTime = startTimeCache.get(entity); + } else { + // try to look up the start time in the db + byte[] b = createStartTimeLookupKey(entity.getId(), entity.getType()); + byte[] v = db.get(b); + if (v == null) { + // did not find the start time in the db + // if this is a put, try to set it from the provided events + if (events == null || writeBatch == null) { + // no events, or not a put, so return null + return null; + } + Long min = Long.MAX_VALUE; + for (ATSEvent e : events) + if (min > e.getTimestamp()) + min = e.getTimestamp(); + startTime = min; + // selected start time as minimum timestamp of provided events + // write start time to db and cache + writeBatch.put(b, writeReverseOrderedLong(startTime)); + startTimeCache.put(entity, startTime); + } else { + // found the start time in the db + startTime = readReverseOrderedLong(v, 0); + if (writeBatch != null) { + // if this is a put, re-add the start time to the cache + startTimeCache.put(entity, startTime); + } + } + } + } else { + // start time is provided + // TODO: verify start time in db as well as cache? + if (startTimeCache.containsKey(entity)) { + // if the start time is already in the cache, + // and it is different from the provided start time, + // use the one from the cache + if (!startTime.equals(startTimeCache.get(entity))) + startTime = startTimeCache.get(entity); + } else if (writeBatch != null) { + // if this is a put, write the provided start time to the db and the + // cache + byte[] b = createStartTimeLookupKey(entity.getId(), entity.getType()); + writeBatch.put(b, writeReverseOrderedLong(startTime)); + startTimeCache.put(entity, startTime); + } + } + return writeReverseOrderedLong(startTime); + } + + /** + * Creates a key for looking up the start time of a given entity, + * of the form START_TIME_LOOKUP_PREFIX + entitytype + entity. + */ + private static byte[] createStartTimeLookupKey(String entity, + String entitytype) throws IOException { + return KeyBuilder.newInstance().add(START_TIME_LOOKUP_PREFIX) + .add(entitytype).add(entity).getBytes(); + } + + /** + * Creates an index entry for the given key of the form + * INDEXED_ENTRY_PREFIX + primaryfiltername + primaryfiltervalue + key. + */ + private static byte[] addPrimaryFilterToKey(String primaryFilterName, + Object primaryFilterValue, byte[] key) throws IOException { + return KeyBuilder.newInstance().add(INDEXED_ENTRY_PREFIX) + .add(primaryFilterName) + .add(GenericObjectMapper.write(primaryFilterValue), true).add(key) + .getBytes(); + } + + /** + * Creates an event key, serializing ENTITY_ENTRY_PREFIX + entitytype + + * revstarttime + entity + TIME_COLUMN + reveventtimestamp + eventtype. + */ + private static byte[] createEntityEventKey(String entity, String entitytype, + byte[] revStartTime, byte[] reveventtimestamp, String eventtype) + throws IOException { + return KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX) + .add(entitytype).add(revStartTime).add(entity).add(TIME_COLUMN) + .add(reveventtimestamp).add(eventtype).getBytes(); + } + + /** + * Creates an event object from the given key, offset, and value. If the + * event type is not contained in the specified set of event types, + * returns null. + */ + private static ATSEvent getEntityEvent(Set eventTypes, byte[] key, + int offset, byte[] value) throws IOException { + KeyParser kp = new KeyParser(key, offset); + long ts = kp.getNextLong(); + String tstype = kp.getNextString(); + if (eventTypes == null || eventTypes.contains(tstype)) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(ts); + event.setEventType(tstype); + Object o = GenericObjectMapper.read(value); + if (o == null) { + event.setEventInfo(null); + } else if (o instanceof Map) { + @SuppressWarnings("unchecked") + Map m = (Map) o; + event.setEventInfo(m); + } else { + throw new IOException("Couldn't deserialize event info map"); + } + return event; + } + return null; + } + + /** + * Creates a primary filter key, serializing ENTITY_ENTRY_PREFIX + + * entitytype + revstarttime + entity + PRIMARY_FILTER_COLUMN + name. + */ + private static byte[] createPrimaryFilterKey(String entity, + String entitytype, byte[] revStartTime, String name) throws IOException { + return KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX).add(entitytype) + .add(revStartTime).add(entity).add(PRIMARY_FILTER_COLUMN).add(name) + .getBytes(); + } + + /** + * Creates an other info key, serializing ENTITY_ENTRY_PREFIX + entitytype + + * revstarttime + entity + OTHER_INFO_COLUMN + name. + */ + private static byte[] createOtherInfoKey(String entity, String entitytype, + byte[] revStartTime, String name) throws IOException { + return KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX).add(entitytype) + .add(revStartTime).add(entity).add(OTHER_INFO_COLUMN).add(name) + .getBytes(); + } + + /** + * Creates a string representation of the byte array from the given offset + * to the end of the array (for parsing other info keys). + */ + private static String parseRemainingKey(byte[] b, int offset) { + return new String(b, offset, b.length - offset); + } + + /** + * Creates a related entity key, serializing ENTITY_ENTRY_PREFIX + + * entitytype + revstarttime + entity + RELATED_COLUMN + relatedentitytype + + * relatedentity. + */ + private static byte[] createReleatedEntityKey(String entity, + String entitytype, byte[] revStartTime, String relatedEntity, + String relatedEntityType) throws IOException { + return KeyBuilder.newInstance().add(ENTITY_ENTRY_PREFIX).add(entitytype) + .add(revStartTime).add(entity).add(RELATED_COLUMN) + .add(relatedEntityType).add(relatedEntity).getBytes(); + } + + /** + * Parses the related entity from the given key at the given offset and + * adds it to the given entity. + */ + private static void addRelatedEntity(ATSEntity atsEntity, byte[] key, + int offset) throws IOException { + KeyParser kp = new KeyParser(key, offset); + String type = kp.getNextString(); + String id = kp.getNextString(); + atsEntity.addRelatedEntity(type, id); + } + + /** + * Clears the cache to test reloading start times from leveldb (only for + * testing). + */ + @VisibleForTesting + void clearStartTimeCache() { + startTimeCache.clear(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java index 45f0a11d764..1c8e392cfe2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java @@ -53,8 +53,8 @@ import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; public class MemoryApplicationTimelineStore extends AbstractService implements ApplicationTimelineStore { - private Map entities = - new HashMap(); + private Map entities = + new HashMap(); public MemoryApplicationTimelineStore() { super(MemoryApplicationTimelineStore.class.getName()); @@ -125,7 +125,7 @@ public class MemoryApplicationTimelineStore if (fieldsToRetrieve == null) { fieldsToRetrieve = EnumSet.allOf(Field.class); } - ATSEntity entity = entities.get(new EntityId(entityId, entityType)); + ATSEntity entity = entities.get(new EntityIdentifier(entityId, entityType)); if (entity == null) { return null; } else { @@ -152,7 +152,7 @@ public class MemoryApplicationTimelineStore windowEnd = Long.MAX_VALUE; } for (String entityId : entityIds) { - EntityId entityID = new EntityId(entityId, entityType); + EntityIdentifier entityID = new EntityIdentifier(entityId, entityType); ATSEntity entity = entities.get(entityID); if (entity == null) { continue; @@ -184,8 +184,8 @@ public class MemoryApplicationTimelineStore public ATSPutErrors put(ATSEntities data) { ATSPutErrors errors = new ATSPutErrors(); for (ATSEntity entity : data.getEntities()) { - EntityId entityId = - new EntityId(entity.getEntityId(), entity.getEntityType()); + EntityIdentifier entityId = + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); // store entity info in memory ATSEntity existingEntity = entities.get(entityId); if (existingEntity == null) { @@ -210,7 +210,7 @@ public class MemoryApplicationTimelineStore ATSPutError error = new ATSPutError(); error.setEntityId(entityId.getId()); error.setEntityType(entityId.getType()); - error.setErrorCode(1); + error.setErrorCode(ATSPutError.NO_START_TIME); errors.addError(error); entities.remove(entityId); continue; @@ -242,12 +242,20 @@ public class MemoryApplicationTimelineStore continue; } for (String idStr : partRelatedEntities.getValue()) { - EntityId relatedEntityId = - new EntityId(idStr, partRelatedEntities.getKey()); + EntityIdentifier relatedEntityId = + new EntityIdentifier(idStr, partRelatedEntities.getKey()); ATSEntity relatedEntity = entities.get(relatedEntityId); if (relatedEntity != null) { relatedEntity.addRelatedEntity( existingEntity.getEntityType(), existingEntity.getEntityId()); + } else { + relatedEntity = new ATSEntity(); + relatedEntity.setEntityId(relatedEntityId.getId()); + relatedEntity.setEntityType(relatedEntityId.getType()); + relatedEntity.setStartTime(existingEntity.getStartTime()); + relatedEntity.addRelatedEntity(existingEntity.getEntityType(), + existingEntity.getEntityId()); + entities.put(relatedEntityId, relatedEntity); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java index 4ea501d89a8..063b67afd07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -45,6 +46,8 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; @@ -64,6 +67,8 @@ import com.google.inject.Singleton; //TODO: support XML serialization/deserialization public class ATSWebServices { + private static final Log LOG = LogFactory.getLog(ATSWebServices.class); + private ApplicationTimelineStore store; @Inject @@ -143,6 +148,10 @@ public class ATSWebServices { "windowStart, windowEnd or limit is not a numeric value."); } catch (IllegalArgumentException e) { throw new BadRequestException("requested invalid field."); + } catch (IOException e) { + LOG.error("Error getting entities", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entities == null) { return new ATSEntities(); @@ -171,6 +180,10 @@ public class ATSWebServices { } catch (IllegalArgumentException e) { throw new BadRequestException( "requested invalid field."); + } catch (IOException e) { + LOG.error("Error getting entity", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entity == null) { throw new WebApplicationException(Response.Status.NOT_FOUND); @@ -206,6 +219,10 @@ public class ATSWebServices { } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); + } catch (IOException e) { + LOG.error("Error getting entity timelines", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (events == null) { return new ATSEvents(); @@ -228,7 +245,13 @@ public class ATSWebServices { if (entities == null) { return new ATSPutErrors(); } - return store.put(entities); + try { + return store.put(entities); + } catch (IOException e) { + LOG.error("Error putting entities", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); + } } private void init(HttpServletResponse response) { @@ -275,7 +298,17 @@ public class ATSWebServices { String[] strs = str.split(delimiter); List fieldList = new ArrayList(); for (String s : strs) { - fieldList.add(Field.valueOf(s.toUpperCase())); + s = s.trim().toUpperCase(); + if (s.equals("EVENTS")) + fieldList.add(Field.EVENTS); + else if (s.equals("LASTEVENTONLY")) + fieldList.add(Field.LAST_EVENT_ONLY); + else if (s.equals("RELATEDENTITIES")) + fieldList.add(Field.RELATED_ENTITIES); + else if (s.equals("PRIMARYFILTERS")) + fieldList.add(Field.PRIMARY_FILTERS); + else if (s.equals("OTHERINFO")) + fieldList.add(Field.OTHER_INFO); } if (fieldList.size() == 0) return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java index 5825af192b8..9afa5c0234a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java @@ -21,6 +21,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -71,7 +73,7 @@ public class ApplicationTimelineStoreTestUtils { /** * Load test data into the given store */ - protected void loadTestData() { + protected void loadTestData() throws IOException { ATSEntities atsEntities = new ATSEntities(); Map primaryFilters = new HashMap(); primaryFilters.put("user", "username"); @@ -126,7 +128,7 @@ public class ApplicationTimelineStoreTestUtils { response = store.put(atsEntities); assertEquals(0, response.getErrors().size()); atsEntities.setEntities(Collections.singletonList(createEntity(entity1b, - entityType1, 123l, Collections.singletonList(ev2), null, + entityType1, 789l, Collections.singletonList(ev2), null, primaryFilters, otherInfo2))); response = store.put(atsEntities); assertEquals(0, response.getErrors().size()); @@ -138,11 +140,11 @@ public class ApplicationTimelineStoreTestUtils { ATSPutError error = response.getErrors().get(0); assertEquals("badentityid", error.getEntityId()); assertEquals("badentity", error.getEntityType()); - assertEquals((Integer) 1, error.getErrorCode()); + assertEquals(ATSPutError.NO_START_TIME, error.getErrorCode()); } /** - * Load veification data + * Load verification data */ protected void loadVerificationData() throws Exception { userFilter = new NameValuePair("user", @@ -197,7 +199,7 @@ public class ApplicationTimelineStoreTestUtils { events2.add(ev4); } - public void testGetSingleEntity() { + public void testGetSingleEntity() throws IOException { // test getting entity info verifyEntityInfo(null, null, null, null, null, null, store.getEntity("id_1", "type_2", EnumSet.allOf(Field.class))); @@ -222,6 +224,10 @@ public class ApplicationTimelineStoreTestUtils { null, null, null, store.getEntity(entity1, entityType1, EnumSet.of(Field.LAST_EVENT_ONLY))); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, store.getEntity(entity1b, entityType1, + null)); + verifyEntityInfo(entity1, entityType1, null, null, primaryFilters, null, store.getEntity(entity1, entityType1, EnumSet.of(Field.PRIMARY_FILTERS))); @@ -234,7 +240,7 @@ public class ApplicationTimelineStoreTestUtils { EnumSet.of(Field.RELATED_ENTITIES))); } - public void testGetEntities() { + public void testGetEntities() throws IOException { // test getting entities assertEquals("nonzero entities size for nonexistent type", 0, store.getEntities("type_0", null, null, null, null, null, @@ -305,7 +311,7 @@ public class ApplicationTimelineStoreTestUtils { primaryFilters, otherInfo, entities.get(1)); } - public void testGetEntitiesWithPrimaryFilters() { + public void testGetEntitiesWithPrimaryFilters() throws IOException { // test using primary filter assertEquals("nonzero entities size for primary filter", 0, store.getEntities("type_1", null, null, null, @@ -361,7 +367,7 @@ public class ApplicationTimelineStoreTestUtils { primaryFilters, otherInfo, entities.get(1)); } - public void testGetEntitiesWithSecondaryFilters() { + public void testGetEntitiesWithSecondaryFilters() throws IOException { // test using secondary filter List entities = store.getEntities("type_1", null, null, null, null, goodTestingFilters, EnumSet.allOf(Field.class)).getEntities(); @@ -388,7 +394,7 @@ public class ApplicationTimelineStoreTestUtils { assertEquals(0, entities.size()); } - public void testGetEvents() { + public void testGetEvents() throws IOException { // test getting entity timelines SortedSet sortedSet = new TreeSet(); sortedSet.add(entity1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java new file mode 100644 index 00000000000..4bb453a41be --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestGenericObjectMapper.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.WritableComparator; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class TestGenericObjectMapper { + + @Test + public void testEncoding() { + testEncoding(Long.MAX_VALUE); + testEncoding(Long.MIN_VALUE); + testEncoding(0l); + testEncoding(128l); + testEncoding(256l); + testEncoding(512l); + testEncoding(-256l); + } + + private static void testEncoding(long l) { + byte[] b = GenericObjectMapper.writeReverseOrderedLong(l); + assertEquals("error decoding", l, + GenericObjectMapper.readReverseOrderedLong(b, 0)); + byte[] buf = new byte[16]; + System.arraycopy(b, 0, buf, 5, 8); + assertEquals("error decoding at offset", l, + GenericObjectMapper.readReverseOrderedLong(buf, 5)); + if (l > Long.MIN_VALUE) { + byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1); + assertEquals("error preserving ordering", 1, + WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length)); + } + if (l < Long.MAX_VALUE) { + byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1); + assertEquals("error preserving ordering", 1, + WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length)); + } + } + + private static void verify(Object o) throws IOException { + assertEquals(o, GenericObjectMapper.read(GenericObjectMapper.write(o))); + } + + @Test + public void testValueTypes() throws IOException { + verify(42l); + verify(42); + verify(1.23); + verify("abc"); + verify(true); + List list = new ArrayList(); + list.add("123"); + list.add("abc"); + verify(list); + Map map = new HashMap(); + map.put("k1","v1"); + map.put("k2","v2"); + verify(map); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java new file mode 100644 index 00000000000..b868049c4fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestLeveldbApplicationTimelineStore.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class TestLeveldbApplicationTimelineStore + extends ApplicationTimelineStoreTestUtils { + private FileContext fsContext; + private File fsPath; + + @Before + public void setup() throws Exception { + fsContext = FileContext.getLocalFSFileContext(); + Configuration conf = new Configuration(); + fsPath = new File("target", this.getClass().getSimpleName() + + "-tmpDir").getAbsoluteFile(); + fsContext.delete(new Path(fsPath.getAbsolutePath()), true); + conf.set(YarnConfiguration.ATS_LEVELDB_PATH_PROPERTY, + fsPath.getAbsolutePath()); + store = new LeveldbApplicationTimelineStore(); + store.init(conf); + store.start(); + loadTestData(); + loadVerificationData(); + } + + @After + public void tearDown() throws Exception { + store.stop(); + fsContext.delete(new Path(fsPath.getAbsolutePath()), true); + } + + @Test + public void testGetSingleEntity() throws IOException { + super.testGetSingleEntity(); + ((LeveldbApplicationTimelineStore)store).clearStartTimeCache(); + super.testGetSingleEntity(); + } + + @Test + public void testGetEntities() throws IOException { + super.testGetEntities(); + } + + @Test + public void testGetEntitiesWithPrimaryFilters() throws IOException { + super.testGetEntitiesWithPrimaryFilters(); + } + + @Test + public void testGetEntitiesWithSecondaryFilters() throws IOException { + super.testGetEntitiesWithSecondaryFilters(); + } + + @Test + public void testGetEvents() throws IOException { + super.testGetEvents(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java index aa88b74a901..07a3955bf67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java @@ -23,6 +23,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; public class TestMemoryApplicationTimelineStore extends ApplicationTimelineStoreTestUtils { @@ -46,27 +47,27 @@ public class TestMemoryApplicationTimelineStore } @Test - public void testGetSingleEntity() { + public void testGetSingleEntity() throws IOException { super.testGetSingleEntity(); } @Test - public void testGetEntities() { + public void testGetEntities() throws IOException { super.testGetEntities(); } @Test - public void testGetEntitiesWithPrimaryFilters() { + public void testGetEntitiesWithPrimaryFilters() throws IOException { super.testGetEntitiesWithPrimaryFilters(); } @Test - public void testGetEntitiesWithSecondaryFilters() { + public void testGetEntitiesWithSecondaryFilters() throws IOException { super.testGetEntitiesWithSecondaryFilters(); } @Test - public void testGetEvents() { + public void testGetEvents() throws IOException { super.testGetEvents(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java index 1ff73ff35a2..58a826c9ac0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java @@ -156,6 +156,43 @@ public class TestATSWebServices extends JerseyTest { Assert.assertEquals(4, entity.getOtherInfo().size()); } + @Test + public void testGetEntityFields1() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("id_1").queryParam("fields", "events,otherinfo") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntity entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("id_1", entity.getEntityId()); + Assert.assertEquals("type_1", entity.getEntityType()); + Assert.assertEquals(123l, entity.getStartTime().longValue()); + Assert.assertEquals(2, entity.getEvents().size()); + Assert.assertEquals(0, entity.getPrimaryFilters().size()); + Assert.assertEquals(4, entity.getOtherInfo().size()); + } + + @Test + public void testGetEntityFields2() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("id_1").queryParam("fields", "lasteventonly," + + "primaryfilters,relatedentities") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntity entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("id_1", entity.getEntityId()); + Assert.assertEquals("type_1", entity.getEntityType()); + Assert.assertEquals(123l, entity.getStartTime().longValue()); + Assert.assertEquals(1, entity.getEvents().size()); + Assert.assertEquals(2, entity.getPrimaryFilters().size()); + Assert.assertEquals(0, entity.getOtherInfo().size()); + } + @Test public void testGetEvents() throws Exception { WebResource r = resource();