state;
+ private FederationQueue crt;
+
+ FedQueueIterator(FederationQueue root) {
+ this.state = new ArrayDeque<>();
+ state.push(root);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !state.isEmpty();
+ }
+
+ @Override
+ public FederationQueue next() {
+ crt = state.pop();
+ if (crt.getChildren() != null && !crt.getChildren().isEmpty()) {
+ state.addAll(crt.getChildren().values());
+ }
+ return crt;
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java
new file mode 100644
index 00000000000..9d0881801f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java
new file mode 100644
index 00000000000..731becd7d84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * This class provides support methods for all global queue tests.
+ */
+public final class GlobalQueueTestUtil {
+
+ private GlobalQueueTestUtil() {
+ }
+
+ public static List generateFedCluster() throws IOException {
+ int numSubclusters = 20;
+
+ List toMerge = new ArrayList<>();
+ String queueJson = loadFile("globalqueues/tree-queue.json");
+ for (int i = 0; i < numSubclusters; i++) {
+ FederationQueue temp = parseQueue(queueJson);
+ toMerge.add(temp);
+ }
+ return toMerge;
+ }
+
+ public static String loadFile(String filename) throws IOException {
+ ClassLoader cL = Thread.currentThread().getContextClassLoader();
+ if (cL == null) {
+ cL = Configuration.class.getClassLoader();
+ }
+ URL submitURI = cL.getResource(filename);
+
+ return FileUtils.readFileToString(new File(submitURI.getFile()),
+ Charset.defaultCharset());
+ }
+
+ public static String toJSONString(Resource resInf) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("{");
+ builder.append("\"memory\"");
+ builder.append(" : ");
+ builder.append(resInf.getMemorySize());
+ builder.append(", ");
+ builder.append("\"vCores\"");
+ builder.append(" : ");
+ builder.append(resInf.getVirtualCores());
+ builder.append("}");
+ return builder.toString();
+ }
+
+ public static FederationQueue parseQueue(String queueJson)
+ throws IOException {
+ JsonFactory jsonF = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper();
+ Map jsonMap = mapper.readValue(jsonF.createParser(queueJson), Map.class);
+ FederationQueue fq = parseFedQueue(jsonMap);
+ fq.propagateCapacities();
+ return fq;
+ }
+
+ private static FederationQueue parseFedQueue(Map jsonMap) {
+ FederationQueue fq = new FederationQueue(new Configuration());
+ fq.setQueueName(jsonMap.get("queueName").toString());
+ fq.setSubClusterId(SubClusterId.newInstance(
+ ((Map)(jsonMap.get("scope"))).get("id").toString()));
+ if (jsonMap.get("guarCap") != null) {
+ fq.setGuarCap(parseResource((Map)jsonMap.get("guarCap")));
+ }
+ if (jsonMap.get("maxCap") != null) {
+ fq.setMaxCap(parseResource((Map)jsonMap.get("maxCap")));
+ }
+ if (jsonMap.get("usedCap") != null) {
+ fq.setUsedCap(parseResource((Map)jsonMap.get("usedCap")));
+ }
+ if (jsonMap.get("totCap") != null) {
+ fq.setTotCap(parseResource((Map)jsonMap.get("totCap")));
+ }
+ if (jsonMap.get("demandCap") != null) {
+ fq.setDemandCap(parseResource((Map)jsonMap.get("demandCap")));
+ }
+ if (jsonMap.get("children") != null) {
+ List children = (List) jsonMap.get("children");
+ Map childrenFedQueue = new HashMap<>();
+ for (Object o : children) {
+ FederationQueue child = parseFedQueue((Map)(o));
+ childrenFedQueue.put(child.getQueueName(), child);
+ }
+ fq.setChildren(childrenFedQueue);
+ }
+ return fq;
+ }
+
+ private static Resource parseResource(Map resMap) {
+ Resource res = Resource.newInstance(0, 0);
+ if (resMap.get("memory") != null) {
+ res.setMemorySize(Integer.parseInt(resMap.get("memory").toString()));
+ }
+ if (resMap.get("vCores") != null) {
+ res.setVirtualCores(Integer.parseInt(resMap.get("vCores").toString()));
+ }
+ return res;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java
new file mode 100644
index 00000000000..d20c631b224
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import static org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues.GlobalQueueTestUtil.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides simple tests for the {@code FederationQueue} data class.
+ */
+public class TestFederationQueue {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationQueue.class);
+
+ @Test
+ public void testParseQueue() throws Exception {
+ String queueJson = loadFile("globalqueues/tree-queue.json");
+ FederationQueue fq = parseQueue(queueJson);
+ fq.validate();
+ Assert.assertEquals("root", fq.getQueueName());
+ Assert.assertEquals(2, fq.getChildren().size());
+ Assert.assertEquals(100000, fq.getGuarCap().getMemorySize());
+
+ FederationQueue queueA = fq.getChildByName("A");
+ Assert.assertEquals(2, queueA.getChildren().size());
+ Assert.assertEquals(750, queueA.getUsedCap().getVirtualCores());
+ }
+
+
+ @Test
+ public void testMergeFedQueue() throws Exception {
+ List toMerge = generateFedCluster();
+
+ FederationQueue merged =
+ FederationQueue.mergeQueues(toMerge,
+ SubClusterId.newInstance("merged"));
+
+ merged.propagateCapacities();
+ merged.validate();
+ LOG.info(merged.toQuickString());
+ }
+
+ @Test
+ public void testPropagateFedQueue() throws Exception {
+
+ String queueJson = loadFile("globalqueues/tree-queue-adaptable.json");
+
+ int numSubclusters = 10;
+
+ Resource guar = Resource.newInstance(5 * 1024, 10);
+ Resource max = Resource.newInstance(8 * 1024, 10);
+ Resource used = Resource.newInstance(4 * 1024, 10);
+ Resource dem = Resource.newInstance(1 * 1024, 10);
+
+ List toMerge = new ArrayList<>();
+ for (int i = 0; i < numSubclusters; i++) {
+ queueJson = String.format(queueJson, "A1", toJSONString(guar),
+ toJSONString(max), toJSONString(used), toJSONString(dem));
+ FederationQueue temp = parseQueue(queueJson);
+ temp.propagateCapacities();
+ temp.validate();
+ toMerge.add(temp);
+ }
+
+ FederationQueue merged =
+ FederationQueue.mergeQueues(toMerge,
+ SubClusterId.newInstance("merged"));
+
+ merged.propagateCapacities();
+ merged.validate();
+
+ LOG.info(merged.toQuickString());
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json
new file mode 100644
index 00000000000..14f10f95f8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json
@@ -0,0 +1,9 @@
+{
+ "queueName" : "%s",
+ "totCap" : %s,
+ "guarCap" : %s,
+ "maxCap" : %s,
+ "usedCap" : %s,
+ "demandCap" : %s,
+ "scope" : {"id" : "sc1"}
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json
new file mode 100644
index 00000000000..b1cb43b2d2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json
@@ -0,0 +1,96 @@
+{
+ "queueName": "root",
+ "scope" : {"id" : "sc1"},
+ "totCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "guarCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ },
+ "children": [
+ {
+ "queueName": "A",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "children": [
+ {
+ "queueName": "%s",
+ "guarCap": %s,
+ "maxCap": %s,
+ "usedCap": %s,
+ "demandCap": %s,
+ "scope" : {"id" : "sc1"}
+ },
+ {
+ "queueName": "A2",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+ },
+ {
+ "queueName": "B",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ }
+ }
+ ]
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json
new file mode 100644
index 00000000000..e3f25328485
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json
@@ -0,0 +1,128 @@
+{
+ "queueName": "root",
+ "scope" : {"id" : "sc1"},
+ "totCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "guarCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "children": [
+ {
+ "queueName": "A",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "75000",
+ "vCores": "750"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "10000",
+ "vCores": "100"
+ },
+ "children": [
+ {
+ "queueName": "A1",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "10000",
+ "vCores": "100"
+ }
+ },
+ {
+ "queueName": "A2",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+ },
+ {
+ "queueName": "B",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "10000",
+ "vCores": "100"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+}