From c17810a84c98c808403fe4caf4532efa1e781221 Mon Sep 17 00:00:00 2001 From: Jian He Date: Sun, 7 Sep 2014 18:25:44 -0700 Subject: [PATCH] YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. Contributed by Tsuyoshi OZAWA (cherry picked from commit 0974f434c47ffbf4b77a8478937fd99106c8ddbd) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/api/records/ContainerId.java | 39 ++++++++++++++++++- .../hadoop/yarn/util/ConverterUtils.java | 15 +------ .../hadoop/yarn/api/TestContainerId.java | 4 ++ .../hadoop/yarn/util/TestConverterUtils.java | 9 +++++ 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8b362a5b5b4..912467893e9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -167,6 +167,9 @@ Release 2.6.0 - UNRELEASED YARN-2507. Documented CrossOriginFilter configurations for the timeline server. (Jonathan Eagles via zjshen) + YARN-2515. Updated ConverterUtils#toContainerId to parse epoch. + (Tsuyoshi OZAWA via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index fc7f40488ec..321052b062b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; +import com.google.common.base.Splitter; +import java.text.NumberFormat; +import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.util.Records; @Public @Stable public abstract class ContainerId implements Comparable{ + private static final Splitter _SPLITTER = Splitter.on('_').trimResults(); + private static final String CONTAINER_PREFIX = "container"; @Private @Unstable @@ -163,5 +167,38 @@ public abstract class ContainerId implements Comparable{ return sb.toString(); } + @Public + @Unstable + public static ContainerId fromString(String containerIdStr) { + Iterator it = _SPLITTER.split(containerIdStr).iterator(); + if (!it.next().equals(CONTAINER_PREFIX)) { + throw new IllegalArgumentException("Invalid ContainerId prefix: " + + containerIdStr); + } + try { + ApplicationAttemptId appAttemptID = toApplicationAttemptId(it); + int id = Integer.parseInt(it.next()); + int epoch = 0; + if (it.hasNext()) { + epoch = Integer.parseInt(it.next()); + } + int cid = (epoch << 22) | id; + ContainerId containerId = ContainerId.newInstance(appAttemptID, cid); + return containerId; + } catch (NumberFormatException n) { + throw new IllegalArgumentException("Invalid ContainerId: " + + containerIdStr, n); + } + } + + private static ApplicationAttemptId toApplicationAttemptId( + Iterator it) throws NumberFormatException { + ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), + Integer.parseInt(it.next())); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); + return appAttemptId; + } + protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index f731af95fc5..27f7bc107e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -168,20 +168,7 @@ public class ConverterUtils { } public static ContainerId toContainerId(String containerIdStr) { - Iterator it = _split(containerIdStr).iterator(); - if (!it.next().equals(CONTAINER_PREFIX)) { - throw new IllegalArgumentException("Invalid ContainerId prefix: " - + containerIdStr); - } - try { - ApplicationAttemptId appAttemptID = toApplicationAttemptId(it); - ContainerId containerId = - ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next())); - return containerId; - } catch (NumberFormatException n) { - throw new IllegalArgumentException("Invalid ContainerId: " - + containerIdStr, n); - } + return ContainerId.fromString(containerIdStr); } public static ApplicationAttemptId toApplicationAttemptId( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java index b23d0ed3fbc..8baf244dda2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java @@ -54,10 +54,14 @@ public class TestContainerId { long ts = System.currentTimeMillis(); ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811); Assert.assertEquals("container_10_0001_01_000001", c1.toString()); + Assert.assertEquals(c1, + ContainerId.fromString("container_10_0001_01_000001")); Assert.assertEquals(479987, 0x003fffff & c6.getId()); Assert.assertEquals(6, c6.getId() >> 22); Assert.assertEquals("container_" + ts + "_36473_4365472_479987_06", c6.toString()); + Assert.assertEquals(c6, + ContainerId.fromString("container_" + ts + "_36473_4365472_479987_06")); } public static ContainerId newContainerId(int appId, int appAttemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java index 21af4555e37..3f4147c4189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java @@ -55,6 +55,15 @@ public class TestConverterUtils { assertEquals(gen, id); } + @Test + public void testContainerIdWithEpoch() throws URISyntaxException { + ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811); + String cid = ConverterUtils.toString(id); + assertEquals("container_0_0000_00_479987_06", cid); + ContainerId gen = ConverterUtils.toContainerId(cid); + assertEquals(gen.toString(), id.toString()); + } + @Test public void testContainerIdNull() throws URISyntaxException { assertNull(ConverterUtils.toString((ContainerId)null));