YARN-6768. Improve performance of yarn api record toString and fromString. Contributed by Jonathan Eagles

(cherry picked from commit 24853bf32a)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+/**
+ * Fast thread-safe version of NumberFormat
+ */
+public class FastNumberFormat {
+
+  public static StringBuilder format(StringBuilder sb, long value, int minimumDigits) {
+    if (value < 0) {
+      sb.append('-');
+      value = -value;
+    }
+
+    long tmp = value;
+    do {
+      tmp /= 10;
+    } while (--minimumDigits > 0 && tmp > 0);
+
+    for (int i = minimumDigits; i > 0; --i) {
+      sb.append('0');
+    }
+
+    sb.append(value);
+    return sb;
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.NumberFormat;
+
+/**
+ * Test for FastNumberFormat
+ */
+public class TestFastNumberFormat {
+  private final int MIN_DIGITS = 6;
+
+  @Test(timeout = 1000)
+  public void testLongWithPadding() throws Exception {
+    NumberFormat numberFormat = NumberFormat.getInstance();
+    numberFormat.setGroupingUsed(false);
+    numberFormat.setMinimumIntegerDigits(6);
+    long[] testLongs = {1, 23, 456, 7890, 12345, 678901, 2345689, 0, -0, -1,
+      -23, -456, -7890, -12345, -678901, -2345689};
+    for (long l: testLongs) {
+      StringBuilder sb = new StringBuilder();
+      FastNumberFormat.format(sb, l, MIN_DIGITS);
+      String fastNumberStr = sb.toString();
+      Assert.assertEquals("Number formats should be equal",
+          numberFormat.format(l), fastNumberStr);
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java

@@ -18,18 +18,13 @@
 
 package org.apache.hadoop.yarn.api.records;
 
-import java.text.NumberFormat;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.FastNumberFormat;
 import org.apache.hadoop.yarn.util.Records;
 
-import com.google.common.base.Splitter;
-
 /**
  * <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
  * of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
@@ -42,12 +37,14 @@ import com.google.common.base.Splitter;
 @Stable
 public abstract class ApplicationAttemptId implements
     Comparable<ApplicationAttemptId> {
-  private static Splitter _spliter = Splitter.on('_').trimResults();
 
   @Private
   @Unstable
   public static final String appAttemptIdStrPrefix = "appattempt";
 
+  private static final String APP_ATTEMPT_ID_PREFIX = appAttemptIdStrPrefix
+    + '_';
+
   @Public
   @Unstable
   public static ApplicationAttemptId newInstance(ApplicationId appId,
@@ -84,16 +81,8 @@ public abstract class ApplicationAttemptId implements
   @Unstable
   protected abstract void setAttemptId(int attemptId);
 
-  static final ThreadLocal<NumberFormat> attemptIdFormat =
+  private static final int ATTEMPT_ID_MIN_DIGITS = 6;
-      new ThreadLocal<NumberFormat>() {
+  private static final int APP_ID_MIN_DIGITS = 4;
-        @Override
-        public NumberFormat initialValue() {
-          NumberFormat fmt = NumberFormat.getInstance();
-          fmt.setGroupingUsed(false);
-          fmt.setMinimumIntegerDigits(6);
-          return fmt;
-        }
-      };
 
   @Override
   public int hashCode() {
@@ -135,12 +124,14 @@ public abstract class ApplicationAttemptId implements
 
   @Override
   public String toString() {
-    StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix);
+    StringBuilder sb = new StringBuilder(64);
-    sb.append("_");
+    sb.append(APP_ATTEMPT_ID_PREFIX);
-    sb.append(this.getApplicationId().getClusterTimestamp()).append("_");
+    ApplicationId appId = getApplicationId();
-    sb.append(ApplicationId.appIdFormat.get().format(
+    sb.append(appId.getClusterTimestamp());
-        this.getApplicationId().getId()));
+    sb.append('_');
-    sb.append("_").append(attemptIdFormat.get().format(getAttemptId()));
+    FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS);
+    sb.append('_');
+    FastNumberFormat.format(sb, getAttemptId(), ATTEMPT_ID_MIN_DIGITS);
     return sb.toString();
   }
 
@@ -148,29 +139,33 @@ public abstract class ApplicationAttemptId implements
   
   @Public
   @Stable
-  public static ApplicationAttemptId fromString(String applicationAttemptIdStr) {
+  public static ApplicationAttemptId fromString(String appAttemptIdStr) {
-    Iterator<String> it = _spliter.split(applicationAttemptIdStr).iterator();
+    if (!appAttemptIdStr.startsWith(APP_ATTEMPT_ID_PREFIX)) {
-    if (!it.next().equals(appAttemptIdStrPrefix)) {
       throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
-          + applicationAttemptIdStr);
+          + appAttemptIdStr);
     }
     try {
-      return toApplicationAttemptId(it);
+      int pos1 = APP_ATTEMPT_ID_PREFIX.length() - 1;
+      int pos2 = appAttemptIdStr.indexOf('_', pos1 + 1);
+      if (pos2 < 0) {
+        throw new IllegalArgumentException("Invalid AppAttemptId: "
+            + appAttemptIdStr);
+      }
+      long rmId = Long.parseLong(appAttemptIdStr.substring(pos1 + 1, pos2));
+      int pos3 = appAttemptIdStr.indexOf('_', pos2 + 1);
+      if (pos3 < 0) {
+        throw new IllegalArgumentException("Invalid AppAttemptId: "
+            + appAttemptIdStr);
+      }
+      int appId = Integer.parseInt(appAttemptIdStr.substring(pos2 + 1, pos3));
+      ApplicationId applicationId = ApplicationId.newInstance(rmId, appId);
+      int attemptId = Integer.parseInt(appAttemptIdStr.substring(pos3 + 1));
+      ApplicationAttemptId applicationAttemptId =
+        ApplicationAttemptId.newInstance(applicationId, attemptId);
+      return applicationAttemptId;
     } catch (NumberFormatException n) {
       throw new IllegalArgumentException("Invalid AppAttemptId: "
-          + applicationAttemptIdStr, n);
+          + appAttemptIdStr, n);
-    } catch (NoSuchElementException e) {
-      throw new IllegalArgumentException("Invalid AppAttemptId: "
-          + applicationAttemptIdStr, e);
     }
   }
-  
-  private static ApplicationAttemptId toApplicationAttemptId(
-      Iterator<String> it) throws NumberFormatException {
-    ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
-        Integer.parseInt(it.next()));
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
-    return appAttemptId;
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java

@@ -18,18 +18,13 @@
 
 package org.apache.hadoop.yarn.api.records;
 
-import java.text.NumberFormat;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.FastNumberFormat;
 import org.apache.hadoop.yarn.util.Records;
 
-import com.google.common.base.Splitter;
-
 /**
  * <p><code>ApplicationId</code> represents the <em>globally unique</em> 
  * identifier for an application.</p>
@@ -42,12 +37,13 @@ import com.google.common.base.Splitter;
 @Public
 @Stable
 public abstract class ApplicationId implements Comparable<ApplicationId> {
-  private static Splitter _spliter = Splitter.on('_').trimResults();
 
   @Private
   @Unstable
   public static final String appIdStrPrefix = "application";
 
+  private static final String APPLICATION_ID_PREFIX = appIdStrPrefix + '_';
+
   @Public
   @Unstable
   public static ApplicationId newInstance(long clusterTimestamp, int id) {
@@ -87,16 +83,7 @@ public abstract class ApplicationId implements Comparable<ApplicationId> {
 
   protected abstract void build();
   
-  static final ThreadLocal<NumberFormat> appIdFormat =
+  private static final int APP_ID_MIN_DIGITS = 4;
-    new ThreadLocal<NumberFormat>() {
-      @Override
-      public NumberFormat initialValue() {
-        NumberFormat fmt = NumberFormat.getInstance();
-        fmt.setGroupingUsed(false);
-        fmt.setMinimumIntegerDigits(4);
-        return fmt;
-      }
-    };
 
   @Override
   public int compareTo(ApplicationId other) {
@@ -110,37 +97,38 @@ public abstract class ApplicationId implements Comparable<ApplicationId> {
 
   @Override
   public String toString() {
-    return appIdStrPrefix + "_" + this.getClusterTimestamp() + "_" + appIdFormat
+    StringBuilder sb = new StringBuilder(64);
-        .get().format(getId());
+    sb.append(APPLICATION_ID_PREFIX);
-  }
+    sb.append(getClusterTimestamp());
-  
+    sb.append('_');
-  private static ApplicationId toApplicationId(
+    FastNumberFormat.format(sb, getId(), APP_ID_MIN_DIGITS);
-      Iterator<String> it) throws NumberFormatException {
+    return sb.toString();
-    ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
-        Integer.parseInt(it.next()));
-    return appId;
   }
   
   @Public
   @Stable
   public static ApplicationId fromString(String appIdStr) {
-    Iterator<String> it = _spliter.split((appIdStr)).iterator();
+    if (!appIdStr.startsWith(APPLICATION_ID_PREFIX)) {
-    if (!it.next().equals(appIdStrPrefix)) {
       throw new IllegalArgumentException("Invalid ApplicationId prefix: "
           + appIdStr + ". The valid ApplicationId should start with prefix "
           + appIdStrPrefix);
     }
     try {
-      return toApplicationId(it);
+      int pos1 = APPLICATION_ID_PREFIX.length() - 1;
+      int pos2 = appIdStr.indexOf('_', pos1 + 1);
+      if (pos2 < 0) {
+        throw new IllegalArgumentException("Invalid ApplicationId: "
+            + appIdStr);
+      }
+      long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2));
+      int appId = Integer.parseInt(appIdStr.substring(pos2 + 1));
+      ApplicationId applicationId = ApplicationId.newInstance(rmId, appId);
+      return applicationId;
     } catch (NumberFormatException n) {
       throw new IllegalArgumentException("Invalid ApplicationId: "
           + appIdStr, n);
-    } catch (NoSuchElementException e) {
-      throw new IllegalArgumentException("Invalid ApplicationId: "
-          + appIdStr, e);
     }
   }
-
   @Override
   public int hashCode() {
     // Generated by eclipse.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java

@@ -18,16 +18,11 @@
 
 package org.apache.hadoop.yarn.api.records;
 
-import com.google.common.base.Splitter;
-
-import java.text.NumberFormat;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.FastNumberFormat;
 import org.apache.hadoop.yarn.util.Records;
 
 /**
@@ -38,8 +33,7 @@ import org.apache.hadoop.yarn.util.Records;
 @Stable
 public abstract class ContainerId implements Comparable<ContainerId>{
   public static final long CONTAINER_ID_BITMASK = 0xffffffffffL;
-  private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
+  private static final String CONTAINER_PREFIX = "container_";
-  private static final String CONTAINER_PREFIX = "container";
   private static final String EPOCH_PREFIX = "e";
 
   @Public
@@ -115,29 +109,13 @@ public abstract class ContainerId implements Comparable<ContainerId>{
   protected abstract void setContainerId(long id);
  
   
-  // TODO: fail the app submission if attempts are more than 10 or something
+  private static final int APP_ID_MIN_DIGITS = 4;
-  private static final ThreadLocal<NumberFormat> appAttemptIdAndEpochFormat =
+
-      new ThreadLocal<NumberFormat>() {
+  private static final int ATTEMPT_ID_MIN_DIGITS = 2;
-        @Override
+
-        public NumberFormat initialValue() {
+  private static final int EPOCH_MIN_DIGITS = 2;
-          NumberFormat fmt = NumberFormat.getInstance();
+
-          fmt.setGroupingUsed(false);
+  private static final int CONTAINER_ID_MIN_DIGITS = 6;
-          fmt.setMinimumIntegerDigits(2);
-          return fmt;
-        }
-      };
-  // TODO: Why thread local?
-  // ^ NumberFormat instances are not threadsafe
-  private static final ThreadLocal<NumberFormat> containerIdFormat =
-      new ThreadLocal<NumberFormat>() {
-        @Override
-        public NumberFormat initialValue() {
-          NumberFormat fmt = NumberFormat.getInstance();
-          fmt.setGroupingUsed(false);
-          fmt.setMinimumIntegerDigits(6);
-          return fmt;
-        }
-      };
 
   @Override
   public int hashCode() {
@@ -185,72 +163,85 @@ public abstract class ContainerId implements Comparable<ContainerId>{
    */
   @Override
   public String toString() {
-    StringBuilder sb = new StringBuilder();
+    StringBuilder sb = new StringBuilder(64);
-    sb.append(CONTAINER_PREFIX + "_");
+    sb.append(CONTAINER_PREFIX);
     long epoch = getContainerId() >> 40;
     if (epoch > 0) {
-      sb.append(EPOCH_PREFIX)
+      sb.append(EPOCH_PREFIX);
-          .append(appAttemptIdAndEpochFormat.get().format(epoch)).append("_");;
+      FastNumberFormat.format(sb, epoch, EPOCH_MIN_DIGITS);
+      sb.append('_');
     }
     ApplicationId appId = getApplicationAttemptId().getApplicationId();
-    sb.append(appId.getClusterTimestamp()).append("_");
+    sb.append(appId.getClusterTimestamp());
-    sb.append(ApplicationId.appIdFormat.get().format(appId.getId()))
+    sb.append('_');
-        .append("_");
+    FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS);
-    sb.append(
+    sb.append('_');
-        appAttemptIdAndEpochFormat.get().format(
+    FastNumberFormat.format(sb, getApplicationAttemptId().getAttemptId(),
-            getApplicationAttemptId().getAttemptId())).append("_");
+        ATTEMPT_ID_MIN_DIGITS);
-    sb.append(containerIdFormat.get()
+    sb.append('_');
-        .format(CONTAINER_ID_BITMASK & getContainerId()));
+    FastNumberFormat.format(sb, CONTAINER_ID_BITMASK & getContainerId(),
+        CONTAINER_ID_MIN_DIGITS);
     return sb.toString();
   }
 
   @Public
   @Stable
   public static ContainerId fromString(String containerIdStr) {
-    Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
+    if (!containerIdStr.startsWith(CONTAINER_PREFIX)) {
-    if (!it.next().equals(CONTAINER_PREFIX)) {
       throw new IllegalArgumentException("Invalid ContainerId prefix: "
           + containerIdStr);
     }
     try {
-      String epochOrClusterTimestampStr = it.next();
+      int pos1 = CONTAINER_PREFIX.length() - 1;
+
       long epoch = 0;
-      ApplicationAttemptId appAttemptID = null;
+      if (containerIdStr.regionMatches(pos1 + 1, EPOCH_PREFIX, 0,
-      if (epochOrClusterTimestampStr.startsWith(EPOCH_PREFIX)) {
+            EPOCH_PREFIX.length())) {
-        String epochStr = epochOrClusterTimestampStr;
+        int pos2 = containerIdStr.indexOf('_', pos1 + 1);
-        epoch = Integer.parseInt(epochStr.substring(EPOCH_PREFIX.length()));
+        if (pos2 < 0) {
-        appAttemptID = toApplicationAttemptId(it);
+          throw new IllegalArgumentException("Invalid ContainerId: "
-      } else {
+              + containerIdStr);
-        String clusterTimestampStr = epochOrClusterTimestampStr;
-        long clusterTimestamp = Long.parseLong(clusterTimestampStr);
-        appAttemptID = toApplicationAttemptId(clusterTimestamp, it);
         }
-      long id = Long.parseLong(it.next());
+        String epochStr = containerIdStr.substring(
+            pos1 + 1 + EPOCH_PREFIX.length(), pos2);
+        epoch = Integer.parseInt(epochStr);
+        // rewind the current position
+        pos1 = pos2;
+      }
+      int pos2 = containerIdStr.indexOf('_', pos1 + 1);
+      if (pos2 < 0) {
+        throw new IllegalArgumentException("Invalid ContainerId: "
+            + containerIdStr);
+      }
+      long clusterTimestamp = Long.parseLong(
+        containerIdStr.substring(pos1 + 1, pos2));
+
+      int pos3 = containerIdStr.indexOf('_', pos2 + 1);
+      if (pos3 < 0) {
+        throw new IllegalArgumentException("Invalid ContainerId: "
+            + containerIdStr);
+      }
+      int appId = Integer.parseInt(containerIdStr.substring(pos2 + 1, pos3));
+      ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp,
+          appId);
+      int pos4 = containerIdStr.indexOf('_', pos3 + 1);
+      if (pos4 < 0) {
+        throw new IllegalArgumentException("Invalid ContainerId: "
+            + containerIdStr);
+      }
+      int attemptId = Integer.parseInt(
+          containerIdStr.substring(pos3 + 1, pos4));
+      ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(applicationId, attemptId);
+      long id = Long.parseLong(containerIdStr.substring(pos4 + 1));
       long cid = (epoch << 40) | id;
-      ContainerId containerId = ContainerId.newContainerId(appAttemptID, cid);
+      ContainerId containerId = ContainerId.newContainerId(appAttemptId, cid);
       return containerId;
     } catch (NumberFormatException n) {
       throw new IllegalArgumentException("Invalid ContainerId: "
           + containerIdStr, n);
-    } catch (NoSuchElementException e) {
-      throw new IllegalArgumentException("Invalid ContainerId: "
-          + containerIdStr, e);
     }
   }
 
-  private static ApplicationAttemptId toApplicationAttemptId(
-      Iterator<String> it) throws NumberFormatException {
-    return toApplicationAttemptId(Long.parseLong(it.next()), it);
-  }
-
-  private static ApplicationAttemptId toApplicationAttemptId(
-      long clusterTimestamp, Iterator<String> it) throws NumberFormatException {
-    ApplicationId appId = ApplicationId.newInstance(clusterTimestamp,
-        Integer.parseInt(it.next()));
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
-    return appAttemptId;
-  }
-
   protected abstract void build();
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java

@@ -19,11 +19,11 @@
 package org.apache.hadoop.yarn.api.records;
 
 import java.io.IOException;
-import java.text.NumberFormat;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.FastNumberFormat;
 import org.apache.hadoop.yarn.util.Records;
 
 /**
@@ -89,16 +89,7 @@ public abstract class ReservationId implements Comparable<ReservationId> {
 
   protected abstract void build();
 
-  static final ThreadLocal<NumberFormat> reservIdFormat =
+  private static final int RESERVATION_ID_MIN_DIGITS = 4;
-      new ThreadLocal<NumberFormat>() {
-        @Override
-        public NumberFormat initialValue() {
-          NumberFormat fmt = NumberFormat.getInstance();
-          fmt.setGroupingUsed(false);
-          fmt.setMinimumIntegerDigits(4);
-          return fmt;
-        }
-      };
 
   @Override
   public int compareTo(ReservationId other) {
@@ -112,8 +103,12 @@ public abstract class ReservationId implements Comparable<ReservationId> {
 
   @Override
   public String toString() {
-    return reserveIdStrPrefix + this.getClusterTimestamp() + "_"
+    StringBuilder sb = new StringBuilder(64);
-        + reservIdFormat.get().format(getId());
+    sb.append(reserveIdStrPrefix);
+    sb.append(getClusterTimestamp());
+    sb.append('_');
+    FastNumberFormat.format(sb, getId(), RESERVATION_ID_MIN_DIGITS);
+    return sb.toString();
   }
 
   /**