HDFS-11984. Ozone: Ensures listKey lists all required key fields. Contributed by Yiqun Lin.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java

@@ -51,11 +51,11 @@ import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
 import org.apache.hadoop.ozone.web.response.ListVolumes;
 import org.apache.hadoop.ozone.web.response.VolumeInfo;
 import org.apache.hadoop.ozone.web.response.VolumeOwner;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.ozone.web.response.ListBuckets;
 import org.apache.hadoop.ozone.web.response.BucketInfo;
 import org.apache.hadoop.ozone.web.response.KeyInfo;
 import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -433,8 +433,10 @@ public final class DistributedStorageHandler implements StorageHandler {
     keyInfo.setVersion(0);
     keyInfo.setKeyName(ksmKeyInfo.getKeyName());
     keyInfo.setSize(ksmKeyInfo.getDataSize());
-    keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime()));
+    keyInfo.setCreatedOn(
-    keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime()));
+        OzoneUtils.formatTime(ksmKeyInfo.getCreationTime()));
+    keyInfo.setModifiedOn(
+        OzoneUtils.formatTime(ksmKeyInfo.getModificationTime()));
     return keyInfo;
   }
 
@@ -474,6 +476,10 @@ public final class DistributedStorageHandler implements StorageHandler {
         tempInfo.setVersion(0);
         tempInfo.setKeyName(info.getKeyName());
         tempInfo.setSize(info.getDataSize());
+        tempInfo.setCreatedOn(
+            OzoneUtils.formatTime(info.getCreationTime()));
+        tempInfo.setModifiedOn(
+            OzoneUtils.formatTime(info.getModificationTime()));
 
         result.addKey(tempInfo);
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.handlers.UserArgs;
 import org.apache.hadoop.ozone.web.headers.Header;
+import org.apache.hadoop.util.Time;
 
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Request;
@@ -58,6 +59,21 @@ public final class OzoneUtils {
     // Never constructed
   }
 
+  /**
+   * Date format that used in ozone. Here the format is thread safe to use.
+   */
+  private static final ThreadLocal<SimpleDateFormat> DATE_FORMAT =
+      new ThreadLocal<SimpleDateFormat>() {
+    @Override
+    protected SimpleDateFormat initialValue() {
+      SimpleDateFormat format = new SimpleDateFormat(
+          OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
+      format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
+
+      return format;
+    }
+  };
+
   /**
    * verifies that bucket name / volume name is a valid DNS name.
    *
@@ -242,12 +258,8 @@ public final class OzoneUtils {
   public static synchronized Date parseDate(String dateString, String reqID,
                                             String resource, String hostname)
       throws OzoneException {
-    SimpleDateFormat format =
-        new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
-    format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-
     try {
-      return format.parse(dateString);
+      return DATE_FORMAT.get().parse(dateString);
     } catch (ParseException ex) {
       OzoneException exp =
           ErrorTable.newError(ErrorTable.BAD_DATE, reqID, resource, hostname);
@@ -267,10 +279,7 @@ public final class OzoneUtils {
    */
   public static Response getResponse(UserArgs args, int statusCode,
                                      String payload) {
-    SimpleDateFormat format =
+    String date = DATE_FORMAT.get().format(new Date(Time.now()));
-        new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
-    format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-    String date = format.format(new Date(System.currentTimeMillis()));
     return Response.ok(payload)
         .header(Header.OZONE_SERVER_NAME, args.getHostName())
         .header(Header.OZONE_REQUEST_ID, args.getRequestID())
@@ -288,10 +297,7 @@ public final class OzoneUtils {
    */
   public static Response getResponse(UserArgs args, int statusCode,
                                      LengthInputStream stream) {
-    SimpleDateFormat format =
+    String date = DATE_FORMAT.get().format(new Date(Time.now()));
-        new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
-    format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-    String date = format.format(new Date(System.currentTimeMillis()));
     return Response.ok(stream, MediaType.APPLICATION_OCTET_STREAM)
         .header(Header.OZONE_SERVER_NAME, args.getHostName())
         .header(Header.OZONE_REQUEST_ID, args.getRequestID())
@@ -319,4 +325,21 @@ public final class OzoneUtils {
     }
     return dirPath;
   }
+
+  /**
+   * Convert time in millisecond to a human readable format required in ozone.
+   * @return a human readable string for the input time
+   */
+  public static String formatTime(long millis) {
+    return DATE_FORMAT.get().format(millis);
+  }
+
+  /**
+   * Convert time in ozone date format to millisecond.
+   * @return time in milliseconds
+   */
+  public static long formatDate(String date) throws ParseException {
+    Preconditions.checkNotNull(date, "Date string should not be null.");
+    return DATE_FORMAT.get().parse(date).getTime();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java

@@ -961,7 +961,7 @@ public class TestKeySpaceManager {
     String adminName = "admin" + RandomStringUtils.randomNumeric(5);
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-    long currentTime = Time.monotonicNow();
+    long currentTime = Time.now();
 
     VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
     createVolumeArgs.setUserName(userName);
@@ -982,8 +982,12 @@ public class TestKeySpaceManager {
     stream.close();
 
     KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
-    Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime);
+    // Compare the time in second unit since the date string reparsed to
-    Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime);
+    // millisecond will lose precision.
+    Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getCreatedOn())
+        / 1000) >= (currentTime / 1000));
+    Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getModifiedOn())
+        / 1000) >= (currentTime / 1000));
     Assert.assertEquals(keyName, keyInfo.getKeyName());
     Assert.assertEquals(4096, keyInfo.getSize());
   }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java

@@ -626,7 +626,11 @@ public class TestOzoneShell {
 
     // verify the response output
     assertEquals(0, ToolRunner.run(shell, args));
-    assertTrue(out.toString().contains(keyName));
+
+    String output = out.toString();
+    assertTrue(output.contains(keyName));
+    assertTrue(output.contains("createdOn") && output.contains("modifiedOn")
+        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
     // reset stream
     out.reset();
@@ -665,12 +669,23 @@ public class TestOzoneShell {
 
     List<String> keys = getValueLines("keyName", out.toString());
     assertEquals(11, keys.size());
+
+    List<String> creationTime = getValueLines("createdOn", out.toString());
+    List<String> modificationTime = getValueLines("modifiedOn", out.toString());
+    assertEquals(11, creationTime.size());
+    assertEquals(11, modificationTime.size());
+
     // sort key names since the return keys isn't in created order
     Collections.sort(keyNames);
     // return key names should be [test-key0, test-key1,
     // test-key10, test-key2, ,..., test-key9]
     for (int i = 0; i < keys.size(); i++) {
       assertTrue(keys.get(i).contains(keyNames.get(i)));
+
+      // verify the creation/modification time of key
+      assertTrue(creationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
+      assertTrue(
+          modificationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
     }
 
     out.reset();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.junit.AfterClass;
@@ -47,11 +48,13 @@ import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.text.ParseException;
 import java.util.List;
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TestKeys {
@@ -387,7 +390,8 @@ public class TestKeys {
   }
 
   @Test
-  public void testPutAndListKey() throws OzoneException, IOException {
+  public void testPutAndListKey()
+      throws OzoneException, IOException, ParseException {
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
     String delimiter = RandomStringUtils.randomAscii(1);
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
@@ -395,12 +399,13 @@ public class TestKeys {
   }
 
   static void runTestPutAndListKey(PutHelper helper)
-      throws OzoneException, IOException {
+      throws OzoneException, IOException, ParseException {
     final OzoneRestClient client = helper.client;
     helper.putKey();
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());
 
+    long currentTime = Time.now();
     // add keys [list-key0, list-key1, ..., list-key9]
     for (int x = 0; x < 10; x++) {
       String newkeyName = "list-key" + x;
@@ -414,6 +419,22 @@ public class TestKeys {
 
     Assert.assertEquals(keyList1.size(), 11);
     Assert.assertEquals(keyList2.size(), 11);
+    // Verify the key creation/modification time. Here we compare the time in
+    // second unit since the date string reparsed to millisecond will
+    // lose precision.
+    for (OzoneKey key : keyList1) {
+      assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn())
+          / 1000) >= (currentTime / 1000));
+      assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn())
+          / 1000) >= (currentTime / 1000));
+    }
+
+    for (OzoneKey key : keyList2) {
+      assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn())
+          / 1000) >= (currentTime / 1000));
+      assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn())
+          / 1000) >= (currentTime / 1000));
+    }
 
     // test maxLength parameter of list keys
     keyList1 = helper.getBucket().listKeys("1", null, null);
@@ -459,14 +480,17 @@ public class TestKeys {
   }
 
   @Test
-  public void testGetKeyInfo() throws OzoneException, IOException {
+  public void testGetKeyInfo()
+      throws OzoneException, IOException, ParseException {
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
     String delimiter = RandomStringUtils.randomAscii(1);
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
         getMultiPartKey(delimiter)));
   }
 
-  static void runTestGetKeyInfo(PutHelper helper) throws OzoneException {
+  static void runTestGetKeyInfo(PutHelper helper)
+      throws OzoneException, ParseException {
+    long currentTime = Time.now();
     String keyName = helper.putKey();
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());
@@ -474,5 +498,14 @@ public class TestKeys {
     OzoneKey keyInfo = helper.getBucket().getKeyInfo(keyName);
     assertNotNull(keyInfo.getObjectInfo());
     assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
+
+    // Compare the time in second unit since the date string reparsed to
+    // millisecond will lose precision.
+    Assert.assertTrue(
+        (OzoneUtils.formatDate(keyInfo.getObjectInfo().getCreatedOn())
+            / 1000) >= (currentTime / 1000));
+    Assert.assertTrue(
+        (OzoneUtils.formatDate(keyInfo.getObjectInfo().getModifiedOn())
+            / 1000) >= (currentTime / 1000));
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java

@@ -29,6 +29,7 @@ import org.junit.rules.Timeout;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.text.ParseException;
 
 import static org.apache.hadoop.ozone.web.client.TestKeys.*;
 
@@ -91,7 +92,8 @@ public class TestKeysRatis {
   }
 
   @Test
-  public void testPutAndListKey() throws OzoneException, IOException {
+  public void testPutAndListKey()
+      throws OzoneException, IOException, ParseException {
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
     String delimiter = RandomStringUtils.randomAlphanumeric(1);
     runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
@@ -99,7 +101,8 @@ public class TestKeysRatis {
   }
 
   @Test
-  public void testGetKeyInfo() throws OzoneException, IOException {
+  public void testGetKeyInfo()
+      throws OzoneException, IOException, ParseException {
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
     String delimiter = RandomStringUtils.randomAlphanumeric(1);
     runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,