From 1ad95cf2a9360646c289230498e68fc936f71693 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Tue, 1 Aug 2017 13:47:44 +0800 Subject: [PATCH] HDFS-11984. Ozone: Ensures listKey lists all required key fields. Contributed by Yiqun Lin. --- .../storage/DistributedStorageHandler.java | 12 +++-- .../hadoop/ozone/web/utils/OzoneUtils.java | 49 ++++++++++++++----- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 10 ++-- .../hadoop/ozone/ozShell/TestOzoneShell.java | 17 ++++++- .../hadoop/ozone/web/client/TestKeys.java | 41 ++++++++++++++-- .../ozone/web/client/TestKeysRatis.java | 7 ++- 6 files changed, 110 insertions(+), 26 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 51affd3a016..c5754e7863a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -51,11 +51,11 @@ import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.response.ListVolumes; import org.apache.hadoop.ozone.web.response.VolumeInfo; import org.apache.hadoop.ozone.web.response.VolumeOwner; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.KeyInfo; import org.apache.hadoop.ozone.web.response.ListKeys; -import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -433,8 +433,10 @@ public final class DistributedStorageHandler implements StorageHandler { keyInfo.setVersion(0); keyInfo.setKeyName(ksmKeyInfo.getKeyName()); keyInfo.setSize(ksmKeyInfo.getDataSize()); - keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime())); - keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime())); + keyInfo.setCreatedOn( + OzoneUtils.formatTime(ksmKeyInfo.getCreationTime())); + keyInfo.setModifiedOn( + OzoneUtils.formatTime(ksmKeyInfo.getModificationTime())); return keyInfo; } @@ -474,6 +476,10 @@ public final class DistributedStorageHandler implements StorageHandler { tempInfo.setVersion(0); tempInfo.setKeyName(info.getKeyName()); tempInfo.setSize(info.getDataSize()); + tempInfo.setCreatedOn( + OzoneUtils.formatTime(info.getCreationTime())); + tempInfo.setModifiedOn( + OzoneUtils.formatTime(info.getModificationTime())); result.addKey(tempInfo); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java index 437a1762354..c417601da33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.web.exceptions.ErrorTable; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.handlers.UserArgs; import org.apache.hadoop.ozone.web.headers.Header; +import org.apache.hadoop.util.Time; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Request; @@ -58,6 +59,21 @@ public final class OzoneUtils { // Never constructed } + /** + * Date format that used in ozone. Here the format is thread safe to use. + */ + private static final ThreadLocal DATE_FORMAT = + new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + SimpleDateFormat format = new SimpleDateFormat( + OzoneConsts.OZONE_DATE_FORMAT, Locale.US); + format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE)); + + return format; + } + }; + /** * verifies that bucket name / volume name is a valid DNS name. * @@ -242,12 +258,8 @@ public final class OzoneUtils { public static synchronized Date parseDate(String dateString, String reqID, String resource, String hostname) throws OzoneException { - SimpleDateFormat format = - new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US); - format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE)); - try { - return format.parse(dateString); + return DATE_FORMAT.get().parse(dateString); } catch (ParseException ex) { OzoneException exp = ErrorTable.newError(ErrorTable.BAD_DATE, reqID, resource, hostname); @@ -267,10 +279,7 @@ public final class OzoneUtils { */ public static Response getResponse(UserArgs args, int statusCode, String payload) { - SimpleDateFormat format = - new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US); - format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE)); - String date = format.format(new Date(System.currentTimeMillis())); + String date = DATE_FORMAT.get().format(new Date(Time.now())); return Response.ok(payload) .header(Header.OZONE_SERVER_NAME, args.getHostName()) .header(Header.OZONE_REQUEST_ID, args.getRequestID()) @@ -288,10 +297,7 @@ public final class OzoneUtils { */ public static Response getResponse(UserArgs args, int statusCode, LengthInputStream stream) { - SimpleDateFormat format = - new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US); - format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE)); - String date = format.format(new Date(System.currentTimeMillis())); + String date = DATE_FORMAT.get().format(new Date(Time.now())); return Response.ok(stream, MediaType.APPLICATION_OCTET_STREAM) .header(Header.OZONE_SERVER_NAME, args.getHostName()) .header(Header.OZONE_REQUEST_ID, args.getRequestID()) @@ -319,4 +325,21 @@ public final class OzoneUtils { } return dirPath; } + + /** + * Convert time in millisecond to a human readable format required in ozone. + * @return a human readable string for the input time + */ + public static String formatTime(long millis) { + return DATE_FORMAT.get().format(millis); + } + + /** + * Convert time in ozone date format to millisecond. + * @return time in milliseconds + */ + public static long formatDate(String date) throws ParseException { + Preconditions.checkNotNull(date, "Date string should not be null."); + return DATE_FORMAT.get().parse(date).getTime(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java index 1a68eb2d746..63654007b9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java @@ -961,7 +961,7 @@ public class TestKeySpaceManager { String adminName = "admin" + RandomStringUtils.randomNumeric(5); String volumeName = "volume" + RandomStringUtils.randomNumeric(5); String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); - long currentTime = Time.monotonicNow(); + long currentTime = Time.now(); VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs); createVolumeArgs.setUserName(userName); @@ -982,8 +982,12 @@ public class TestKeySpaceManager { stream.close(); KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs); - Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime); - Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime); + // Compare the time in second unit since the date string reparsed to + // millisecond will lose precision. + Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getCreatedOn()) + / 1000) >= (currentTime / 1000)); + Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getModifiedOn()) + / 1000) >= (currentTime / 1000)); Assert.assertEquals(keyName, keyInfo.getKeyName()); Assert.assertEquals(4096, keyInfo.getSize()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java index c1597ebb684..b8e353541b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java @@ -626,7 +626,11 @@ public class TestOzoneShell { // verify the response output assertEquals(0, ToolRunner.run(shell, args)); - assertTrue(out.toString().contains(keyName)); + + String output = out.toString(); + assertTrue(output.contains(keyName)); + assertTrue(output.contains("createdOn") && output.contains("modifiedOn") + && output.contains(OzoneConsts.OZONE_TIME_ZONE)); // reset stream out.reset(); @@ -665,12 +669,23 @@ public class TestOzoneShell { List keys = getValueLines("keyName", out.toString()); assertEquals(11, keys.size()); + + List creationTime = getValueLines("createdOn", out.toString()); + List modificationTime = getValueLines("modifiedOn", out.toString()); + assertEquals(11, creationTime.size()); + assertEquals(11, modificationTime.size()); + // sort key names since the return keys isn't in created order Collections.sort(keyNames); // return key names should be [test-key0, test-key1, // test-key10, test-key2, ,..., test-key9] for (int i = 0; i < keys.size(); i++) { assertTrue(keys.get(i).contains(keyNames.get(i))); + + // verify the creation/modification time of key + assertTrue(creationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE)); + assertTrue( + modificationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE)); } out.reset(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index d9168ab20d4..456b16892a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.AfterClass; @@ -47,11 +48,13 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.ParseException; import java.util.List; import java.util.Random; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TestKeys { @@ -387,7 +390,8 @@ public class TestKeys { } @Test - public void testPutAndListKey() throws OzoneException, IOException { + public void testPutAndListKey() + throws OzoneException, IOException, ParseException { runTestPutAndListKey(new PutHelper(ozoneRestClient, path)); String delimiter = RandomStringUtils.randomAscii(1); runTestPutAndListKey(new PutHelper(ozoneRestClient, path, @@ -395,12 +399,13 @@ public class TestKeys { } static void runTestPutAndListKey(PutHelper helper) - throws OzoneException, IOException { + throws OzoneException, IOException, ParseException { final OzoneRestClient client = helper.client; helper.putKey(); assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); + long currentTime = Time.now(); // add keys [list-key0, list-key1, ..., list-key9] for (int x = 0; x < 10; x++) { String newkeyName = "list-key" + x; @@ -414,6 +419,22 @@ public class TestKeys { Assert.assertEquals(keyList1.size(), 11); Assert.assertEquals(keyList2.size(), 11); + // Verify the key creation/modification time. Here we compare the time in + // second unit since the date string reparsed to millisecond will + // lose precision. + for (OzoneKey key : keyList1) { + assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn()) + / 1000) >= (currentTime / 1000)); + assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn()) + / 1000) >= (currentTime / 1000)); + } + + for (OzoneKey key : keyList2) { + assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn()) + / 1000) >= (currentTime / 1000)); + assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn()) + / 1000) >= (currentTime / 1000)); + } // test maxLength parameter of list keys keyList1 = helper.getBucket().listKeys("1", null, null); @@ -459,14 +480,17 @@ public class TestKeys { } @Test - public void testGetKeyInfo() throws OzoneException, IOException { + public void testGetKeyInfo() + throws OzoneException, IOException, ParseException { runTestGetKeyInfo(new PutHelper(ozoneRestClient, path)); String delimiter = RandomStringUtils.randomAscii(1); runTestGetKeyInfo(new PutHelper(ozoneRestClient, path, getMultiPartKey(delimiter))); } - static void runTestGetKeyInfo(PutHelper helper) throws OzoneException { + static void runTestGetKeyInfo(PutHelper helper) + throws OzoneException, ParseException { + long currentTime = Time.now(); String keyName = helper.putKey(); assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); @@ -474,5 +498,14 @@ public class TestKeys { OzoneKey keyInfo = helper.getBucket().getKeyInfo(keyName); assertNotNull(keyInfo.getObjectInfo()); assertEquals(keyName, keyInfo.getObjectInfo().getKeyName()); + + // Compare the time in second unit since the date string reparsed to + // millisecond will lose precision. + Assert.assertTrue( + (OzoneUtils.formatDate(keyInfo.getObjectInfo().getCreatedOn()) + / 1000) >= (currentTime / 1000)); + Assert.assertTrue( + (OzoneUtils.formatDate(keyInfo.getObjectInfo().getModifiedOn()) + / 1000) >= (currentTime / 1000)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java index 1d33644e766..076297302f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java @@ -29,6 +29,7 @@ import org.junit.rules.Timeout; import java.io.IOException; import java.net.URISyntaxException; +import java.text.ParseException; import static org.apache.hadoop.ozone.web.client.TestKeys.*; @@ -91,7 +92,8 @@ public class TestKeysRatis { } @Test - public void testPutAndListKey() throws OzoneException, IOException { + public void testPutAndListKey() + throws OzoneException, IOException, ParseException { runTestPutAndListKey(new PutHelper(ozoneRestClient, path)); String delimiter = RandomStringUtils.randomAlphanumeric(1); runTestPutAndListKey(new PutHelper(ozoneRestClient, path, @@ -99,7 +101,8 @@ public class TestKeysRatis { } @Test - public void testGetKeyInfo() throws OzoneException, IOException { + public void testGetKeyInfo() + throws OzoneException, IOException, ParseException { runTestGetKeyInfo(new PutHelper(ozoneRestClient, path)); String delimiter = RandomStringUtils.randomAlphanumeric(1); runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,