From 712bd70ac7032f178acd5193d869faa94f85ef40 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Tue, 26 Sep 2017 14:43:52 +0800 Subject: [PATCH] HDFS-12205. Ozone: List Key on an empty ozone bucket fails with command failed error. Contributed by Lokesh Jain. --- .../localstorage/OzoneMetadataManager.java | 11 ++-- .../ozone/web/TestLocalOzoneVolumes.java | 4 ++ .../hadoop/ozone/web/TestOzoneHelper.java | 60 +++++++++++++++++++ 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java index c0f96302d78..69692df2ef0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java @@ -634,6 +634,10 @@ public void createBucket(BucketArgs args) throws OzoneException { userDB.put(args.getParentName().getBytes(encoding), bucketList.toDBString().getBytes(encoding)); + // Update userDB with volume/bucket -> empty key list + userDB.put(args.getResourceName().getBytes(encoding), + new ListKeys().toDBString().getBytes(encoding)); + // and update the metadataDB with volume/bucket->BucketInfo metadataDB.put(args.getResourceName().getBytes(encoding), bucketInfo.toDBString().getBytes(encoding)); @@ -916,12 +920,7 @@ public void commitKey(KeyArgs args, OutputStream stream) ListKeys keyList; byte[] bucketListBytes = userDB.get(args.getParentName() .getBytes(encoding)); - if (bucketListBytes == null) { - keyList = new ListKeys(); - } else { - keyList = ListKeys.parse(new String(bucketListBytes, encoding)); - } - + keyList = ListKeys.parse(new String(bucketListBytes, encoding)); KeyInfo keyInfo; byte[] objectBytes = metadataDB.get(args.getResourceName() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestLocalOzoneVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestLocalOzoneVolumes.java index 561eaf37b39..d78bc66dbfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestLocalOzoneVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestLocalOzoneVolumes.java @@ -178,4 +178,8 @@ public void testGetVolumesOfAnotherUserShouldFail() throws IOException { super.testGetVolumesOfAnotherUserShouldFail(port); } + @Test + public void testListKeyOnEmptyBucket() throws IOException { + super.testListKeyOnEmptyBucket(port); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneHelper.java index 5393f331a24..eb23ac2d149 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneHelper.java @@ -351,4 +351,64 @@ public void testGetVolumesOfAnotherUserShouldFail(int port) } } + public void testListKeyOnEmptyBucket(int port) throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + CloseableHttpClient client = createHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + String bucketName = OzoneUtils.getRequestID().toLowerCase() + "bucket"; + try { + + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s", port, volumeName)); + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), HTTP_CREATED, + response.getStatusLine().getStatusCode()); + client.close(); + + client = createHttpClient(); + httppost = new HttpPost(String + .format("http://localhost:%d/%s/%s", port, volumeName, bucketName)); + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + response = client.execute(httppost); + assertEquals(response.toString(), HTTP_CREATED, + response.getStatusLine().getStatusCode()); + client.close(); + + client = createHttpClient(); + HttpGet httpget = new HttpGet(String + .format("http://localhost:%d/%s/%s", port, volumeName, bucketName)); + httpget.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httpget.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httpget.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httpget.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + response = client.execute(httpget); + assertEquals(response.toString() + " " + response.getStatusLine() + .getReasonPhrase(), HTTP_OK, + response.getStatusLine().getStatusCode()); + + } finally { + client.close(); + } + } + }