From f6ef841f748f2b0a3ef91731d6e1756ca65a0ee7 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Mon, 22 Feb 2016 17:18:49 -0800 Subject: [PATCH] HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone. Contributed by Anu Engineer. --- .../hadoop-hdfs/CHANGES-HDFS-7240.txt | 6 + .../ozone/web/ObjectStoreApplication.java | 2 + .../web/exceptions/OzoneExceptionMapper.java | 37 +++ .../hadoop/ozone/web/TestOzoneVolumes.java | 288 ++++++++++++++++++ 4 files changed, 333 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt index 78522d897e1..8595b33e3b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt @@ -27,3 +27,9 @@ HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via Arpit Agarwal) + + HDFS-8757. OzoneHandler : Add localStorageHandler support for Buckets. + (Anu Engineer) + + HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone. + (Anu Engineer via cnauroth) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java index d0973478cd4..0257e069009 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.web; +import org.apache.hadoop.ozone.web.exceptions.OzoneExceptionMapper; import org.apache.hadoop.ozone.web.handlers.BucketHandler; import org.apache.hadoop.ozone.web.handlers.VolumeHandler; @@ -38,6 +39,7 @@ public class ObjectStoreApplication extends Application { HashSet> set = new HashSet<>(); set.add(BucketHandler.class); set.add(VolumeHandler.class); + set.add(OzoneExceptionMapper.class); return set; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java new file mode 100644 index 00000000000..d16a64f13de --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.web.exceptions; + + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; + +/** + * Class the represents various errors returned by the + * Object Layer. + */ +public class OzoneExceptionMapper implements ExceptionMapper { + + @Override + public Response toResponse(OzoneException exception) { + return Response.status((int)exception.getHttpCode()) + .entity(exception.toJsonString()).build(); + } + +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java new file mode 100644 index 00000000000..5a2e63cbe49 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.web; + +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.web.exceptions.ErrorTable; +import org.apache.hadoop.ozone.web.headers.Header; +import org.apache.hadoop.ozone.web.utils.OzoneConsts; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; +import org.apache.hadoop.util.Time; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.DefaultHttpClient; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.ws.rs.core.HttpHeaders; +import java.io.IOException; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; + +import static java.net.HttpURLConnection.HTTP_CREATED; +import static org.junit.Assert.assertEquals; + +public class TestOzoneVolumes { + static MiniDFSCluster cluster = null; + static int port = 0; + + /** + * Create a MiniDFSCluster for testing. + *

+ * Ozone is made active by setting DFS_OBJECTSTORE_ENABLED_KEY = true and + * DFS_STORAGE_HANDLER_TYPE_KEY = "local" , which uses a local directory to + * emulate Ozone backend. + * + * @throws IOException + */ + @BeforeClass + public static void init() throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + + URL p = conf.getClass().getResource(""); + String path = p.getPath(); + path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, + OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT); + + conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path); + conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true); + conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local"); + + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + DataNode dataNode = cluster.getDataNodes().get(0); + port = dataNode.getInfoPort(); + } + + /** + * shutdown MiniDFSCluster + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Creates Volumes on Ozone Store. + * + * @throws IOException + */ + @Test + public void testCreateVolumes() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + try { + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), HTTP_CREATED, + response.getStatusLine().getStatusCode()); + } finally { + client.getConnectionManager().shutdown(); + } + } + + /** + * Create Volumes with Quota. + * + * @throws IOException + */ + @Test + public void testCreateVolumesWithQuota() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + try { + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s?quota=10TB", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), HTTP_CREATED, + response.getStatusLine().getStatusCode()); + } finally { + client.getConnectionManager().shutdown(); + } + } + + /** + * Create Volumes with Invalid Quota. + * + * @throws IOException + */ + @Test + public void testCreateVolumesWithInvalidQuota() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + try { + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s?quota=NaN", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), ErrorTable.MALFORMED_QUOTA + .getHttpCode(), + response.getStatusLine().getStatusCode()); + } finally { + client.getConnectionManager().shutdown(); + } + } + + /** + * To create a volume a user name must be specified using OZONE_USER header. + * This test verifies that we get an error in case we call without a OZONE + * user name. + * + * @throws IOException + */ + @Test + public void testCreateVolumesWithInvalidUser() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + try { + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s?quota=1TB", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + + HttpResponse response = client.execute(httppost); + + assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(), + response.getStatusLine().getStatusCode()); + } finally { + client.getConnectionManager().shutdown(); + } + } + + /** + * Only Admins can create volumes in Ozone. This test uses simple userauth as + * backend and hdfs and root are admin users in the simple backend. + *

+ * This test tries to create a volume as user bilbo. + * + * @throws IOException + */ + @Test + public void testCreateVolumesWithOutAdminRights() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + try { + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s?quota=NaN", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + "bilbo"); // This is not a root user in Simple + httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER); + + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), ErrorTable.ACCESS_DENIED.getHttpCode(), + response.getStatusLine().getStatusCode()); + } finally { + client.getConnectionManager().shutdown(); + } + } + + /** + * Create a bunch of volumes in a loop + * + * @throws IOException + */ + @Test + public void testCreateVolumesInLoop() throws IOException { + SimpleDateFormat format = + new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); + + for (int x = 0; x < 1000; x++) { + HttpClient client = new DefaultHttpClient(); + String volumeName = OzoneUtils.getRequestID().toLowerCase(); + String userName = OzoneUtils.getRequestID().toLowerCase(); + + HttpPost httppost = new HttpPost( + String.format("http://localhost:%d/%s?quota=10TB", port, volumeName)); + + httppost.addHeader(Header.OZONE_VERSION_HEADER, + Header.OZONE_V1_VERSION_HEADER); + httppost.addHeader(HttpHeaders.DATE, + format.format(new Date(Time.monotonicNow()))); + httppost.addHeader(HttpHeaders.AUTHORIZATION, + Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + + OzoneConsts.OZONE_SIMPLE_HDFS_USER); + httppost.addHeader(Header.OZONE_USER, userName); + + HttpResponse response = client.execute(httppost); + assertEquals(response.toString(), HTTP_CREATED, + response.getStatusLine().getStatusCode()); + client.getConnectionManager().shutdown(); + } + } +}