HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone. Contributed by Anu Engineer.
This commit is contained in:
cnauroth
Arpit Agarwal
parent
14b85da0ea
commit
ed258dc633
|
|
@ -27,3 +27,9 @@
|
||||||
|
|
||||||
HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via
|
HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via
|
||||||
Arpit Agarwal)
|
Arpit Agarwal)
|
||||||
|
|
||||||
|
HDFS-8757. OzoneHandler : Add localStorageHandler support for Buckets.
|
||||||
|
(Anu Engineer)
|
||||||
|
|
||||||
|
HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone.
|
||||||
|
(Anu Engineer via cnauroth)
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@
|
||||||
|
|
||||||
package org.apache.hadoop.ozone.web;
|
package org.apache.hadoop.ozone.web;
|
||||||
|
|
||||||
|
import org.apache.hadoop.ozone.web.exceptions.OzoneExceptionMapper;
|
||||||
import org.apache.hadoop.ozone.web.handlers.BucketHandler;
|
import org.apache.hadoop.ozone.web.handlers.BucketHandler;
|
||||||
import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
|
import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
|
||||||
|
|
||||||
|
|
@ -38,6 +39,7 @@ public class ObjectStoreApplication extends Application {
|
||||||
HashSet<Class<?>> set = new HashSet<>();
|
HashSet<Class<?>> set = new HashSet<>();
|
||||||
set.add(BucketHandler.class);
|
set.add(BucketHandler.class);
|
||||||
set.add(VolumeHandler.class);
|
set.add(VolumeHandler.class);
|
||||||
|
set.add(OzoneExceptionMapper.class);
|
||||||
return set;
|
return set;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.web.exceptions;
|
||||||
|
|
||||||
|
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
import javax.ws.rs.ext.ExceptionMapper;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class the represents various errors returned by the
|
||||||
|
* Object Layer.
|
||||||
|
*/
|
||||||
|
public class OzoneExceptionMapper implements ExceptionMapper<OzoneException> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Response toResponse(OzoneException exception) {
|
||||||
|
return Response.status((int)exception.getHttpCode())
|
||||||
|
.entity(exception.toJsonString()).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,288 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.ozone.web;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||||
|
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
||||||
|
import org.apache.hadoop.ozone.OzoneConfiguration;
|
||||||
|
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
|
||||||
|
import org.apache.hadoop.ozone.web.headers.Header;
|
||||||
|
import org.apache.hadoop.ozone.web.utils.OzoneConsts;
|
||||||
|
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
|
||||||
|
import org.apache.hadoop.util.Time;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.methods.HttpPost;
|
||||||
|
import org.apache.http.impl.client.DefaultHttpClient;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import javax.ws.rs.core.HttpHeaders;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Locale;
|
||||||
|
|
||||||
|
import static java.net.HttpURLConnection.HTTP_CREATED;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
|
||||||
|
public class TestOzoneVolumes {
|
||||||
|
static MiniDFSCluster cluster = null;
|
||||||
|
static int port = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a MiniDFSCluster for testing.
|
||||||
|
* <p>
|
||||||
|
* Ozone is made active by setting DFS_OBJECTSTORE_ENABLED_KEY = true and
|
||||||
|
* DFS_STORAGE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
|
||||||
|
* emulate Ozone backend.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws IOException {
|
||||||
|
OzoneConfiguration conf = new OzoneConfiguration();
|
||||||
|
|
||||||
|
URL p = conf.getClass().getResource("");
|
||||||
|
String path = p.getPath();
|
||||||
|
path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
|
||||||
|
OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
|
||||||
|
|
||||||
|
conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
|
||||||
|
conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
|
||||||
|
conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
|
||||||
|
|
||||||
|
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||||
|
cluster.waitActive();
|
||||||
|
DataNode dataNode = cluster.getDataNodes().get(0);
|
||||||
|
port = dataNode.getInfoPort();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* shutdown MiniDFSCluster
|
||||||
|
*/
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() {
|
||||||
|
if (cluster != null) {
|
||||||
|
cluster.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates Volumes on Ozone Store.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumes() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
try {
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
assertEquals(response.toString(), HTTP_CREATED,
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
} finally {
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create Volumes with Quota.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumesWithQuota() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
try {
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
assertEquals(response.toString(), HTTP_CREATED,
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
} finally {
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create Volumes with Invalid Quota.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumesWithInvalidQuota() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
try {
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
assertEquals(response.toString(), ErrorTable.MALFORMED_QUOTA
|
||||||
|
.getHttpCode(),
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
} finally {
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To create a volume a user name must be specified using OZONE_USER header.
|
||||||
|
* This test verifies that we get an error in case we call without a OZONE
|
||||||
|
* user name.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumesWithInvalidUser() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
try {
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s?quota=1TB", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
|
||||||
|
assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(),
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
} finally {
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Only Admins can create volumes in Ozone. This test uses simple userauth as
|
||||||
|
* backend and hdfs and root are admin users in the simple backend.
|
||||||
|
* <p>
|
||||||
|
* This test tries to create a volume as user bilbo.
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumesWithOutAdminRights() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
try {
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
"bilbo"); // This is not a root user in Simple
|
||||||
|
httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
assertEquals(response.toString(), ErrorTable.ACCESS_DENIED.getHttpCode(),
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
} finally {
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a bunch of volumes in a loop
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumesInLoop() throws IOException {
|
||||||
|
SimpleDateFormat format =
|
||||||
|
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
|
||||||
|
|
||||||
|
for (int x = 0; x < 1000; x++) {
|
||||||
|
HttpClient client = new DefaultHttpClient();
|
||||||
|
String volumeName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
String userName = OzoneUtils.getRequestID().toLowerCase();
|
||||||
|
|
||||||
|
HttpPost httppost = new HttpPost(
|
||||||
|
String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
|
||||||
|
|
||||||
|
httppost.addHeader(Header.OZONE_VERSION_HEADER,
|
||||||
|
Header.OZONE_V1_VERSION_HEADER);
|
||||||
|
httppost.addHeader(HttpHeaders.DATE,
|
||||||
|
format.format(new Date(Time.monotonicNow())));
|
||||||
|
httppost.addHeader(HttpHeaders.AUTHORIZATION,
|
||||||
|
Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
|
||||||
|
OzoneConsts.OZONE_SIMPLE_HDFS_USER);
|
||||||
|
httppost.addHeader(Header.OZONE_USER, userName);
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(httppost);
|
||||||
|
assertEquals(response.toString(), HTTP_CREATED,
|
||||||
|
response.getStatusLine().getStatusCode());
|
||||||
|
client.getConnectionManager().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue