Merge -r 1365989:1365990 from trunk to branch. FIXES: HDFS-3113
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1365991 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur
parent
c7bbda5ec9
commit
1849fca985
|
|
@ -1,513 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.http.client;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Writer;
|
||||
import java.net.URL;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.fs.ContentSummary;
|
||||
import org.apache.hadoop.fs.FileChecksum;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.http.server.HttpFSServerWebApp;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.test.HFSTestCase;
|
||||
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
|
||||
import org.apache.hadoop.test.TestDir;
|
||||
import org.apache.hadoop.test.TestDirHelper;
|
||||
import org.apache.hadoop.test.TestHdfs;
|
||||
import org.apache.hadoop.test.TestHdfsHelper;
|
||||
import org.apache.hadoop.test.TestJetty;
|
||||
import org.apache.hadoop.test.TestJettyHelper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.webapp.WebAppContext;
|
||||
|
||||
@RunWith(value = Parameterized.class)
|
||||
public class TestHttpFSFileSystem extends HFSTestCase {
|
||||
|
||||
private void createHttpFSServer() throws Exception {
|
||||
File homeDir = TestDirHelper.getTestDir();
|
||||
Assert.assertTrue(new File(homeDir, "conf").mkdir());
|
||||
Assert.assertTrue(new File(homeDir, "log").mkdir());
|
||||
Assert.assertTrue(new File(homeDir, "temp").mkdir());
|
||||
HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());
|
||||
|
||||
File secretFile = new File(new File(homeDir, "conf"), "secret");
|
||||
Writer w = new FileWriter(secretFile);
|
||||
w.write("secret");
|
||||
w.close();
|
||||
|
||||
//HDFS configuration
|
||||
String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
|
||||
Configuration conf = new Configuration(false);
|
||||
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
|
||||
File hdfsSite = new File(new File(homeDir, "conf"), "hdfs-site.xml");
|
||||
OutputStream os = new FileOutputStream(hdfsSite);
|
||||
conf.writeXml(os);
|
||||
os.close();
|
||||
|
||||
//HTTPFS configuration
|
||||
conf = new Configuration(false);
|
||||
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
|
||||
HadoopUsersConfTestHelper.getHadoopProxyUserGroups());
|
||||
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
|
||||
HadoopUsersConfTestHelper.getHadoopProxyUserHosts());
|
||||
conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath());
|
||||
File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml");
|
||||
os = new FileOutputStream(httpfsSite);
|
||||
conf.writeXml(os);
|
||||
os.close();
|
||||
|
||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||
URL url = cl.getResource("webapp");
|
||||
WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs");
|
||||
Server server = TestJettyHelper.getJettyServer();
|
||||
server.addHandler(context);
|
||||
server.start();
|
||||
}
|
||||
|
||||
protected FileSystem getHttpFileSystem() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set("fs.http.impl", HttpFSFileSystem.class.getName());
|
||||
return FileSystem.get(TestJettyHelper.getJettyURL().toURI(), conf);
|
||||
}
|
||||
|
||||
protected void testGet() throws Exception {
|
||||
FileSystem fs = getHttpFileSystem();
|
||||
Assert.assertNotNull(fs);
|
||||
Assert.assertEquals(fs.getUri(), TestJettyHelper.getJettyURL().toURI());
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testOpen() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
fs.close();
|
||||
fs = getHttpFileSystem();
|
||||
InputStream is = fs.open(new Path(path.toUri().getPath()));
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testCreate(Path path, boolean override) throws Exception {
|
||||
FileSystem fs = getHttpFileSystem();
|
||||
FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
|
||||
OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024,
|
||||
(short) 2, 100 * 1024 * 1024, null);
|
||||
os.write(1);
|
||||
os.close();
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
FileStatus status = fs.getFileStatus(path);
|
||||
Assert.assertEquals(status.getReplication(), 2);
|
||||
Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
|
||||
Assert.assertEquals(status.getPermission(), permission);
|
||||
InputStream is = fs.open(path);
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testCreate() throws Exception {
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
testCreate(path, false);
|
||||
testCreate(path, true);
|
||||
try {
|
||||
testCreate(path, false);
|
||||
Assert.fail();
|
||||
} catch (IOException ex) {
|
||||
|
||||
} catch (Exception ex) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
private void testAppend() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
fs.close();
|
||||
fs = getHttpFileSystem();
|
||||
os = fs.append(new Path(path.toUri().getPath()));
|
||||
os.write(2);
|
||||
os.close();
|
||||
fs.close();
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
InputStream is = fs.open(path);
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
Assert.assertEquals(is.read(), 2);
|
||||
Assert.assertEquals(is.read(), -1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testRename() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo");
|
||||
fs.mkdirs(path);
|
||||
fs.close();
|
||||
fs = getHttpFileSystem();
|
||||
Path oldPath = new Path(path.toUri().getPath());
|
||||
Path newPath = new Path(path.getParent(), "bar");
|
||||
fs.rename(oldPath, newPath);
|
||||
fs.close();
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Assert.assertFalse(fs.exists(oldPath));
|
||||
Assert.assertTrue(fs.exists(newPath));
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testDelete() throws Exception {
|
||||
Path foo = new Path(TestHdfsHelper.getHdfsTestDir(), "foo");
|
||||
Path bar = new Path(TestHdfsHelper.getHdfsTestDir(), "bar");
|
||||
Path foe = new Path(TestHdfsHelper.getHdfsTestDir(), "foe");
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
fs.mkdirs(foo);
|
||||
fs.mkdirs(new Path(bar, "a"));
|
||||
fs.mkdirs(foe);
|
||||
|
||||
FileSystem hoopFs = getHttpFileSystem();
|
||||
Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
|
||||
Assert.assertFalse(fs.exists(foo));
|
||||
try {
|
||||
hoopFs.delete(new Path(bar.toUri().getPath()), false);
|
||||
Assert.fail();
|
||||
} catch (IOException ex) {
|
||||
} catch (Exception ex) {
|
||||
Assert.fail();
|
||||
}
|
||||
Assert.assertTrue(fs.exists(bar));
|
||||
Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
|
||||
Assert.assertFalse(fs.exists(bar));
|
||||
|
||||
Assert.assertTrue(fs.exists(foe));
|
||||
Assert.assertTrue(hoopFs.delete(foe, true));
|
||||
Assert.assertFalse(fs.exists(foe));
|
||||
|
||||
hoopFs.close();
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testListStatus() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath()));
|
||||
fs.close();
|
||||
|
||||
Assert.assertEquals(status2.getPermission(), status1.getPermission());
|
||||
Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath());
|
||||
Assert.assertEquals(status2.getReplication(), status1.getReplication());
|
||||
Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize());
|
||||
Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime());
|
||||
Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime());
|
||||
Assert.assertEquals(status2.getOwner(), status1.getOwner());
|
||||
Assert.assertEquals(status2.getGroup(), status1.getGroup());
|
||||
Assert.assertEquals(status2.getLen(), status1.getLen());
|
||||
|
||||
FileStatus[] stati = fs.listStatus(path.getParent());
|
||||
Assert.assertEquals(stati.length, 1);
|
||||
Assert.assertEquals(stati[0].getPath().getName(), path.getName());
|
||||
}
|
||||
|
||||
private void testWorkingdirectory() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path workingDir = fs.getWorkingDirectory();
|
||||
fs.close();
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
Path hoopWorkingDir = fs.getWorkingDirectory();
|
||||
fs.close();
|
||||
Assert.assertEquals(hoopWorkingDir.toUri().getPath(), workingDir.toUri().getPath());
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
fs.setWorkingDirectory(new Path("/tmp"));
|
||||
workingDir = fs.getWorkingDirectory();
|
||||
fs.close();
|
||||
Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath());
|
||||
}
|
||||
|
||||
private void testMkdirs() throws Exception {
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo");
|
||||
FileSystem fs = getHttpFileSystem();
|
||||
fs.mkdirs(path);
|
||||
fs.close();
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Assert.assertTrue(fs.exists(path));
|
||||
fs.close();
|
||||
}
|
||||
|
||||
private void testSetTimes() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
long at = status1.getAccessTime();
|
||||
long mt = status1.getModificationTime();
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
fs.setTimes(path, mt + 10, at + 20);
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
long atNew = status1.getAccessTime();
|
||||
long mtNew = status1.getModificationTime();
|
||||
Assert.assertEquals(mtNew, mt + 10);
|
||||
Assert.assertEquals(atNew, at + 20);
|
||||
}
|
||||
|
||||
private void testSetPermission() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foodir");
|
||||
fs.mkdirs(path);
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
FsPermission permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
|
||||
fs.setPermission(path, permission1);
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
FsPermission permission2 = status1.getPermission();
|
||||
Assert.assertEquals(permission2, permission1);
|
||||
|
||||
//sticky bit
|
||||
fs = getHttpFileSystem();
|
||||
permission1 = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE, true);
|
||||
fs.setPermission(path, permission1);
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
permission2 = status1.getPermission();
|
||||
Assert.assertTrue(permission2.getStickyBit());
|
||||
Assert.assertEquals(permission2, permission1);
|
||||
}
|
||||
|
||||
private void testSetOwner() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
fs.close();
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
String user = HadoopUsersConfTestHelper.getHadoopUsers()[1];
|
||||
String group = HadoopUsersConfTestHelper.getHadoopUserGroups(user)[0];
|
||||
fs.setOwner(path, user, group);
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(status1.getOwner(), user);
|
||||
Assert.assertEquals(status1.getGroup(), group);
|
||||
}
|
||||
|
||||
private void testSetReplication() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
fs.close();
|
||||
fs.setReplication(path, (short) 2);
|
||||
|
||||
fs = getHttpFileSystem();
|
||||
fs.setReplication(path, (short) 1);
|
||||
fs.close();
|
||||
|
||||
fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(status1.getReplication(), (short) 1);
|
||||
}
|
||||
|
||||
private void testChecksum() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
FileChecksum hdfsChecksum = fs.getFileChecksum(path);
|
||||
fs.close();
|
||||
fs = getHttpFileSystem();
|
||||
FileChecksum httpChecksum = fs.getFileChecksum(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(httpChecksum.getAlgorithmName(), hdfsChecksum.getAlgorithmName());
|
||||
Assert.assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
|
||||
Assert.assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
|
||||
}
|
||||
|
||||
private void testContentSummary() throws Exception {
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
Path path = new Path(TestHdfsHelper.getHdfsTestDir(), "foo.txt");
|
||||
OutputStream os = fs.create(path);
|
||||
os.write(1);
|
||||
os.close();
|
||||
ContentSummary hdfsContentSummary = fs.getContentSummary(path);
|
||||
fs.close();
|
||||
fs = getHttpFileSystem();
|
||||
ContentSummary httpContentSummary = fs.getContentSummary(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(httpContentSummary.getDirectoryCount(), hdfsContentSummary.getDirectoryCount());
|
||||
Assert.assertEquals(httpContentSummary.getFileCount(), hdfsContentSummary.getFileCount());
|
||||
Assert.assertEquals(httpContentSummary.getLength(), hdfsContentSummary.getLength());
|
||||
Assert.assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota());
|
||||
Assert.assertEquals(httpContentSummary.getSpaceConsumed(), hdfsContentSummary.getSpaceConsumed());
|
||||
Assert.assertEquals(httpContentSummary.getSpaceQuota(), hdfsContentSummary.getSpaceQuota());
|
||||
}
|
||||
|
||||
protected enum Operation {
|
||||
GET, OPEN, CREATE, APPEND, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS,
|
||||
SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY
|
||||
}
|
||||
|
||||
private void operation(Operation op) throws Exception {
|
||||
switch (op) {
|
||||
case GET:
|
||||
testGet();
|
||||
break;
|
||||
case OPEN:
|
||||
testOpen();
|
||||
break;
|
||||
case CREATE:
|
||||
testCreate();
|
||||
break;
|
||||
case APPEND:
|
||||
testAppend();
|
||||
break;
|
||||
case RENAME:
|
||||
testRename();
|
||||
break;
|
||||
case DELETE:
|
||||
testDelete();
|
||||
break;
|
||||
case LIST_STATUS:
|
||||
testListStatus();
|
||||
break;
|
||||
case WORKING_DIRECTORY:
|
||||
testWorkingdirectory();
|
||||
break;
|
||||
case MKDIRS:
|
||||
testMkdirs();
|
||||
break;
|
||||
case SET_TIMES:
|
||||
testSetTimes();
|
||||
break;
|
||||
case SET_PERMISSION:
|
||||
testSetPermission();
|
||||
break;
|
||||
case SET_OWNER:
|
||||
testSetOwner();
|
||||
break;
|
||||
case SET_REPLICATION:
|
||||
testSetReplication();
|
||||
break;
|
||||
case CHECKSUM:
|
||||
testChecksum();
|
||||
break;
|
||||
case CONTENT_SUMMARY:
|
||||
testContentSummary();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@Parameterized.Parameters
|
||||
public static Collection operations() {
|
||||
Object[][] ops = new Object[Operation.values().length][];
|
||||
for (int i = 0; i < Operation.values().length; i++) {
|
||||
ops[i] = new Object[]{Operation.values()[i]};
|
||||
}
|
||||
return Arrays.asList(ops);
|
||||
// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}});
|
||||
}
|
||||
|
||||
private Operation operation;
|
||||
|
||||
public TestHttpFSFileSystem(Operation operation) {
|
||||
this.operation = operation;
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void testOperation() throws Exception {
|
||||
createHttpFSServer();
|
||||
operation(operation);
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void testOperationDoAs() throws Exception {
|
||||
createHttpFSServer();
|
||||
UserGroupInformation ugi = UserGroupInformation.createProxyUser(HadoopUsersConfTestHelper.getHadoopUsers()[0],
|
||||
UserGroupInformation.getCurrentUser());
|
||||
ugi.doAs(new PrivilegedExceptionAction<Void>() {
|
||||
@Override
|
||||
public Void run() throws Exception {
|
||||
operation(operation);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,236 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.http.server;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Writer;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.lib.server.Service;
|
||||
import org.apache.hadoop.lib.server.ServiceException;
|
||||
import org.apache.hadoop.lib.service.Groups;
|
||||
import org.apache.hadoop.test.HFSTestCase;
|
||||
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
|
||||
import org.apache.hadoop.test.TestDir;
|
||||
import org.apache.hadoop.test.TestDirHelper;
|
||||
import org.apache.hadoop.test.TestHdfs;
|
||||
import org.apache.hadoop.test.TestHdfsHelper;
|
||||
import org.apache.hadoop.test.TestJetty;
|
||||
import org.apache.hadoop.test.TestJettyHelper;
|
||||
import org.junit.Test;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.webapp.WebAppContext;
|
||||
|
||||
public class TestHttpFSServer extends HFSTestCase {
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
public void server() throws Exception {
|
||||
String dir = TestDirHelper.getTestDir().getAbsolutePath();
|
||||
|
||||
Configuration httpfsConf = new Configuration(false);
|
||||
HttpFSServerWebApp server = new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf);
|
||||
server.init();
|
||||
server.destroy();
|
||||
}
|
||||
|
||||
public static class MockGroups implements Service,Groups {
|
||||
|
||||
@Override
|
||||
public void init(org.apache.hadoop.lib.server.Server server) throws ServiceException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void postInit() throws ServiceException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class[] getServiceDependencies() {
|
||||
return new Class[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class getInterface() {
|
||||
return Groups.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serverStatusChange(org.apache.hadoop.lib.server.Server.Status oldStatus,
|
||||
org.apache.hadoop.lib.server.Server.Status newStatus) throws ServiceException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getGroups(String user) throws IOException {
|
||||
return Arrays.asList(HadoopUsersConfTestHelper.getHadoopUserGroups(user));
|
||||
}
|
||||
|
||||
}
|
||||
private void createHttpFSServer() throws Exception {
|
||||
File homeDir = TestDirHelper.getTestDir();
|
||||
assertTrue(new File(homeDir, "conf").mkdir());
|
||||
assertTrue(new File(homeDir, "log").mkdir());
|
||||
assertTrue(new File(homeDir, "temp").mkdir());
|
||||
HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());
|
||||
|
||||
File secretFile = new File(new File(homeDir, "conf"), "secret");
|
||||
Writer w = new FileWriter(secretFile);
|
||||
w.write("secret");
|
||||
w.close();
|
||||
|
||||
//HDFS configuration
|
||||
File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
|
||||
hadoopConfDir.mkdirs();
|
||||
String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
|
||||
Configuration conf = new Configuration(false);
|
||||
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
|
||||
File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml");
|
||||
OutputStream os = new FileOutputStream(hdfsSite);
|
||||
conf.writeXml(os);
|
||||
os.close();
|
||||
|
||||
//HTTPFS configuration
|
||||
conf = new Configuration(false);
|
||||
conf.set("httpfs.services.ext", MockGroups.class.getName());
|
||||
conf.set("httpfs.admin.group", HadoopUsersConfTestHelper.
|
||||
getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]);
|
||||
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".groups",
|
||||
HadoopUsersConfTestHelper.getHadoopProxyUserGroups());
|
||||
conf.set("httpfs.proxyuser." + HadoopUsersConfTestHelper.getHadoopProxyUser() + ".hosts",
|
||||
HadoopUsersConfTestHelper.getHadoopProxyUserHosts());
|
||||
conf.set("httpfs.authentication.signature.secret.file", secretFile.getAbsolutePath());
|
||||
File httpfsSite = new File(new File(homeDir, "conf"), "httpfs-site.xml");
|
||||
os = new FileOutputStream(httpfsSite);
|
||||
conf.writeXml(os);
|
||||
os.close();
|
||||
|
||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||
URL url = cl.getResource("webapp");
|
||||
WebAppContext context = new WebAppContext(url.getPath(), "/webhdfs");
|
||||
Server server = TestJettyHelper.getJettyServer();
|
||||
server.addHandler(context);
|
||||
server.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void instrumentation() throws Exception {
|
||||
createHttpFSServer();
|
||||
|
||||
URL url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody"));
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED);
|
||||
|
||||
url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation",
|
||||
HadoopUsersConfTestHelper.getHadoopUsers()[0]));
|
||||
conn = (HttpURLConnection) url.openConnection();
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
|
||||
String line = reader.readLine();
|
||||
reader.close();
|
||||
assertTrue(line.contains("\"counters\":{"));
|
||||
|
||||
url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation",
|
||||
HadoopUsersConfTestHelper.getHadoopUsers()[0]));
|
||||
conn = (HttpURLConnection) url.openConnection();
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void testHdfsAccess() throws Exception {
|
||||
createHttpFSServer();
|
||||
|
||||
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
|
||||
URL url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus", user));
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
|
||||
reader.readLine();
|
||||
reader.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void testGlobFilter() throws Exception {
|
||||
createHttpFSServer();
|
||||
|
||||
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
|
||||
fs.mkdirs(new Path("/tmp"));
|
||||
fs.create(new Path("/tmp/foo.txt")).close();
|
||||
|
||||
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
|
||||
URL url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user));
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
|
||||
reader.readLine();
|
||||
reader.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestDir
|
||||
@TestJetty
|
||||
@TestHdfs
|
||||
public void testPutNoOperation() throws Exception {
|
||||
createHttpFSServer();
|
||||
|
||||
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
|
||||
URL url = new URL(TestJettyHelper.getJettyURL(),
|
||||
MessageFormat.format("/webhdfs/v1/foo?user.name={0}", user));
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setDoInput(true);
|
||||
conn.setDoOutput(true);
|
||||
conn.setRequestMethod("PUT");
|
||||
assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.test;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.URL;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.MethodRule;
|
||||
import org.junit.runners.model.FrameworkMethod;
|
||||
import org.junit.runners.model.Statement;
|
||||
import org.mortbay.jetty.Server;
|
||||
|
||||
public class TestJettyHelper implements MethodRule {
|
||||
|
||||
@Test
|
||||
public void dummy() {
|
||||
}
|
||||
|
||||
private static ThreadLocal<Server> TEST_SERVLET_TL = new InheritableThreadLocal<Server>();
|
||||
|
||||
@Override
|
||||
public Statement apply(final Statement statement, final FrameworkMethod frameworkMethod, final Object o) {
|
||||
return new Statement() {
|
||||
@Override
|
||||
public void evaluate() throws Throwable {
|
||||
Server server = null;
|
||||
TestJetty testJetty = frameworkMethod.getAnnotation(TestJetty.class);
|
||||
if (testJetty != null) {
|
||||
server = createJettyServer();
|
||||
}
|
||||
try {
|
||||
TEST_SERVLET_TL.set(server);
|
||||
statement.evaluate();
|
||||
} finally {
|
||||
TEST_SERVLET_TL.remove();
|
||||
if (server != null && server.isRunning()) {
|
||||
try {
|
||||
server.stop();
|
||||
} catch (Exception ex) {
|
||||
throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private Server createJettyServer() {
|
||||
try {
|
||||
|
||||
String host = InetAddress.getLocalHost().getHostName();
|
||||
ServerSocket ss = new ServerSocket(0);
|
||||
int port = ss.getLocalPort();
|
||||
ss.close();
|
||||
Server server = new Server(0);
|
||||
server.getConnectors()[0].setHost(host);
|
||||
server.getConnectors()[0].setPort(port);
|
||||
return server;
|
||||
} catch (Exception ex) {
|
||||
throw new RuntimeException("Could not stop embedded servlet container, " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a Jetty server ready to be configured and the started. This server
|
||||
* is only available when the test method has been annotated with
|
||||
* {@link TestJetty}. Refer to {@link HTestCase} header for details.
|
||||
* <p/>
|
||||
* Once configured, the Jetty server should be started. The server will be
|
||||
* automatically stopped when the test method ends.
|
||||
*
|
||||
* @return a Jetty server ready to be configured and the started.
|
||||
*/
|
||||
public static Server getJettyServer() {
|
||||
Server server = TEST_SERVLET_TL.get();
|
||||
if (server == null) {
|
||||
throw new IllegalStateException("This test does not use @TestJetty");
|
||||
}
|
||||
return server;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the base URL (SCHEMA://HOST:PORT) of the test Jetty server
|
||||
* (see {@link #getJettyServer()}) once started.
|
||||
*
|
||||
* @return the base URL (SCHEMA://HOST:PORT) of the test Jetty server.
|
||||
*/
|
||||
public static URL getJettyURL() {
|
||||
Server server = TEST_SERVLET_TL.get();
|
||||
if (server == null) {
|
||||
throw new IllegalStateException("This test does not use @TestJetty");
|
||||
}
|
||||
try {
|
||||
return new URL("http://" + server.getConnectors()[0].getHost() + ":" + server.getConnectors()[0].getPort());
|
||||
} catch (MalformedURLException ex) {
|
||||
throw new RuntimeException("It should never happen, " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue