HADOOP-13610. Clean up AliyunOss integration tests. Contributed by Genmao Yu

This commit is contained in:
Kai Zheng 2016-09-18 19:10:48 +08:00
parent dbb28eb1f2
commit a1940464a4
17 changed files with 65 additions and 73 deletions

View File

@ -24,15 +24,13 @@ import org.junit.internal.AssumptionViolatedException;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.Date;
import java.util.Random;
/** /**
* Utility class for OSS Tests. * Utility class for Aliyun OSS Tests.
*/ */
public final class OSSTestUtils { public final class AliyunOSSTestUtils {
private OSSTestUtils() { private AliyunOSSTestUtils() {
} }
/** /**
@ -48,7 +46,7 @@ public final class OSSTestUtils {
public static AliyunOSSFileSystem createTestFileSystem(Configuration conf) public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
throws IOException { throws IOException {
String fsname = conf.getTrimmed( String fsname = conf.getTrimmed(
TestOSSFileSystemContract.TEST_FS_OSS_NAME, ""); TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");
boolean liveTest = !StringUtils.isEmpty(fsname); boolean liveTest = !StringUtils.isEmpty(fsname);
URI testURI = null; URI testURI = null;
@ -59,7 +57,7 @@ public final class OSSTestUtils {
if (!liveTest) { if (!liveTest) {
throw new AssumptionViolatedException("No test filesystem in " throw new AssumptionViolatedException("No test filesystem in "
+ TestOSSFileSystemContract.TEST_FS_OSS_NAME); + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
} }
AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem(); AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
ossfs.initialize(testURI, conf); ossfs.initialize(testURI, conf);
@ -72,9 +70,8 @@ public final class OSSTestUtils {
* @return root test path * @return root test path
*/ */
public static String generateUniqueTestPath() { public static String generateUniqueTestPath() {
Long time = new Date().getTime(); String testUniqueForkId = System.getProperty("test.unique.fork.id");
Random rand = new Random(); return testUniqueForkId == null ? "/test" :
return "/test_" + Long.toString(time) + "_" "/" + testUniqueForkId + "/test";
+ Long.toString(Math.abs(rand.nextLong()));
} }
} }

View File

@ -22,32 +22,28 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystemContractBaseTest; import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
/** /**
* Tests a live OSS system. * Tests a live Aliyun OSS system.
* *
* This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
* TestCase which uses the old Junit3 runner that doesn't ignore assumptions * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
* properly making it impossible to skip the tests if we don't have a valid * properly making it impossible to skip the tests if we don't have a valid
* bucket. * bucket.
*/ */
public class TestOSSFileSystemContract extends FileSystemContractBaseTest { public class TestAliyunOSSFileSystemContract
extends FileSystemContractBaseTest {
protected static final Logger LOG =
LoggerFactory.getLogger(TestOSSFileSystemContract.class);
public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); private static String testRootPath =
AliyunOSSTestUtils.generateUniqueTestPath();
@Override @Override
public void setUp() throws Exception { public void setUp() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
fs = OSSTestUtils.createTestFileSystem(conf); fs = AliyunOSSTestUtils.createTestFileSystem(conf);
super.setUp(); super.setUp();
} }

View File

@ -41,7 +41,7 @@ import static org.junit.Assume.assumeNotNull;
* Test the bridging logic between Hadoop's abstract filesystem and * Test the bridging logic between Hadoop's abstract filesystem and
* Aliyun OSS. * Aliyun OSS.
*/ */
public class TestOSSFileSystemStore { public class TestAliyunOSSFileSystemStore {
private Configuration conf; private Configuration conf;
private AliyunOSSFileSystemStore store; private AliyunOSSFileSystemStore store;
private AliyunOSSFileSystem fs; private AliyunOSSFileSystem fs;
@ -67,8 +67,8 @@ public class TestOSSFileSystemStore {
@BeforeClass @BeforeClass
public static void checkSettings() throws Exception { public static void checkSettings() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
assumeNotNull(conf.get("fs.oss.accessKeyId")); assumeNotNull(conf.get(Constants.ACCESS_KEY));
assumeNotNull(conf.get("fs.oss.accessKeySecret")); assumeNotNull(conf.get(Constants.SECRET_KEY));
assumeNotNull(conf.get("test.fs.oss.name")); assumeNotNull(conf.get("test.fs.oss.name"));
} }

View File

@ -37,14 +37,15 @@ import static org.junit.Assert.assertTrue;
* Tests basic functionality for AliyunOSSInputStream, including seeking and * Tests basic functionality for AliyunOSSInputStream, including seeking and
* reading files. * reading files.
*/ */
public class TestOSSInputStream { public class TestAliyunOSSInputStream {
private FileSystem fs; private FileSystem fs;
protected static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(TestOSSInputStream.class); LoggerFactory.getLogger(TestAliyunOSSInputStream.class);
private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); private static String testRootPath =
AliyunOSSTestUtils.generateUniqueTestPath();
@Rule @Rule
public Timeout testTimeout = new Timeout(30 * 60 * 1000); public Timeout testTimeout = new Timeout(30 * 60 * 1000);
@ -52,7 +53,7 @@ public class TestOSSInputStream {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
fs = OSSTestUtils.createTestFileSystem(conf); fs = AliyunOSSTestUtils.createTestFileSystem(conf);
} }
@After @After

View File

@ -33,9 +33,10 @@ import java.io.IOException;
/** /**
* Tests regular and multi-part upload functionality for AliyunOSSOutputStream. * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
*/ */
public class TestOSSOutputStream { public class TestAliyunOSSOutputStream {
private FileSystem fs; private FileSystem fs;
private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); private static String testRootPath =
AliyunOSSTestUtils.generateUniqueTestPath();
@Rule @Rule
public Timeout testTimeout = new Timeout(30 * 60 * 1000); public Timeout testTimeout = new Timeout(30 * 60 * 1000);
@ -45,7 +46,7 @@ public class TestOSSOutputStream {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024); conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024); conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
fs = OSSTestUtils.createTestFileSystem(conf); fs = AliyunOSSTestUtils.createTestFileSystem(conf);
} }
@After @After
@ -56,7 +57,7 @@ public class TestOSSOutputStream {
} }
protected Path getTestPath() { protected Path getTestPath() {
return new Path(testRootPath + "/testoss"); return new Path(testRootPath + "/test-aliyun-oss");
} }
@Test @Test

View File

@ -21,7 +21,7 @@ package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.InvalidCredentialsException; import com.aliyun.oss.common.auth.InvalidCredentialsException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.aliyun.oss.contract.OSSContract; import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
import org.junit.Test; import org.junit.Test;
@ -38,11 +38,12 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
* should only be used against transient filesystems where you don't care about * should only be used against transient filesystems where you don't care about
* the data. * the data.
*/ */
public class TestOSSTemporaryCredentials extends AbstractFSContractTestBase { public class TestAliyunOSSTemporaryCredentials
extends AbstractFSContractTestBase {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
@Test @Test

View File

@ -20,21 +20,16 @@ package org.apache.hadoop.fs.aliyun.oss.contract;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract; import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
/** /**
* The contract of OSS: only enabled if the test bucket is provided. * The contract of Aliyun OSS: only enabled if the test bucket is provided.
*/ */
public class OSSContract extends AbstractBondedFSContract { public class AliyunOSSContract extends AbstractBondedFSContract {
public static final String CONTRACT_XML = "contract/oss.xml"; public static final String CONTRACT_XML = "contract/aliyun-oss.xml";
public static final String CONTRACT_TEST_OSS_FS_NAME =
"fs.contract.test.fs.oss";
private static String testPath = OSSTestUtils.generateUniqueTestPath(); public AliyunOSSContract(Configuration conf) {
public OSSContract(Configuration conf) {
super(conf); super(conf);
//insert the base features //insert the base features
addConfResource(CONTRACT_XML); addConfResource(CONTRACT_XML);
@ -47,7 +42,8 @@ public class OSSContract extends AbstractBondedFSContract {
@Override @Override
public Path getTestPath() { public Path getTestPath() {
Path path = new Path(testPath); String testUniqueForkId = System.getProperty("test.unique.fork.id");
return path; return testUniqueForkId == null ? super.getTestPath() :
new Path("/" + testUniqueForkId, "test");
} }
} }

View File

@ -23,13 +23,13 @@ import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract creating tests. * Aliyun OSS contract creating tests.
*/ */
public class TestOSSContractCreate extends AbstractContractCreateTest { public class TestAliyunOSSContractCreate extends AbstractContractCreateTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract deleting tests. * Aliyun OSS contract deleting tests.
*/ */
public class TestOSSContractDelete extends AbstractContractDeleteTest { public class TestAliyunOSSContractDelete extends AbstractContractDeleteTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -25,7 +25,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
/** /**
* Contract test suite covering Aliyun OSS integration with DistCp. * Contract test suite covering Aliyun OSS integration with DistCp.
*/ */
public class TestOSSContractDispCp extends AbstractContractDistCpTest { public class TestAliyunOSSContractDispCp extends AbstractContractDistCpTest {
private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
@ -38,7 +38,7 @@ public class TestOSSContractDispCp extends AbstractContractDistCpTest {
} }
@Override @Override
protected OSSContract createContract(Configuration conf) { protected AliyunOSSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -24,12 +24,12 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* Test getFileStatus and related listing operations. * Test getFileStatus and related listing operations.
*/ */
public class TestOSSContractGetFileStatus public class TestAliyunOSSContractGetFileStatus
extends AbstractContractGetFileStatusTest { extends AbstractContractGetFileStatusTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract directory tests. * Aliyun OSS contract directory tests.
*/ */
public class TestOSSContractMkdir extends AbstractContractMkdirTest { public class TestAliyunOSSContractMkdir extends AbstractContractMkdirTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract opening file tests. * Aliyun OSS contract opening file tests.
*/ */
public class TestOSSContractOpen extends AbstractContractOpenTest { public class TestAliyunOSSContractOpen extends AbstractContractOpenTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -23,13 +23,13 @@ import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract renaming tests. * Aliyun OSS contract renaming tests.
*/ */
public class TestOSSContractRename extends AbstractContractRenameTest { public class TestAliyunOSSContractRename extends AbstractContractRenameTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }

View File

@ -29,15 +29,15 @@ import java.io.IOException;
/** /**
* Root dir operations against an Aliyun OSS bucket. * Root dir operations against an Aliyun OSS bucket.
*/ */
public class TestOSSContractRootDir extends public class TestAliyunOSSContractRootDir extends
AbstractContractRootDirectoryTest { AbstractContractRootDirectoryTest {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(TestOSSContractRootDir.class); LoggerFactory.getLogger(TestAliyunOSSContractRootDir.class);
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
@Override @Override

View File

@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
/** /**
* OSS contract seeking tests. * Aliyun OSS contract seeking tests.
*/ */
public class TestOSSContractSeek extends AbstractContractSeekTest { public class TestAliyunOSSContractSeek extends AbstractContractSeekTest {
@Override @Override
protected AbstractFSContract createContract(Configuration conf) { protected AbstractFSContract createContract(Configuration conf) {
return new OSSContract(conf); return new AliyunOSSContract(conf);
} }
} }