From a5d5342228050a778b20e95adf7885bdba39985d Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Thu, 4 Aug 2016 21:21:10 +0800 Subject: [PATCH 01/13] HADOOP-12756. Incorporate Aliyun OSS file system implementation. Contributed by Mingfei Shi and Lin Zhou --- .gitignore | 2 + hadoop-project/pom.xml | 22 + .../dev-support/findbugs-exclude.xml | 18 + hadoop-tools/hadoop-aliyun/pom.xml | 133 +++ .../fs/aliyun/oss/AliyunOSSFileSystem.java | 847 ++++++++++++++++++ .../fs/aliyun/oss/AliyunOSSInputStream.java | 268 ++++++ .../fs/aliyun/oss/AliyunOSSOutputStream.java | 219 +++++ .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 151 ++++ .../hadoop/fs/aliyun/oss/Constants.java | 110 +++ .../hadoop/fs/aliyun/oss/package-info.java | 22 + .../hadoop/fs/aliyun/oss/OSSTestUtils.java | 80 ++ .../aliyun/oss/TestOSSFileSystemContract.java | 253 ++++++ .../fs/aliyun/oss/TestOSSInputStream.java | 141 +++ .../fs/aliyun/oss/TestOSSOutputStream.java | 71 ++ .../fs/aliyun/oss/contract/OSSContract.java | 54 ++ .../oss/contract/TestOSSContractCreate.java | 41 + .../oss/contract/TestOSSContractDelete.java | 34 + .../oss/contract/TestOSSContractMkdir.java | 34 + .../oss/contract/TestOSSContractOpen.java | 34 + .../oss/contract/TestOSSContractRename.java | 35 + .../oss/contract/TestOSSContractSeek.java | 34 + .../src/test/resources/contract/oss.xml | 105 +++ .../src/test/resources/core-site.xml | 46 + .../src/test/resources/log4j.properties | 23 + hadoop-tools/hadoop-tools-dist/pom.xml | 6 + hadoop-tools/pom.xml | 1 + 26 files changed, 2784 insertions(+) create mode 100644 hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml create mode 100644 hadoop-tools/hadoop-aliyun/pom.xml create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml create mode 100644 hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml create mode 100644 hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties diff --git a/.gitignore b/.gitignore index a5d69d094c8..194862b3114 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml patchprocess/ +hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml +hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 65e9672ce42..29c2760d7d1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -436,6 +436,12 @@ ${project.version} + + org.apache.hadoop + hadoop-aliyun + ${project.version} + + org.apache.hadoop hadoop-kms @@ -1004,6 +1010,22 @@ 4.2.0 + + com.aliyun.oss + aliyun-sdk-oss + 2.2.1 + + + org.apache.httpcomponents + httpclient + + + commons-beanutils + commons-beanutils + + + + xerces xercesImpl diff --git a/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml new file mode 100644 index 00000000000..40d78d0cd6c --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml @@ -0,0 +1,18 @@ + + + diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml new file mode 100644 index 00000000000..c87d13f24cc --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/pom.xml @@ -0,0 +1,133 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-aliyun + Apache Hadoop Aliyun OSS support + jar + + + UTF-8 + true + + + + + tests-off + + + src/test/resources/auth-keys.xml + + + + true + + + + tests-on + + + src/test/resources/auth-keys.xml + + + + false + + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + + true + true + ${basedir}/dev-support/findbugs-exclude.xml + + Max + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + false + false + + + + org.apache.maven.plugins + maven-surefire-plugin + + 3600 + + + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + + + + + + + junit + junit + test + + + + com.aliyun.oss + aliyun-sdk-oss + compile + + + + org.apache.hadoop + hadoop-common + compile + + + + org.apache.hadoop + hadoop-common + test + test-jar + + + + diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java new file mode 100644 index 00000000000..30ddf8c6a66 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -0,0 +1,847 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.UserInfo; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.util.Progressable; + +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.comm.Protocol; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CannedAccessControlList; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadResult; +import com.aliyun.oss.model.CopyObjectResult; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PartETag; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; + +/** + * Implementation of {@link FileSystem} for + * Aliyun OSS, used to access OSS blob system in a filesystem style. + */ +public class AliyunOSSFileSystem extends FileSystem { + + private URI uri; + private Path workingDir; + private OSSClient ossClient; + private String bucketName; + private long uploadPartSize; + private long multipartThreshold; + private int maxKeys; + private String serverSideEncryptionAlgorithm; + + @Override + public FSDataOutputStream append(Path path, int bufferSize, + Progressable progress) throws IOException { + throw new IOException("Append is not supported!"); + } + + @Override + public void close() throws IOException { + try { + if (ossClient != null) { + ossClient.shutdown(); + } + } finally { + super.close(); + } + } + + @Override + public FSDataOutputStream create(Path path, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + String key = pathToKey(path); + + if (!overwrite && exists(path)) { + throw new FileAlreadyExistsException(path + " already exists"); + } + + return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(), + ossClient, bucketName, key, progress, statistics, + serverSideEncryptionAlgorithm), (Statistics)(null)); + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + FileStatus status; + try { + status = getFileStatus(path); + } catch (FileNotFoundException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Couldn't delete " + path + ": Does not exist!"); + } + return false; + } + + String key = pathToKey(status.getPath()); + if (status.isDirectory()) { + if (!key.endsWith("/")) { + key += "/"; + } + if (!recursive) { + FileStatus[] statuses = listStatus(status.getPath()); + // Check whether it is an empty directory or not + if (statuses.length > 0) { + throw new IOException("Cannot remove directory" + path + + ": It is not empty!"); + } else { + // Delete empty directory without '-r' + ossClient.deleteObject(bucketName, key); + statistics.incrementWriteOps(1); + } + } else { + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(key); + listRequest.setMaxKeys(maxKeys); + + while (true) { + ObjectListing objects = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + List keysToDelete = new ArrayList(); + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + keysToDelete.add(objectSummary.getKey()); + } + DeleteObjectsRequest deleteRequest = + new DeleteObjectsRequest(bucketName); + deleteRequest.setKeys(keysToDelete); + ossClient.deleteObjects(deleteRequest); + statistics.incrementWriteOps(1); + if (objects.isTruncated()) { + listRequest.setMarker(objects.getNextMarker()); + } else { + break; + } + } + } + } else { + ossClient.deleteObject(bucketName, key); + statistics.incrementWriteOps(1); + } + //TODO: optimize logic here + try { + Path pPath = status.getPath().getParent(); + FileStatus pStatus = getFileStatus(pPath); + if (pStatus.isDirectory()) { + return true; + } else { + throw new IOException("Path " + pPath + + " is assumed to be a directory!"); + } + } catch (FileNotFoundException fnfe) { + // Make sure the parent directory exists + return mkdir(bucketName, pathToKey(status.getPath().getParent())); + } + } + + @Override + public FileStatus getFileStatus(Path path) throws IOException { + Path qualifiedPath = path.makeQualified(uri, workingDir); + String key = pathToKey(qualifiedPath); + + // Root always exists + if (key.length() == 0) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } + + ObjectMetadata meta = getObjectMetadata(key); + // If key not found and key does not end with "/" + if (meta == null && !key.endsWith("/")) { + // Case: dir + "/" + key += "/"; + meta = getObjectMetadata(key); + } + if (meta == null) { + // Case: dir + "/" + file + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(key); + listRequest.setDelimiter("/"); + listRequest.setMaxKeys(1); + + ObjectListing listing = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + if (!listing.getObjectSummaries().isEmpty() || + !listing.getCommonPrefixes().isEmpty()) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } else { + throw new FileNotFoundException(path + ": No such file or directory!"); + } + } else if (objectRepresentsDirectory(key, meta.getContentLength())) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } else { + return new FileStatus(meta.getContentLength(), false, 1, + getDefaultBlockSize(path), meta.getLastModified().getTime(), + qualifiedPath); + } + } + + /** + * Return object metadata given object key. + * + * @param key object key + * @return return null if key does not exist + */ + private ObjectMetadata getObjectMetadata(String key) { + try { + return ossClient.getObjectMetadata(bucketName, key); + } catch (OSSException osse) { + return null; + } finally { + statistics.incrementReadOps(1); + } + } + + @Override + public String getScheme() { + return "oss"; + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public Path getWorkingDirectory() { + return workingDir; + } + + @Deprecated + public long getDefaultBlockSize() { + return getConf().getLong(FS_OSS_BLOCK_SIZE_KEY, FS_OSS_BLOCK_SIZE_DEFAULT); + } + + @Override + public String getCanonicalServiceName() { + // Does not support Token + return null; + } + + /** + * Initialize new FileSystem. + * + * @param name the uri of the file system, including host, port, etc. + * + * @param conf configuration of the file system + * @throws IOException IO problems + */ + public void initialize(URI name, Configuration conf) throws IOException { + super.initialize(name, conf); + + uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority()); + workingDir = + new Path("/user", + System.getProperty("user.name")).makeQualified(uri, null); + + bucketName = name.getHost(); + + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY, + MAXIMUM_CONNECTIONS_DEFAULT)); + boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY, + SECURE_CONNECTIONS_DEFAULT); + clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); + clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY, + MAX_ERROR_RETRIES_DEFAULT)); + clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY, + ESTABLISH_TIMEOUT_DEFAULT)); + clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY, + SOCKET_TIMEOUT_DEFAULT)); + + String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); + int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); + if (!proxyHost.isEmpty()) { + clientConf.setProxyHost(proxyHost); + if (proxyPort >= 0) { + clientConf.setProxyPort(proxyPort); + } else { + if (secureConnections) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + clientConf.setProxyPort(443); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + clientConf.setProxyPort(80); + } + } + String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY); + String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " + + PROXY_PASSWORD_KEY + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + clientConf.setProxyUsername(proxyUsername); + clientConf.setProxyPassword(proxyPassword); + clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY)); + clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY)); + } else if (proxyPort >= 0) { + String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " + + PROXY_HOST_KEY; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + + String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); + ossClient = + new OSSClient(endPoint, getCredentialsProvider(name, conf), clientConf); + + maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, + MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); + + if (uploadPartSize < 5 * 1024 * 1024) { + LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB"); + uploadPartSize = 5 * 1024 * 1024; + } + + if (multipartThreshold < 5 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB"); + multipartThreshold = 5 * 1024 * 1024; + } + + if (multipartThreshold > 1024 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB"); + multipartThreshold = 1024 * 1024 * 1024; + } + + String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); + if (!cannedACLName.isEmpty()) { + CannedAccessControlList cannedACL = + CannedAccessControlList.valueOf(cannedACLName); + ossClient.setBucketAcl(bucketName, cannedACL); + } + + serverSideEncryptionAlgorithm = + conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, ""); + + setConf(conf); + } + + /** + * Create the default credential provider, or load in one explicitly + * identified in the configuration. + * @param name the uri of the file system + * @param conf configuration + * @return a credential provider + * @throws IOException on any problem. Class construction issues may be + * nested inside the IOE. + */ + private CredentialsProvider getCredentialsProvider(URI name, + Configuration conf) throws IOException { + CredentialsProvider credentials; + + String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); + if (StringUtils.isEmpty(className)) { + Configuration newConf = + ProviderUtils.excludeIncompatibleCredentialProviders(conf, + AliyunOSSFileSystem.class); + String accessKey = + AliyunOSSUtils.getPassword(newConf, ACCESS_KEY, + UserInfo.EMPTY.getUser()); + String secretKey = + AliyunOSSUtils.getPassword(newConf, SECRET_KEY, + UserInfo.EMPTY.getPassword()); + credentials = + new DefaultCredentialProvider( + new DefaultCredentials(accessKey, secretKey)); + + } else { + try { + LOG.debug("Credential provider class is:" + className); + Class credClass = Class.forName(className); + try { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor( + URI.class, Configuration.class).newInstance(this.uri, conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } + } catch (ClassNotFoundException e) { + throw new IOException(className + " not found.", e); + } catch (NoSuchMethodException | SecurityException e) { + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), e); + } catch (ReflectiveOperationException | IllegalArgumentException e) { + throw new IOException(className + " instantiation exception.", e); + } + } + + return credentials; + } + + /** + * Check if OSS object represents a directory. + * + * @param name object key + * @param size object content length + * @return true if object represents a directory + */ + private boolean objectRepresentsDirectory(final String name, + final long size) { + return !name.isEmpty() && name.endsWith("/") && size == 0L; + } + + /** + * Turns a path (relative or otherwise) into an OSS key. + * + * @param path the path of the file + * @return the key of the object that represent the file + */ + private String pathToKey(Path path) { + if (!path.isAbsolute()) { + path = new Path(workingDir, path); + } + + if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) { + return ""; + } + + return path.toUri().getPath().substring(1); + } + + private Path keyToPath(String key) { + return new Path("/" + key); + } + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + String key = pathToKey(path); + if (LOG.isDebugEnabled()) { + LOG.debug("List status for path: " + path); + } + + final List result = new ArrayList(); + final FileStatus fileStatus = getFileStatus(path); + + if (fileStatus.isDirectory()) { + if (!key.endsWith("/")) { + key = key + "/"; + } + + ListObjectsRequest listObjectsRequest = + new ListObjectsRequest(bucketName); + listObjectsRequest.setPrefix(key); + listObjectsRequest.setDelimiter("/"); + listObjectsRequest.setMaxKeys(maxKeys); + + if (LOG.isDebugEnabled()) { + LOG.debug("listStatus: doing listObjects for directory " + key); + } + + while (true) { + ObjectListing objects = ossClient.listObjects(listObjectsRequest); + statistics.incrementReadOps(1); + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + Path keyPath = keyToPath(objectSummary.getKey()) + .makeQualified(uri, workingDir); + if (keyPath.equals(path)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring: " + keyPath); + } + continue; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: fi: " + keyPath); + } + result.add(new FileStatus(objectSummary.getSize(), false, 1, + getDefaultBlockSize(keyPath), + objectSummary.getLastModified().getTime(), keyPath)); + } + } + + for (String prefix : objects.getCommonPrefixes()) { + Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir); + if (keyPath.equals(path)) { + continue; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: rd: " + keyPath); + } + result.add(new FileStatus(0, true, 1, 0, 0, keyPath)); + } + } + + if (objects.isTruncated()) { + if (LOG.isDebugEnabled()) { + LOG.debug("listStatus: list truncated - getting next batch"); + } + listObjectsRequest.setMarker(objects.getNextMarker()); + statistics.incrementReadOps(1); + } else { + break; + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: rd (not a dir): " + path); + } + result.add(fileStatus); + } + + return result.toArray(new FileStatus[result.size()]); + } + + /** + * Used to create an empty file that represents an empty directory. + * + * @param bucketName the bucket this directory belongs to + * @param objectName directory path + * @return true if directory successfully created + * @throws IOException + */ + private boolean mkdir(final String bucket, final String objectName) + throws IOException { + String dirName = objectName; + ObjectMetadata dirMeta = new ObjectMetadata(); + byte[] buffer = new byte[0]; + ByteArrayInputStream in = new ByteArrayInputStream(buffer); + dirMeta.setContentLength(0); + if (!objectName.endsWith("/")) { + dirName += "/"; + } + try { + ossClient.putObject(bucket, dirName, in, dirMeta); + return true; + } finally { + in.close(); + } + } + + @Override + public boolean mkdirs(Path path, FsPermission permission) + throws IOException { + try { + FileStatus fileStatus = getFileStatus(path); + + if (fileStatus.isDirectory()) { + return true; + } else { + throw new FileAlreadyExistsException("Path is a file: " + path); + } + } catch (FileNotFoundException e) { + validatePath(path); + String key = pathToKey(path); + return mkdir(bucketName, key); + } + } + + /** + * Check whether the path is a valid path. + * + * @param path the path to be checked + * @throws IOException + */ + private void validatePath(Path path) throws IOException { + Path fPart = path.getParent(); + do { + try { + FileStatus fileStatus = getFileStatus(fPart); + if (fileStatus.isDirectory()) { + // If path exists and a directory, exit + break; + } else { + throw new FileAlreadyExistsException(String.format( + "Can't make directory for path '%s', it is a file.", fPart)); + } + } catch (FileNotFoundException fnfe) { + } + fPart = fPart.getParent(); + } while (fPart != null); + } + + @Override + public FSDataInputStream open(Path path, int bufferSize) throws IOException { + final FileStatus fileStatus = getFileStatus(path); + if (fileStatus.isDirectory()) { + throw new FileNotFoundException("Can't open " + path + + " because it is a directory"); + } + + return new FSDataInputStream(new AliyunOSSInputStream(getConf(), ossClient, + bucketName, pathToKey(path), fileStatus.getLen(), statistics)); + } + + @Override + public boolean rename(Path srcPath, Path dstPath) throws IOException { + if (srcPath.isRoot()) { + // Cannot rename root of file system + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot rename the root of a filesystem"); + } + return false; + } + Path parent = dstPath.getParent(); + while (parent != null && !srcPath.equals(parent)) { + parent = parent.getParent(); + } + if (parent != null) { + return false; + } + FileStatus srcStatus = getFileStatus(srcPath); + FileStatus dstStatus; + try { + dstStatus = getFileStatus(dstPath); + } catch (FileNotFoundException fnde) { + dstStatus = null; + } + if (dstStatus == null) { + // If dst doesn't exist, check whether dst dir exists or not + dstStatus = getFileStatus(dstPath.getParent()); + if (!dstStatus.isDirectory()) { + throw new IOException(String.format( + "Failed to rename %s to %s, %s is a file", srcPath, dstPath, + dstPath.getParent())); + } + } else { + if (srcStatus.getPath().equals(dstStatus.getPath())) { + return !srcStatus.isDirectory(); + } else if (dstStatus.isDirectory()) { + // If dst is a directory + dstPath = new Path(dstPath, srcPath.getName()); + FileStatus[] statuses; + try { + statuses = listStatus(dstPath); + } catch (FileNotFoundException fnde) { + statuses = null; + } + if (statuses != null && statuses.length > 0) { + // If dst exists and not a directory / not empty + throw new FileAlreadyExistsException(String.format( + "Failed to rename %s to %s, file already exists or not empty!", + srcPath, dstPath)); + } + } else { + // If dst is not a directory + throw new FileAlreadyExistsException(String.format( + "Failed to rename %s to %s, file already exists!", srcPath, + dstPath)); + } + } + if (srcStatus.isDirectory()) { + copyDirectory(srcPath, dstPath); + } else { + copyFile(srcPath, dstPath); + } + if (srcPath.equals(dstPath)) { + return true; + } else { + return delete(srcPath, true); + } + } + + /** + * Copy file from source path to destination path. + * (the caller should make sure srcPath is a file and dstPath is valid.) + * + * @param srcPath source path + * @param dstPath destination path + * @return true if successfully copied + */ + private boolean copyFile(Path srcPath, Path dstPath) { + String srcKey = pathToKey(srcPath); + String dstKey = pathToKey(dstPath); + return copyFile(srcKey, dstKey); + } + + /** + * Copy an object from source key to destination key. + * + * @param srcKey source key + * @param dstKey destination key + * @return true if successfully copied + */ + private boolean copyFile(String srcKey, String dstKey) { + ObjectMetadata objectMeta = + ossClient.getObjectMetadata(bucketName, srcKey); + long dataLen = objectMeta.getContentLength(); + if (dataLen <= multipartThreshold) { + return singleCopy(srcKey, dstKey); + } else { + return multipartCopy(srcKey, dataLen, dstKey); + } + } + + /** + * Use single copy to copy an oss object. + * + * @param srcKey source key + * @param dstKey destination key + * @return true if successfully copied + * (the caller should make sure srcPath is a file and dstPath is valid) + */ + private boolean singleCopy(String srcKey, String dstKey) { + CopyObjectResult copyResult = + ossClient.copyObject(bucketName, srcKey, bucketName, dstKey); + LOG.debug(copyResult.getETag()); + return true; + } + + /** + * Use multipart copy to copy an oss object. + * (the caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcKey source key + * @param dataLen data size of the object to copy + * @param dstKey destination key + * @return true if successfully copied, or false if upload is aborted + */ + private boolean multipartCopy(String srcKey, long dataLen, String dstKey) { + int partNum = (int)(dataLen / uploadPartSize); + if (dataLen % uploadPartSize != 0) { + partNum++; + } + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, dstKey); + ObjectMetadata meta = new ObjectMetadata(); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + String uploadId = initiateMultipartUploadResult.getUploadId(); + List partETags = new ArrayList(); + try { + for (int i = 0; i < partNum; i++) { + long skipBytes = uploadPartSize * i; + long size = (uploadPartSize < dataLen - skipBytes) ? + uploadPartSize : dataLen - skipBytes; + UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest(); + partCopyRequest.setSourceBucketName(bucketName); + partCopyRequest.setSourceKey(srcKey); + partCopyRequest.setBucketName(bucketName); + partCopyRequest.setKey(dstKey); + partCopyRequest.setUploadId(uploadId); + partCopyRequest.setPartSize(size); + partCopyRequest.setBeginIndex(skipBytes); + partCopyRequest.setPartNumber(i + 1); + UploadPartCopyResult partCopyResult = + ossClient.uploadPartCopy(partCopyRequest); + statistics.incrementWriteOps(1); + partETags.add(partCopyResult.getPartETag()); + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, dstKey, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + return true; + } catch (Exception e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + return false; + } + } + + /** + * Copy a directory from source path to destination path. + * (the caller should make sure srcPath is a directory, and dstPath is valid) + * + * @param srcPath source path + * @param dstPath destination path + * @return true if successfully copied + */ + private boolean copyDirectory(Path srcPath, Path dstPath) { + String srcKey = pathToKey(srcPath); + String dstKey = pathToKey(dstPath); + + if (!srcKey.endsWith("/")) { + srcKey = srcKey + "/"; + } + if (!dstKey.endsWith("/")) { + dstKey = dstKey + "/"; + } + + if (dstKey.startsWith(srcKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot rename a directory to a subdirectory of self"); + } + return false; + } + + ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName); + listObjectsRequest.setPrefix(srcKey); + listObjectsRequest.setMaxKeys(maxKeys); + + ObjectListing objects = ossClient.listObjects(listObjectsRequest); + statistics.incrementReadOps(1); + // Copy files from src folder to dst + while (true) { + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + String newKey = + dstKey.concat(objectSummary.getKey().substring(srcKey.length())); + copyFile(objectSummary.getKey(), newKey); + } + if (objects.isTruncated()) { + listObjectsRequest.setMarker(objects.getNextMarker()); + statistics.incrementReadOps(1); + } else { + break; + } + } + return true; + } + + @Override + public void setWorkingDirectory(Path dir) { + this.workingDir = dir; + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java new file mode 100644 index 00000000000..bcd00dc50eb --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem.Statistics; + +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.model.GetObjectRequest; + +/** + * The input stream for OSS blob system. + * The class uses multi-part downloading to read data from the object content + * stream. + */ +public class AliyunOSSInputStream extends FSInputStream { + public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class); + private static final int MAX_RETRIES = 10; + private final long downloadPartSize; + + private String bucketName; + private String key; + private OSSClient ossClient; + private Statistics statistics; + private boolean closed; + private InputStream wrappedStream = null; + private long dataLen; + private long position; + private long partRemaining; + + public AliyunOSSInputStream(Configuration conf, OSSClient client, + String bucketName, String key, Long dataLen, Statistics statistics) + throws IOException { + this.bucketName = bucketName; + this.key = key; + ossClient = client; + this.statistics = statistics; + this.dataLen = dataLen; + downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY, + MULTIPART_DOWNLOAD_SIZE_DEFAULT); + reopen(0); + closed = false; + } + + /** + * Reopen the wrapped stream at give position, by seeking for + * data of a part length from object content stream. + * + * @param pos position from start of a file + * @throws IOException if failed to reopen + */ + private synchronized void reopen(long pos) throws IOException { + + long partLen; + + if (pos < 0) { + throw new EOFException("Cannot seek at negtive position:" + pos); + } else if (pos > dataLen) { + throw new EOFException("Cannot seek after EOF, fileLen:" + dataLen + + " position:" + pos); + } else if (pos + downloadPartSize > dataLen) { + partLen = dataLen - pos; + } else { + partLen = downloadPartSize; + } + + if (wrappedStream != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Aborting old stream to open at pos " + pos); + } + wrappedStream.close(); + } + + GetObjectRequest request = new GetObjectRequest(bucketName, key); + request.setRange(pos, pos + partLen - 1); + wrappedStream = ossClient.getObject(request).getObjectContent(); + if (wrappedStream == null) { + throw new IOException("Null IO stream"); + } + position = pos; + partRemaining = partLen; + } + + @Override + public synchronized int read() throws IOException { + checkNotClosed(); + + if (partRemaining <= 0 && position < dataLen) { + reopen(position); + } + + int tries = MAX_RETRIES; + boolean retry; + int byteRead = -1; + do { + retry = false; + try { + byteRead = wrappedStream.read(); + } catch (Exception e) { + handleReadException(e, --tries); + retry = true; + } + } while (retry); + if (byteRead >= 0) { + position++; + partRemaining--; + } + + if (statistics != null && byteRead >= 0) { + statistics.incrementBytesRead(1); + } + return byteRead; + } + + + /** + * Check whether the input stream is closed. + * + * @throws IOException if stream is closed + */ + private void checkNotClosed() throws IOException { + if (closed) { + throw new IOException("Stream is closed!"); + } + } + + @Override + public synchronized int read(byte[] buf, int off, int len) + throws IOException { + checkNotClosed(); + + if (buf == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > buf.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + int bytesRead = 0; + // Not EOF, and read not done + while (position < dataLen && bytesRead < len) { + if (partRemaining == 0) { + reopen(position); + } + + int tries = MAX_RETRIES; + boolean retry; + int bytes = -1; + do { + retry = false; + try { + bytes = wrappedStream.read(buf, off + bytesRead, len - bytesRead); + } catch (Exception e) { + handleReadException(e, --tries); + retry = true; + } + } while (retry); + + if (bytes > 0) { + bytesRead += bytes; + position += bytes; + partRemaining -= bytes; + } else if (partRemaining != 0) { + throw new IOException("Failed to read from stream. Remaining:" + + partRemaining); + } + } + + if (statistics != null && bytesRead > 0) { + statistics.incrementBytesRead(bytesRead); + } + + // Read nothing, but attempt to read something + if (bytesRead == 0 && len > 0) { + return -1; + } else { + return bytesRead; + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (wrappedStream != null) { + wrappedStream.close(); + } + } + + @Override + public synchronized int available() throws IOException { + checkNotClosed(); + + long remaining = dataLen - position; + if (remaining > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return (int)remaining; + } + + @Override + public void seek(long pos) throws IOException { + checkNotClosed(); + if (position == pos) { + return; + } else if (pos > position && pos < position + partRemaining) { + wrappedStream.skip(pos - position); + position = pos; + } else { + reopen(pos); + } + } + + @Override + public long getPos() throws IOException { + checkNotClosed(); + return position; + } + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + checkNotClosed(); + return false; + } + + private void handleReadException(Exception e, int tries) throws IOException{ + if (tries == 0) { + throw new IOException(e); + } + + LOG.warn("Some exceptions occurred in oss connection, try to reopen oss" + + " connection at position '" + position + "', " + e.getMessage()); + try { + Thread.sleep(100); + } catch (InterruptedException e2) { + LOG.warn(e2.getMessage()); + } + reopen(position); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java new file mode 100644 index 00000000000..589e014f452 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem.Statistics; +import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.util.Progressable; + +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadResult; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PartETag; +import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.UploadPartRequest; +import com.aliyun.oss.model.UploadPartResult; + +/** + * The output stream for OSS blob system. + * Data will be buffered on local disk, then uploaded to OSS in + * {@link #close()} method. + */ +public class AliyunOSSOutputStream extends OutputStream { + public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class); + private String bucketName; + private String key; + private Statistics statistics; + private Progressable progress; + private String serverSideEncryptionAlgorithm; + private long partSize; + private long partSizeThreshold; + private LocalDirAllocator dirAlloc; + private boolean closed; + private File tmpFile; + private BufferedOutputStream backupStream; + private OSSClient ossClient; + + public AliyunOSSOutputStream(Configuration conf, OSSClient client, + String bucketName, String key, Progressable progress, + Statistics statistics, String serverSideEncryptionAlgorithm) + throws IOException { + this.bucketName = bucketName; + this.key = key; + // The caller cann't get any progress information + this.progress = progress; + ossClient = client; + this.statistics = statistics; + this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; + + partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, + MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); + + if (conf.get(BUFFER_DIR_KEY) == null) { + conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); + } + dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY); + + tmpFile = dirAlloc.createTmpFileForWrite("output-", + LocalDirAllocator.SIZE_UNKNOWN, conf); + backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile)); + closed = false; + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (backupStream != null) { + backupStream.close(); + } + long dataLen = tmpFile.length(); + try { + if (dataLen <= partSizeThreshold) { + uploadObject(); + } else { + multipartUploadObject(); + } + } finally { + tmpFile.delete(); + } + } + + /** + * Upload temporary file as an OSS object, using single upload. + * + * @throws IOException + */ + private void uploadObject() throws IOException { + File object = tmpFile.getAbsoluteFile(); + FileInputStream fis = new FileInputStream(object); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(object.length()); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + try { + PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta); + LOG.debug(result.getETag()); + statistics.incrementWriteOps(1); + } finally { + fis.close(); + } + } + + /** + * Upload temporary file as an OSS object, using multipart upload. + * + * @throws IOException + */ + private void multipartUploadObject() throws IOException { + File object = tmpFile.getAbsoluteFile(); + long dataLen = object.length(); + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, key); + ObjectMetadata meta = new ObjectMetadata(); + // meta.setContentLength(dataLen); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + int partNum = (int)(dataLen / partSize); + if (dataLen % partSize != 0) { + partNum += 1; + } + if (partNum > MULTIPART_UPLOAD_PART_NUM_LIMIT) { + throw new IOException("Number of parts " + partNum + " should not be " + + "bigger than limit " + MULTIPART_UPLOAD_PART_NUM_LIMIT); + } + List partETags = new ArrayList(); + String uploadId = initiateMultipartUploadResult.getUploadId(); + + try { + for (int i = 0; i < partNum; i++) { + // TODO: Optimize this, avoid opening the object multiple times + FileInputStream fis = new FileInputStream(object); + try { + long skipBytes = partSize * i; + fis.skip(skipBytes); + long size = (partSize < dataLen - skipBytes) ? + partSize : dataLen - skipBytes; + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(key); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(fis); + uploadPartRequest.setPartSize(size); + uploadPartRequest.setPartNumber(i + 1); + UploadPartResult uploadPartResult = + ossClient.uploadPart(uploadPartRequest); + statistics.incrementWriteOps(1); + partETags.add(uploadPartResult.getPartETag()); + } finally { + fis.close(); + } + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, key, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + } catch (Exception e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, key, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + } + } + + @Override + public synchronized void flush() throws IOException { + backupStream.flush(); + } + + @Override + public synchronized void write(int b) throws IOException { + backupStream.write(b); + statistics.incrementBytesWritten(1); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java new file mode 100644 index 00000000000..3f66a4fc5ee --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLDecoder; +import java.util.Objects; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +/** + * Utility methods for Aliyun OSS code. + */ +final public class AliyunOSSUtils { + private AliyunOSSUtils() { + } + + /** + * User information includes user name and password. + */ + static public class UserInfo { + private final String user; + private final String password; + + public static final UserInfo EMPTY = new UserInfo("", ""); + + public UserInfo(String user, String password) { + this.user = user; + this.password = password; + } + + /** + * Predicate to verify user information is set. + * @return true if the username is defined (not null, not empty). + */ + public boolean hasLogin() { + return StringUtils.isNotEmpty(user); + } + + /** + * Equality test matches user and password. + * @param o other object + * @return true if the objects are considered equivalent. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UserInfo that = (UserInfo) o; + return Objects.equals(user, that.user) && + Objects.equals(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hash(user, password); + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + } + + /** + * Used to get password from configuration, if default value is not available. + * @param conf configuration that contains password information + * @param key the key of the password + * @param val the default value of the key + * @return the value for the key + * @throws IOException if failed to get password from configuration + */ + static public String getPassword(Configuration conf, String key, String val) + throws IOException { + if (StringUtils.isEmpty(val)) { + try { + final char[] pass = conf.getPassword(key); + if (pass != null) { + return (new String(pass)).trim(); + } else { + return ""; + } + } catch (IOException ioe) { + throw new IOException("Cannot find password option " + key, ioe); + } + } else { + return val; + } + } + + /** + * Extract the user information details from a URI. + * @param name URI of the filesystem + * @return a login tuple, possibly empty. + */ + public static UserInfo extractLoginDetails(URI name) { + try { + String authority = name.getAuthority(); + if (authority == null) { + return UserInfo.EMPTY; + } + int loginIndex = authority.indexOf('@'); + if (loginIndex < 0) { + // No user information + return UserInfo.EMPTY; + } + String login = authority.substring(0, loginIndex); + int loginSplit = login.indexOf(':'); + if (loginSplit > 0) { + String user = login.substring(0, loginSplit); + String password = URLDecoder.decode(login.substring(loginSplit + 1), + "UTF-8"); + return new UserInfo(user, password); + } else if (loginSplit == 0) { + // There is no user, just a password. + return UserInfo.EMPTY; + } else { + return new UserInfo(login, ""); + } + } catch (UnsupportedEncodingException e) { + // This should never happen; translate it if it does. + throw new RuntimeException(e); + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java new file mode 100644 index 00000000000..4ee4cd4f475 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +/** + * ALL configuration constants for OSS filesystem. + */ +public final class Constants { + + private Constants() { + } + + // Class of credential provider + public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY = + "fs.oss.credentials.provider"; + + // OSS access verification + public static final String ACCESS_KEY = "fs.oss.access.key"; + public static final String SECRET_KEY = "fs.oss.secret.key"; + + // Number of simultaneous connections to oss + public static final String MAXIMUM_CONNECTIONS_KEY = + "fs.oss.connection.maximum"; + public static final int MAXIMUM_CONNECTIONS_DEFAULT = 32; + + // Connect to oss over ssl + public static final String SECURE_CONNECTIONS_KEY = + "fs.oss.connection.secure.enabled"; + public static final boolean SECURE_CONNECTIONS_DEFAULT = true; + + // Use a custom endpoint + public static final String ENDPOINT_KEY = "fs.oss.endpoint"; + + // Connect to oss through a proxy server + public static final String PROXY_HOST_KEY = "fs.oss.proxy.host"; + public static final String PROXY_PORT_KEY = "fs.oss.proxy.port"; + public static final String PROXY_USERNAME_KEY = "fs.oss.proxy.username"; + public static final String PROXY_PASSWORD_KEY = "fs.oss.proxy.password"; + public static final String PROXY_DOMAIN_KEY = "fs.oss.proxy.domain"; + public static final String PROXY_WORKSTATION_KEY = + "fs.oss.proxy.workstation"; + + // Number of times we should retry errors + public static final String MAX_ERROR_RETRIES_KEY = "fs.oss.attempts.maximum"; + public static final int MAX_ERROR_RETRIES_DEFAULT = 20; + + // Time until we give up trying to establish a connection to oss + public static final String ESTABLISH_TIMEOUT_KEY = + "fs.oss.connection.establish.timeout"; + public static final int ESTABLISH_TIMEOUT_DEFAULT = 50000; + + // Time until we give up on a connection to oss + public static final String SOCKET_TIMEOUT_KEY = "fs.oss.connection.timeout"; + public static final int SOCKET_TIMEOUT_DEFAULT = 200000; + + // Number of records to get while paging through a directory listing + public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum"; + public static final int MAX_PAGING_KEYS_DEFAULT = 500; + + // Size of each of or multipart pieces in bytes + public static final String MULTIPART_UPLOAD_SIZE_KEY = + "fs.oss.multipart.upload.size"; + + public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024; + public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 1000; + + // Minimum size in bytes before we start a multipart uploads or copy + public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY = + "fs.oss.multipart.upload.threshold"; + public static final long MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT = + 20 * 1024 * 1024; + + public static final String MULTIPART_DOWNLOAD_SIZE_KEY = + "fs.oss.multipart.download.size"; + + public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 100 * 1024; + + // Comma separated list of directories + public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir"; + + // private | public-read | public-read-write | authenticated-read | + // log-delivery-write | bucket-owner-read | bucket-owner-full-control + public static final String CANNED_ACL_KEY = "fs.oss.acl.default"; + public static final String CANNED_ACL_DEFAULT = ""; + + // OSS server-side encryption + public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY = + "fs.oss.server-side-encryption-algorithm"; + + public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size"; + public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024; + public static final String FS_OSS = "oss"; + +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java new file mode 100644 index 00000000000..234567b2b00 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Aliyun OSS Filesystem. + */ +package org.apache.hadoop.fs.aliyun.oss; \ No newline at end of file diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java new file mode 100644 index 00000000000..37ed8319174 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.internal.AssumptionViolatedException; + +import java.io.IOException; +import java.net.URI; +import java.util.Date; +import java.util.Random; + +/** + * Utility class for OSS Tests. + */ +public final class OSSTestUtils { + + private OSSTestUtils() { + } + + /** + * Create the test filesystem. + * + * If the test.fs.oss.name property is not set, + * tests will fail. + * + * @param conf configuration + * @return the FS + * @throws IOException + */ + public static AliyunOSSFileSystem createTestFileSystem(Configuration conf) + throws IOException { + String fsname = conf.getTrimmed( + TestOSSFileSystemContract.TEST_FS_OSS_NAME, ""); + + boolean liveTest = !StringUtils.isEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(Constants.FS_OSS); + } + + if (!liveTest) { + throw new AssumptionViolatedException("No test filesystem in " + + TestOSSFileSystemContract.TEST_FS_OSS_NAME); + } + AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem(); + ossfs.initialize(testURI, conf); + return ossfs; + } + + /** + * Generate unique test path for multiple user tests. + * + * @return root test path + */ + public static String generateUniqueTestPath() { + Long time = new Date().getTime(); + Random rand = new Random(); + return "/test_" + Long.toString(time) + "_" + + Long.toString(Math.abs(rand.nextLong())); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java new file mode 100644 index 00000000000..de4e5a93152 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Tests a live OSS system. + * + * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from + * TestCase which uses the old Junit3 runner that doesn't ignore assumptions + * properly making it impossible to skip the tests if we don't have a valid + * bucket. + */ +public class TestOSSFileSystemContract extends FileSystemContractBaseTest { + + protected static final Logger LOG = + LoggerFactory.getLogger(TestOSSFileSystemContract.class); + + public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; + private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + + @Override + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fs = OSSTestUtils.createTestFileSystem(conf); + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(super.path(testRootPath), true); + } + super.tearDown(); + } + + @Override + protected Path path(String path) { + if (path.startsWith("/")) { + return super.path(testRootPath + path); + } else { + return super.path(testRootPath + "/" + path); + } + } + + @Override + public void testMkdirsWithUmask() throws Exception { + // not supported + } + + /** + * Assert that root directory renames are not allowed. + * + * @throws Exception on failures + */ + @Override + public void testRootDirAlwaysExists() throws Exception { + //this will throw an exception if the path is not found + fs.getFileStatus(super.path("/")); + //this catches overrides of the base exists() method that don't + //use getFileStatus() as an existence probe + assertTrue("FileSystem.exists() fails for root", + fs.exists(super.path("/"))); + } + + /** + * Assert that root directory renames are not allowed. + * + * @throws Exception on failures + */ + @Override + public void testRenameRootDirForbidden() throws Exception { + if (!renameSupported()) { + return; + } + rename(super.path("/"), + super.path("/test/newRootDir"), + false, true, false); + } + + public void testDeleteSubdir() throws IOException { + Path parentDir = this.path("/test/hadoop"); + Path file = this.path("/test/hadoop/file"); + Path subdir = this.path("/test/hadoop/subdir"); + this.createFile(file); + + assertTrue("Created subdir", this.fs.mkdirs(subdir)); + assertTrue("File exists", this.fs.exists(file)); + assertTrue("Parent dir exists", this.fs.exists(parentDir)); + assertTrue("Subdir exists", this.fs.exists(subdir)); + + assertTrue("Deleted subdir", this.fs.delete(subdir, true)); + assertTrue("Parent should exist", this.fs.exists(parentDir)); + + assertTrue("Deleted file", this.fs.delete(file, false)); + assertTrue("Parent should exist", this.fs.exists(parentDir)); + } + + + @Override + protected boolean renameSupported() { + return true; + } + + @Override + public void testRenameNonExistentPath() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/path"); + Path dst = this.path("/test/new/newpath"); + try { + super.rename(src, dst, false, false, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameFileMoveToNonExistentDirectory() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/file"); + this.createFile(src); + Path dst = this.path("/test/new/newfile"); + try { + super.rename(src, dst, false, true, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/dir"); + this.fs.mkdirs(src); + Path dst = this.path("/test/new/newdir"); + try { + super.rename(src, dst, false, true, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameFileMoveToExistingDirectory() throws Exception { + super.testRenameFileMoveToExistingDirectory(); + } + + @Override + public void testRenameFileAsExistingFile() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/file"); + this.createFile(src); + Path dst = this.path("/test/new/newfile"); + this.createFile(dst); + try { + super.rename(src, dst, false, true, true); + fail("Should throw FileAlreadyExistsException"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + } + + @Override + public void testRenameDirectoryAsExistingFile() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/dir"); + this.fs.mkdirs(src); + Path dst = this.path("/test/new/newfile"); + this.createFile(dst); + try { + super.rename(src, dst, false, true, true); + fail("Should throw FileAlreadyExistsException"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + } + + public void testGetFileStatusFileAndDirectory() throws Exception { + Path filePath = this.path("/test/oss/file1"); + this.createFile(filePath); + assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile()); + assertFalse("Should not be directory", + this.fs.getFileStatus(filePath).isDirectory()); + + Path dirPath = this.path("/test/oss/dir"); + this.fs.mkdirs(dirPath); + assertTrue("Should be directory", + this.fs.getFileStatus(dirPath).isDirectory()); + assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile()); + } + + public void testMkdirsForExistingFile() throws Exception { + Path testFile = this.path("/test/hadoop/file"); + assertFalse(this.fs.exists(testFile)); + this.createFile(testFile); + assertTrue(this.fs.exists(testFile)); + try { + this.fs.mkdirs(testFile); + fail("Should throw FileAlreadyExistsException!"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + + public void testWorkingDirectory() throws Exception { + Path workDir = super.path(this.getDefaultWorkingDirectory()); + assertEquals(workDir, this.fs.getWorkingDirectory()); + this.fs.setWorkingDirectory(super.path(".")); + assertEquals(workDir, this.fs.getWorkingDirectory()); + this.fs.setWorkingDirectory(super.path("..")); + assertEquals(workDir.getParent(), this.fs.getWorkingDirectory()); + Path relativeDir = super.path("hadoop"); + this.fs.setWorkingDirectory(relativeDir); + assertEquals(relativeDir, this.fs.getWorkingDirectory()); + Path absoluteDir = super.path("/test/hadoop"); + this.fs.setWorkingDirectory(absoluteDir); + assertEquals(absoluteDir, this.fs.getWorkingDirectory()); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java new file mode 100644 index 00000000000..411cd576f75 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.io.IOUtils; +import org.junit.*; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Random; + +import static org.junit.Assert.assertTrue; + +/** + * Tests basic functionality for AliyunOSSInputStream, including seeking and + * reading files. + */ +public class TestOSSInputStream { + + private FileSystem fs; + + protected static final Logger LOG = + LoggerFactory.getLogger(TestOSSInputStream.class); + + private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + + @Rule + public Timeout testTimeout = new Timeout(30 * 60 * 1000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fs = OSSTestUtils.createTestFileSystem(conf); + } + + @After + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(new Path(testRootPath), true); + } + } + + private Path setPath(String path) { + if (path.startsWith("/")) { + return new Path(testRootPath + path); + } else { + return new Path(testRootPath + "/" + path); + } + } + + @Test + public void testSeekFile() throws Exception { + Path smallSeekFile = setPath("/test/smallSeekFile.txt"); + long size = 5 * 1024 * 1024; + + ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255); + LOG.info("5MB file created: smallSeekFile.txt"); + + FSDataInputStream instream = this.fs.open(smallSeekFile); + int seekTimes = 5; + LOG.info("multiple fold position seeking test...:"); + for (int i = 0; i < seekTimes; i++) { + long pos = size / (seekTimes - i) - 1; + LOG.info("begin seeking for pos: " + pos); + instream.seek(pos); + assertTrue("expected position at:" + pos + ", but got:" + + instream.getPos(), instream.getPos() == pos); + LOG.info("completed seeking at pos: " + instream.getPos()); + } + LOG.info("random position seeking test...:"); + Random rand = new Random(); + for (int i = 0; i < seekTimes; i++) { + long pos = Math.abs(rand.nextLong()) % size; + LOG.info("begin seeking for pos: " + pos); + instream.seek(pos); + assertTrue("expected position at:" + pos + ", but got:" + + instream.getPos(), instream.getPos() == pos); + LOG.info("completed seeking at pos: " + instream.getPos()); + } + IOUtils.closeStream(instream); + } + + @Test + public void testReadFile() throws Exception { + final int bufLen = 256; + final int sizeFlag = 5; + String filename = "readTestFile_" + sizeFlag + ".txt"; + Path readTestFile = setPath("/test/" + filename); + long size = sizeFlag * 1024 * 1024; + + ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255); + LOG.info(sizeFlag + "MB file created: /test/" + filename); + + FSDataInputStream instream = this.fs.open(readTestFile); + byte[] buf = new byte[bufLen]; + long bytesRead = 0; + while (bytesRead < size) { + int bytes; + if (size - bytesRead < bufLen) { + int remaining = (int)(size - bytesRead); + bytes = instream.read(buf, 0, remaining); + } else { + bytes = instream.read(buf, 0, bufLen); + } + bytesRead += bytes; + + if (bytesRead % (1024 * 1024) == 0) { + int available = instream.available(); + int remaining = (int)(size - bytesRead); + assertTrue("expected remaining:" + remaining + ", but got:" + available, + remaining == available); + LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024)) + + " MB"); + } + } + assertTrue(instream.available() == 0); + IOUtils.closeStream(instream); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java new file mode 100644 index 00000000000..3951529dc6f --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; + +/** + * Tests regular and multi-part upload functionality for AliyunOSSOutputStream. + */ +public class TestOSSOutputStream { + private FileSystem fs; + private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + + @Rule + public Timeout testTimeout = new Timeout(30 * 60 * 1000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024); + conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024); + fs = OSSTestUtils.createTestFileSystem(conf); + } + + @After + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(new Path(testRootPath), true); + } + } + + protected Path getTestPath() { + return new Path(testRootPath + "/testoss"); + } + + @Test + public void testRegularUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024); + } + + @Test + public void testMultiPartUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java new file mode 100644 index 00000000000..8214b9f6bec --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; + +/** + * The contract of OSS: only enabled if the test bucket is provided. + */ +public class OSSContract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/oss.xml"; + public static final String CONTRACT_TEST_OSS_FS_NAME = + "fs.contract.test.fs.oss"; + + private static String testPath = OSSTestUtils.generateUniqueTestPath(); + + public OSSContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + @Override + public String getScheme() { + return "oss"; + } + + @Override + public Path getTestPath() { + Path path = new Path(testPath); + return path; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java new file mode 100644 index 00000000000..cc5a2d17527 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +/** + * OSS contract creating tests. + */ +public class TestOSSContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } + + @Override + public void testOverwriteEmptyDirectory() throws Throwable { + ContractTestUtils.skip( + "blobstores can't distinguish empty directories from files"); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java new file mode 100644 index 00000000000..6a1eb40e3a1 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * OSS contract deleting tests. + */ +public class TestOSSContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java new file mode 100644 index 00000000000..1dcb7f030de --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * OSS contract directory tests. + */ +public class TestOSSContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java new file mode 100644 index 00000000000..ee0c055decb --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * OSS contract opening file tests. + */ +public class TestOSSContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java new file mode 100644 index 00000000000..634fcf12fa6 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * OSS contract renaming tests. + */ +public class TestOSSContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java new file mode 100644 index 00000000000..40ea772c943 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * OSS contract seeking tests. + */ +public class TestOSSContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml new file mode 100644 index 00000000000..2bc34b754b9 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml @@ -0,0 +1,105 @@ + + + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-blobstore + true + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.rename-returns-false-if-source-missing + false + + + + fs.contract.rename-remove-dest-if-empty-dir + false + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + false + + + + fs.contract.supports-atomic-rename + false + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + + + fs.contract.rename-overwrites-dest + true + + + + fs.oss.multipart.download.size + 102400 + + diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml new file mode 100644 index 00000000000..fa4118c2162 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml @@ -0,0 +1,46 @@ + + + + + + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + + + + + hadoop.security.authentication + simple + + + + + + + + diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties new file mode 100644 index 00000000000..bb5cbe5ec32 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# log4j configuration used during build and unit tests + +log4j.rootLogger=INFO,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 899a9455d29..14fa9f05f78 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -100,6 +100,12 @@ compile ${project.version} + + org.apache.hadoop + hadoop-aliyun + compile + ${project.version} + org.apache.hadoop hadoop-sls diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index db002f46fc7..e7e876bc82a 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -47,6 +47,7 @@ hadoop-aws hadoop-kafka hadoop-azure-datalake + hadoop-aliyun From 4d84c814fcaf074022593c057d8f8dec4cd461fa Mon Sep 17 00:00:00 2001 From: Mingfei Date: Thu, 18 Aug 2016 16:06:57 +0800 Subject: [PATCH 02/13] HADOOP-13491. Fix several warnings from findbugs. Contributed by Genmao Yu. --- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 3 ++- .../fs/aliyun/oss/AliyunOSSInputStream.java | 6 ++--- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 10 ++++--- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 26 ++++++++++++++++++- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 30ddf8c6a66..6923b9513af 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.aliyun.oss; import static org.apache.hadoop.fs.aliyun.oss.Constants.*; +import com.aliyun.oss.ClientException; import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.common.auth.DefaultCredentials; @@ -782,7 +783,7 @@ public class AliyunOSSFileSystem extends FileSystem { ossClient.completeMultipartUpload(completeMultipartUploadRequest); LOG.debug(completeMultipartUploadResult.getETag()); return true; - } catch (Exception e) { + } catch (OSSException | ClientException e) { AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); ossClient.abortMultipartUpload(abortMultipartUploadRequest); diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java index bcd00dc50eb..b12e3f0ca57 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -227,12 +227,12 @@ public class AliyunOSSInputStream extends FSInputStream { } @Override - public void seek(long pos) throws IOException { + public synchronized void seek(long pos) throws IOException { checkNotClosed(); if (position == pos) { return; } else if (pos > position && pos < position + partRemaining) { - wrappedStream.skip(pos - position); + AliyunOSSUtils.skipFully(wrappedStream, pos - position); position = pos; } else { reopen(pos); @@ -240,7 +240,7 @@ public class AliyunOSSInputStream extends FSInputStream { } @Override - public long getPos() throws IOException { + public synchronized long getPos() throws IOException { checkNotClosed(); return position; } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java index 589e014f452..654b81dab41 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -29,6 +29,8 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSSException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -113,7 +115,9 @@ public class AliyunOSSOutputStream extends OutputStream { multipartUploadObject(); } } finally { - tmpFile.delete(); + if (!tmpFile.delete()) { + LOG.warn("Can not delete file: " + tmpFile); + } } } @@ -174,7 +178,7 @@ public class AliyunOSSOutputStream extends OutputStream { FileInputStream fis = new FileInputStream(object); try { long skipBytes = partSize * i; - fis.skip(skipBytes); + AliyunOSSUtils.skipFully(fis, skipBytes); long size = (partSize < dataLen - skipBytes) ? partSize : dataLen - skipBytes; UploadPartRequest uploadPartRequest = new UploadPartRequest(); @@ -198,7 +202,7 @@ public class AliyunOSSOutputStream extends OutputStream { CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest); LOG.debug(completeMultipartUploadResult.getETag()); - } catch (Exception e) { + } catch (OSSException | ClientException e) { AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId); ossClient.abortMultipartUpload(abortMultipartUploadRequest); diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 3f66a4fc5ee..9acde00606a 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.aliyun.oss; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLDecoder; @@ -116,7 +117,7 @@ final public class AliyunOSSUtils { /** * Extract the user information details from a URI. - * @param name URI of the filesystem + * @param name URI of the filesystem. * @return a login tuple, possibly empty. */ public static UserInfo extractLoginDetails(URI name) { @@ -148,4 +149,27 @@ final public class AliyunOSSUtils { throw new RuntimeException(e); } } + + /** + * Skips the requested number of bytes or fail if there are not enough left. + * This allows for the possibility that {@link InputStream#skip(long)} may not + * skip as many bytes as requested (most likely because of reaching EOF). + * @param is the input stream to skip. + * @param n the number of bytes to skip. + * @throws IOException thrown when skipped less number of bytes. + */ + public static void skipFully(InputStream is, long n) throws IOException { + long total = 0; + long cur = 0; + + do { + cur = is.skip(n - total); + total += cur; + } while((total < n) && (cur > 0)); + + if (total < n) { + throw new IOException("Failed to skip " + n + " bytes, possibly due " + + "to EOF."); + } + } } From bd2d97adeea55bf2c7e4ab475bcc90f3a14e751a Mon Sep 17 00:00:00 2001 From: Mingfei Date: Tue, 23 Aug 2016 16:18:34 +0800 Subject: [PATCH 03/13] HADOOP-13483. File create should throw error rather than overwrite directories. Contributed by Genmao Yu. --- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 25 ++++++++++++++++--- .../oss/contract/TestOSSContractCreate.java | 6 ----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 6923b9513af..99a60dbf664 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -63,6 +63,8 @@ import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.PartETag; import com.aliyun.oss.model.UploadPartCopyRequest; import com.aliyun.oss.model.UploadPartCopyResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Implementation of {@link FileSystem} for @@ -70,6 +72,8 @@ import com.aliyun.oss.model.UploadPartCopyResult; */ public class AliyunOSSFileSystem extends FileSystem { + private static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSFileSystem.class); private URI uri; private Path workingDir; private OSSClient ossClient; @@ -101,9 +105,24 @@ public class AliyunOSSFileSystem extends FileSystem { boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { String key = pathToKey(path); + FileStatus status = null; - if (!overwrite && exists(path)) { - throw new FileAlreadyExistsException(path + " already exists"); + try { + // get the status or throw a FNFE + status = getFileStatus(path); + + // if the thread reaches here, there is something at the path + if (status.isDirectory()) { + // path references a directory + throw new FileAlreadyExistsException(path + " is a directory"); + } + if (!overwrite) { + // path references a file and overwrite is disabled + throw new FileAlreadyExistsException(path + " already exists"); + } + LOG.debug("Overwriting file {}", path); + } catch (FileNotFoundException e) { + // this means the file is not found } return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(), @@ -540,7 +559,7 @@ public class AliyunOSSFileSystem extends FileSystem { /** * Used to create an empty file that represents an empty directory. * - * @param bucketName the bucket this directory belongs to + * @param bucket the bucket this directory belongs to * @param objectName directory path * @return true if directory successfully created * @throws IOException diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java index cc5a2d17527..ce927a97b39 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java @@ -21,7 +21,6 @@ package org.apache.hadoop.fs.aliyun.oss.contract; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractCreateTest; import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; /** * OSS contract creating tests. @@ -33,9 +32,4 @@ public class TestOSSContractCreate extends AbstractContractCreateTest { return new OSSContract(conf); } - @Override - public void testOverwriteEmptyDirectory() throws Throwable { - ContractTestUtils.skip( - "blobstores can't distinguish empty directories from files"); - } } From 6bb741b9f811d3a1c0ce4ecc91a78ac47513bb8e Mon Sep 17 00:00:00 2001 From: Mingfei Date: Tue, 23 Aug 2016 17:10:00 +0800 Subject: [PATCH 04/13] HADOOP-13499. Support session credentials for authenticating with Aliyun. Contributed by Genmao Yu. --- .../hadoop/fs/aliyun/oss/Constants.java | 5 +- .../TemporaryAliyunCredentialsProvider.java | 64 +++++++++++++++++++ .../oss/TestOSSTemporaryCredentials.java | 64 +++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 4ee4cd4f475..0bc6d578ed6 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -31,8 +31,9 @@ public final class Constants { "fs.oss.credentials.provider"; // OSS access verification - public static final String ACCESS_KEY = "fs.oss.access.key"; - public static final String SECRET_KEY = "fs.oss.secret.key"; + public static final String ACCESS_KEY = "fs.oss.accessKeyId"; + public static final String SECRET_KEY = "fs.oss.accessKeySecret"; + public static final String SECURITY_TOKEN = "fs.oss.securityToken"; // Number of simultaneous connections to oss public static final String MAXIMUM_CONNECTIONS_KEY = diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java new file mode 100644 index 00000000000..ec8e7fe8386 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import com.aliyun.oss.common.auth.InvalidCredentialsException; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Support session credentials for authenticating with ALiyun. + */ +public class TemporaryAliyunCredentialsProvider implements CredentialsProvider { + public static final String NAME + = "org.apache.hadoop.fs.aliyun.oss.TemporaryAliyunCredentialsProvider"; + private final String accessKeyId; + private final String accessKeySecret; + private final String securityToken; + + public TemporaryAliyunCredentialsProvider(URI uri, Configuration conf) { + this.accessKeyId = conf.get(ACCESS_KEY, null); + this.accessKeySecret = conf.get(SECRET_KEY, null); + this.securityToken = conf.get(SECURITY_TOKEN, null); + } + + @Override + public void setCredentials(Credentials creds) { + + } + + @Override + public Credentials getCredentials() { + if (!StringUtils.isEmpty(accessKeyId) + && !StringUtils.isEmpty(accessKeySecret) + && !StringUtils.isEmpty(securityToken)) { + return new DefaultCredentials(accessKeyId, accessKeySecret, + securityToken); + } + throw new InvalidCredentialsException( + "AccessKeyId, AccessKeySecret or SecurityToken is unset"); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java new file mode 100644 index 00000000000..ca2c0bc203d --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.InvalidCredentialsException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.aliyun.oss.contract.OSSContract; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.junit.Test; + +import java.net.URI; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY; +import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY; +import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; + +/** + * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS). + * This test extends a class that "does things to the root directory", and + * should only be used against transient filesystems where you don't care about + * the data. + */ +public class TestOSSTemporaryCredentials extends AbstractFSContractTestBase { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } + + @Test + public void testTemporaryCredentialValidation() throws Throwable { + Configuration conf = new Configuration(); + conf.set(ACCESS_KEY, "accessKeyId"); + conf.set(SECRET_KEY, "accessKeySecret"); + conf.set(SECURITY_TOKEN, ""); + URI uri = getFileSystem().getUri(); + TemporaryAliyunCredentialsProvider provider + = new TemporaryAliyunCredentialsProvider(uri, conf); + try { + Credentials credentials = provider.getCredentials(); + fail("Expected a CredentialInitializationException, got " + credentials); + } catch (InvalidCredentialsException expected) { + // expected + } + } +} From cdb77110e77b70ed0c1125b2a6a422a8c7c28ec7 Mon Sep 17 00:00:00 2001 From: Mingfei Date: Wed, 24 Aug 2016 10:09:37 +0800 Subject: [PATCH 05/13] HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu. --- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 9 ++++---- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 23 ++++++++++--------- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 15 ++++++++++++ .../hadoop/fs/aliyun/oss/Constants.java | 4 +++- .../fs/aliyun/oss/TestOSSOutputStream.java | 19 +++++++++++++++ 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 99a60dbf664..afe7242780a 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory; * Aliyun OSS, used to access OSS blob system in a filesystem style. */ public class AliyunOSSFileSystem extends FileSystem { - private static final Logger LOG = LoggerFactory.getLogger(AliyunOSSFileSystem.class); private URI uri; @@ -560,18 +559,18 @@ public class AliyunOSSFileSystem extends FileSystem { * Used to create an empty file that represents an empty directory. * * @param bucket the bucket this directory belongs to - * @param objectName directory path + * @param key directory path * @return true if directory successfully created * @throws IOException */ - private boolean mkdir(final String bucket, final String objectName) + private boolean mkdir(final String bucket, final String key) throws IOException { - String dirName = objectName; + String dirName = key; ObjectMetadata dirMeta = new ObjectMetadata(); byte[] buffer = new byte[0]; ByteArrayInputStream in = new ByteArrayInputStream(buffer); dirMeta.setContentLength(0); - if (!objectName.endsWith("/")) { + if (!key.endsWith("/")) { dirName += "/"; } try { diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java index 654b81dab41..1e16df9edb7 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -84,6 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream { partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_UPLOAD_SIZE_DEFAULT); + if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) { + partSize = MIN_MULTIPART_UPLOAD_PART_SIZE; + } partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); @@ -151,6 +154,12 @@ public class AliyunOSSOutputStream extends OutputStream { private void multipartUploadObject() throws IOException { File object = tmpFile.getAbsoluteFile(); long dataLen = object.length(); + long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize); + int partNum = (int)(dataLen / realPartSize); + if (dataLen % realPartSize != 0) { + partNum += 1; + } + InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key); ObjectMetadata meta = new ObjectMetadata(); @@ -161,14 +170,6 @@ public class AliyunOSSOutputStream extends OutputStream { initiateMultipartUploadRequest.setObjectMetadata(meta); InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); - int partNum = (int)(dataLen / partSize); - if (dataLen % partSize != 0) { - partNum += 1; - } - if (partNum > MULTIPART_UPLOAD_PART_NUM_LIMIT) { - throw new IOException("Number of parts " + partNum + " should not be " + - "bigger than limit " + MULTIPART_UPLOAD_PART_NUM_LIMIT); - } List partETags = new ArrayList(); String uploadId = initiateMultipartUploadResult.getUploadId(); @@ -177,10 +178,10 @@ public class AliyunOSSOutputStream extends OutputStream { // TODO: Optimize this, avoid opening the object multiple times FileInputStream fis = new FileInputStream(object); try { - long skipBytes = partSize * i; + long skipBytes = realPartSize * i; AliyunOSSUtils.skipFully(fis, skipBytes); - long size = (partSize < dataLen - skipBytes) ? - partSize : dataLen - skipBytes; + long size = (realPartSize < dataLen - skipBytes) ? + realPartSize : dataLen - skipBytes; UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(key); diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 9acde00606a..d54dd9c44ae 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -28,6 +28,8 @@ import java.util.Objects; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT; + /** * Utility methods for Aliyun OSS code. */ @@ -172,4 +174,17 @@ final public class AliyunOSSUtils { "to EOF."); } } + + /** + * Calculate a proper size of multipart piece. If minPartSize + * is too small, the number of multipart pieces may exceed the limit of + * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}. + * @param contentLength the size of file. + * @param minPartSize the minimum size of multipart piece. + * @return a revisional size of multipart piece. + */ + public static long calculatePartSize(long contentLength, long minPartSize) { + long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1; + return Math.max(minPartSize, tmpPartSize); + } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 0bc6d578ed6..99022754fbe 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -79,7 +79,7 @@ public final class Constants { "fs.oss.multipart.upload.size"; public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024; - public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 1000; + public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000; // Minimum size in bytes before we start a multipart uploads or copy public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY = @@ -108,4 +108,6 @@ public final class Constants { public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024; public static final String FS_OSS = "oss"; + public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L; + } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java index 3951529dc6f..b33ab99cfc3 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java @@ -68,4 +68,23 @@ public class TestOSSOutputStream { public void testMultiPartUpload() throws IOException { ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024); } + + @Test + public void testMultiPartUploadLimit() throws IOException { + long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024); + assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024); + assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024, + 100 * 1024); + assert(10000 * 100 * 1024 / partSize3 + < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024, + 100 * 1024); + assert(10001 * 100 * 1024 / partSize4 + < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + } } From d33e928fbeb1764a724c8f3c051bb0d8be82bbff Mon Sep 17 00:00:00 2001 From: Mingfei Date: Fri, 26 Aug 2016 11:00:03 +0800 Subject: [PATCH 06/13] HADOOP-13529. Do some code refactoring. Contributed by Genmao Yu. --- hadoop-tools/hadoop-aliyun/pom.xml | 23 +- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 455 +++------------- .../aliyun/oss/AliyunOSSFileSystemStore.java | 486 ++++++++++++++++++ .../fs/aliyun/oss/AliyunOSSInputStream.java | 60 +-- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 129 +---- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 193 +++---- .../hadoop/fs/aliyun/oss/Constants.java | 3 +- .../aliyun/oss/TestOSSFileSystemContract.java | 10 - .../fs/aliyun/oss/TestOSSFileSystemStore.java | 121 +++++ .../fs/aliyun/oss/contract/OSSContract.java | 1 - .../oss/contract/TestOSSContractDispCp.java | 44 ++ .../TestOSSContractGetFileStatus.java | 35 ++ .../oss/contract/TestOSSContractRootDir.java | 69 +++ 13 files changed, 968 insertions(+), 661 deletions(-) create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml index c87d13f24cc..358b18b1c45 100644 --- a/hadoop-tools/hadoop-aliyun/pom.xml +++ b/hadoop-tools/hadoop-aliyun/pom.xml @@ -128,6 +128,27 @@ test test-jar - + + org.apache.hadoop + hadoop-distcp + test + + + org.apache.hadoop + hadoop-distcp + test + test-jar + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index afe7242780a..ad321bd2039 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -20,18 +20,12 @@ package org.apache.hadoop.fs.aliyun.oss; import static org.apache.hadoop.fs.aliyun.oss.Constants.*; -import com.aliyun.oss.ClientException; -import com.aliyun.oss.common.auth.CredentialsProvider; -import com.aliyun.oss.common.auth.DefaultCredentialProvider; -import com.aliyun.oss.common.auth.DefaultCredentials; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -39,30 +33,13 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.UserInfo; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.util.Progressable; -import com.aliyun.oss.ClientConfiguration; -import com.aliyun.oss.OSSClient; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.common.comm.Protocol; -import com.aliyun.oss.model.AbortMultipartUploadRequest; -import com.aliyun.oss.model.CannedAccessControlList; -import com.aliyun.oss.model.CompleteMultipartUploadRequest; -import com.aliyun.oss.model.CompleteMultipartUploadResult; -import com.aliyun.oss.model.CopyObjectResult; -import com.aliyun.oss.model.DeleteObjectsRequest; -import com.aliyun.oss.model.InitiateMultipartUploadRequest; -import com.aliyun.oss.model.InitiateMultipartUploadResult; -import com.aliyun.oss.model.ListObjectsRequest; import com.aliyun.oss.model.OSSObjectSummary; import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.ObjectMetadata; -import com.aliyun.oss.model.PartETag; -import com.aliyun.oss.model.UploadPartCopyRequest; -import com.aliyun.oss.model.UploadPartCopyResult; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,12 +52,8 @@ public class AliyunOSSFileSystem extends FileSystem { LoggerFactory.getLogger(AliyunOSSFileSystem.class); private URI uri; private Path workingDir; - private OSSClient ossClient; - private String bucketName; - private long uploadPartSize; - private long multipartThreshold; + private AliyunOSSFileSystemStore store; private int maxKeys; - private String serverSideEncryptionAlgorithm; @Override public FSDataOutputStream append(Path path, int bufferSize, @@ -91,9 +64,7 @@ public class AliyunOSSFileSystem extends FileSystem { @Override public void close() throws IOException { try { - if (ossClient != null) { - ossClient.shutdown(); - } + store.close(); } finally { super.close(); } @@ -125,23 +96,33 @@ public class AliyunOSSFileSystem extends FileSystem { } return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(), - ossClient, bucketName, key, progress, statistics, - serverSideEncryptionAlgorithm), (Statistics)(null)); + store, key, progress, statistics), (Statistics)(null)); } @Override public boolean delete(Path path, boolean recursive) throws IOException { - FileStatus status; try { - status = getFileStatus(path); + return innerDelete(getFileStatus(path), recursive); } catch (FileNotFoundException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Couldn't delete " + path + ": Does not exist!"); - } + LOG.debug("Couldn't delete {} - does not exist", path); return false; } + } - String key = pathToKey(status.getPath()); + /** + * Delete an object. See {@link #delete(Path, boolean)}. + * + * @param status fileStatus object + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. + * @throws IOException due to inability to delete a directory or file. + */ + private boolean innerDelete(FileStatus status, boolean recursive) + throws IOException { + Path f = status.getPath(); + String key = pathToKey(f); if (status.isDirectory()) { if (!key.endsWith("/")) { key += "/"; @@ -150,54 +131,34 @@ public class AliyunOSSFileSystem extends FileSystem { FileStatus[] statuses = listStatus(status.getPath()); // Check whether it is an empty directory or not if (statuses.length > 0) { - throw new IOException("Cannot remove directory" + path + + throw new IOException("Cannot remove directory " + f + ": It is not empty!"); } else { // Delete empty directory without '-r' - ossClient.deleteObject(bucketName, key); - statistics.incrementWriteOps(1); + store.deleteObject(key); } } else { - ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); - listRequest.setPrefix(key); - listRequest.setMaxKeys(maxKeys); - - while (true) { - ObjectListing objects = ossClient.listObjects(listRequest); - statistics.incrementReadOps(1); - List keysToDelete = new ArrayList(); - for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { - keysToDelete.add(objectSummary.getKey()); - } - DeleteObjectsRequest deleteRequest = - new DeleteObjectsRequest(bucketName); - deleteRequest.setKeys(keysToDelete); - ossClient.deleteObjects(deleteRequest); - statistics.incrementWriteOps(1); - if (objects.isTruncated()) { - listRequest.setMarker(objects.getNextMarker()); - } else { - break; - } - } + store.deleteDirs(key); } } else { - ossClient.deleteObject(bucketName, key); - statistics.incrementWriteOps(1); + store.deleteObject(key); } - //TODO: optimize logic here + + createFakeDirectoryIfNecessary(f); + return true; + } + + private void createFakeDirectoryIfNecessary(Path f) throws IOException { try { - Path pPath = status.getPath().getParent(); + Path pPath = f.getParent(); FileStatus pStatus = getFileStatus(pPath); - if (pStatus.isDirectory()) { - return true; - } else { + if (pStatus.isFile()) { throw new IOException("Path " + pPath + " is assumed to be a directory!"); } } catch (FileNotFoundException fnfe) { // Make sure the parent directory exists - return mkdir(bucketName, pathToKey(status.getPath().getParent())); + mkdir(pathToKey(f.getParent())); } } @@ -211,22 +172,15 @@ public class AliyunOSSFileSystem extends FileSystem { return new FileStatus(0, true, 1, 0, 0, qualifiedPath); } - ObjectMetadata meta = getObjectMetadata(key); + ObjectMetadata meta = store.getObjectMetadata(key); // If key not found and key does not end with "/" if (meta == null && !key.endsWith("/")) { // Case: dir + "/" key += "/"; - meta = getObjectMetadata(key); + meta = store.getObjectMetadata(key); } if (meta == null) { - // Case: dir + "/" + file - ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); - listRequest.setPrefix(key); - listRequest.setDelimiter("/"); - listRequest.setMaxKeys(1); - - ObjectListing listing = ossClient.listObjects(listRequest); - statistics.incrementReadOps(1); + ObjectListing listing = store.listObjects(key, 1, "/", null); if (!listing.getObjectSummaries().isEmpty() || !listing.getCommonPrefixes().isEmpty()) { return new FileStatus(0, true, 1, 0, 0, qualifiedPath); @@ -242,22 +196,6 @@ public class AliyunOSSFileSystem extends FileSystem { } } - /** - * Return object metadata given object key. - * - * @param key object key - * @return return null if key does not exist - */ - private ObjectMetadata getObjectMetadata(String key) { - try { - return ossClient.getObjectMetadata(bucketName, key); - } catch (OSSException osse) { - return null; - } finally { - statistics.incrementReadOps(1); - } - } - @Override public String getScheme() { return "oss"; @@ -288,7 +226,6 @@ public class AliyunOSSFileSystem extends FileSystem { * Initialize new FileSystem. * * @param name the uri of the file system, including host, port, etc. - * * @param conf configuration of the file system * @throws IOException IO problems */ @@ -296,153 +233,15 @@ public class AliyunOSSFileSystem extends FileSystem { super.initialize(name, conf); uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority()); - workingDir = - new Path("/user", - System.getProperty("user.name")).makeQualified(uri, null); - - bucketName = name.getHost(); - - ClientConfiguration clientConf = new ClientConfiguration(); - clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY, - MAXIMUM_CONNECTIONS_DEFAULT)); - boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY, - SECURE_CONNECTIONS_DEFAULT); - clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); - clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY, - MAX_ERROR_RETRIES_DEFAULT)); - clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY, - ESTABLISH_TIMEOUT_DEFAULT)); - clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY, - SOCKET_TIMEOUT_DEFAULT)); - - String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); - int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); - if (!proxyHost.isEmpty()) { - clientConf.setProxyHost(proxyHost); - if (proxyPort >= 0) { - clientConf.setProxyPort(proxyPort); - } else { - if (secureConnections) { - LOG.warn("Proxy host set without port. Using HTTPS default 443"); - clientConf.setProxyPort(443); - } else { - LOG.warn("Proxy host set without port. Using HTTP default 80"); - clientConf.setProxyPort(80); - } - } - String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY); - String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY); - if ((proxyUsername == null) != (proxyPassword == null)) { - String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " + - PROXY_PASSWORD_KEY + " set without the other."; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - clientConf.setProxyUsername(proxyUsername); - clientConf.setProxyPassword(proxyPassword); - clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY)); - clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY)); - } else if (proxyPort >= 0) { - String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " + - PROXY_HOST_KEY; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - - String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); - ossClient = - new OSSClient(endPoint, getCredentialsProvider(name, conf), clientConf); + workingDir = new Path("/user", + System.getProperty("user.name")).makeQualified(uri, null); + store = new AliyunOSSFileSystemStore(); + store.initialize(name, conf, statistics); maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); - uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, - MULTIPART_UPLOAD_SIZE_DEFAULT); - multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, - MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); - - if (uploadPartSize < 5 * 1024 * 1024) { - LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB"); - uploadPartSize = 5 * 1024 * 1024; - } - - if (multipartThreshold < 5 * 1024 * 1024) { - LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB"); - multipartThreshold = 5 * 1024 * 1024; - } - - if (multipartThreshold > 1024 * 1024 * 1024) { - LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB"); - multipartThreshold = 1024 * 1024 * 1024; - } - - String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); - if (!cannedACLName.isEmpty()) { - CannedAccessControlList cannedACL = - CannedAccessControlList.valueOf(cannedACLName); - ossClient.setBucketAcl(bucketName, cannedACL); - } - - serverSideEncryptionAlgorithm = - conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, ""); - setConf(conf); } - /** - * Create the default credential provider, or load in one explicitly - * identified in the configuration. - * @param name the uri of the file system - * @param conf configuration - * @return a credential provider - * @throws IOException on any problem. Class construction issues may be - * nested inside the IOE. - */ - private CredentialsProvider getCredentialsProvider(URI name, - Configuration conf) throws IOException { - CredentialsProvider credentials; - - String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); - if (StringUtils.isEmpty(className)) { - Configuration newConf = - ProviderUtils.excludeIncompatibleCredentialProviders(conf, - AliyunOSSFileSystem.class); - String accessKey = - AliyunOSSUtils.getPassword(newConf, ACCESS_KEY, - UserInfo.EMPTY.getUser()); - String secretKey = - AliyunOSSUtils.getPassword(newConf, SECRET_KEY, - UserInfo.EMPTY.getPassword()); - credentials = - new DefaultCredentialProvider( - new DefaultCredentials(accessKey, secretKey)); - - } else { - try { - LOG.debug("Credential provider class is:" + className); - Class credClass = Class.forName(className); - try { - credentials = - (CredentialsProvider)credClass.getDeclaredConstructor( - URI.class, Configuration.class).newInstance(this.uri, conf); - } catch (NoSuchMethodException | SecurityException e) { - credentials = - (CredentialsProvider)credClass.getDeclaredConstructor() - .newInstance(); - } - } catch (ClassNotFoundException e) { - throw new IOException(className + " not found.", e); - } catch (NoSuchMethodException | SecurityException e) { - throw new IOException(String.format("%s constructor exception. A " + - "class specified in %s must provide an accessible constructor " + - "accepting URI and Configuration, or an accessible default " + - "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), e); - } catch (ReflectiveOperationException | IllegalArgumentException e) { - throw new IOException(className + " instantiation exception.", e); - } - } - - return credentials; - } - /** * Check if OSS object represents a directory. * @@ -456,10 +255,10 @@ public class AliyunOSSFileSystem extends FileSystem { } /** - * Turns a path (relative or otherwise) into an OSS key. + * Turn a path (relative or otherwise) into an OSS key. * - * @param path the path of the file - * @return the key of the object that represent the file + * @param path the path of the file. + * @return the key of the object that represents the file. */ private String pathToKey(Path path) { if (!path.isAbsolute()) { @@ -492,18 +291,12 @@ public class AliyunOSSFileSystem extends FileSystem { key = key + "/"; } - ListObjectsRequest listObjectsRequest = - new ListObjectsRequest(bucketName); - listObjectsRequest.setPrefix(key); - listObjectsRequest.setDelimiter("/"); - listObjectsRequest.setMaxKeys(maxKeys); - if (LOG.isDebugEnabled()) { LOG.debug("listStatus: doing listObjects for directory " + key); } + ObjectListing objects = store.listObjects(key, maxKeys, "/", null); while (true) { - ObjectListing objects = ossClient.listObjects(listObjectsRequest); statistics.incrementReadOps(1); for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { Path keyPath = keyToPath(objectSummary.getKey()) @@ -539,7 +332,8 @@ public class AliyunOSSFileSystem extends FileSystem { if (LOG.isDebugEnabled()) { LOG.debug("listStatus: list truncated - getting next batch"); } - listObjectsRequest.setMarker(objects.getNextMarker()); + objects = store.listObjects(key, maxKeys, "/", + objects.getNextMarker()); statistics.incrementReadOps(1); } else { break; @@ -558,27 +352,17 @@ public class AliyunOSSFileSystem extends FileSystem { /** * Used to create an empty file that represents an empty directory. * - * @param bucket the bucket this directory belongs to * @param key directory path - * @return true if directory successfully created + * @return true if directory is successfully created * @throws IOException */ - private boolean mkdir(final String bucket, final String key) - throws IOException { + private boolean mkdir(final String key) throws IOException { String dirName = key; - ObjectMetadata dirMeta = new ObjectMetadata(); - byte[] buffer = new byte[0]; - ByteArrayInputStream in = new ByteArrayInputStream(buffer); - dirMeta.setContentLength(0); if (!key.endsWith("/")) { dirName += "/"; } - try { - ossClient.putObject(bucket, dirName, in, dirMeta); - return true; - } finally { - in.close(); - } + store.storeEmptyFile(dirName); + return true; } @Override @@ -595,14 +379,14 @@ public class AliyunOSSFileSystem extends FileSystem { } catch (FileNotFoundException e) { validatePath(path); String key = pathToKey(path); - return mkdir(bucketName, key); + return mkdir(key); } } /** * Check whether the path is a valid path. * - * @param path the path to be checked + * @param path the path to be checked. * @throws IOException */ private void validatePath(Path path) throws IOException { @@ -631,8 +415,8 @@ public class AliyunOSSFileSystem extends FileSystem { " because it is a directory"); } - return new FSDataInputStream(new AliyunOSSInputStream(getConf(), ossClient, - bucketName, pathToKey(path), fileStatus.getLen(), statistics)); + return new FSDataInputStream(new AliyunOSSInputStream(getConf(), store, + pathToKey(path), fileStatus.getLen(), statistics)); } @Override @@ -696,126 +480,31 @@ public class AliyunOSSFileSystem extends FileSystem { } else { copyFile(srcPath, dstPath); } - if (srcPath.equals(dstPath)) { - return true; - } else { - return delete(srcPath, true); - } + + return srcPath.equals(dstPath) || delete(srcPath, true); } /** * Copy file from source path to destination path. - * (the caller should make sure srcPath is a file and dstPath is valid.) + * (the caller should make sure srcPath is a file and dstPath is valid) * - * @param srcPath source path - * @param dstPath destination path - * @return true if successfully copied + * @param srcPath source path. + * @param dstPath destination path. + * @return true if file is successfully copied. */ private boolean copyFile(Path srcPath, Path dstPath) { String srcKey = pathToKey(srcPath); String dstKey = pathToKey(dstPath); - return copyFile(srcKey, dstKey); - } - - /** - * Copy an object from source key to destination key. - * - * @param srcKey source key - * @param dstKey destination key - * @return true if successfully copied - */ - private boolean copyFile(String srcKey, String dstKey) { - ObjectMetadata objectMeta = - ossClient.getObjectMetadata(bucketName, srcKey); - long dataLen = objectMeta.getContentLength(); - if (dataLen <= multipartThreshold) { - return singleCopy(srcKey, dstKey); - } else { - return multipartCopy(srcKey, dataLen, dstKey); - } - } - - /** - * Use single copy to copy an oss object. - * - * @param srcKey source key - * @param dstKey destination key - * @return true if successfully copied - * (the caller should make sure srcPath is a file and dstPath is valid) - */ - private boolean singleCopy(String srcKey, String dstKey) { - CopyObjectResult copyResult = - ossClient.copyObject(bucketName, srcKey, bucketName, dstKey); - LOG.debug(copyResult.getETag()); - return true; - } - - /** - * Use multipart copy to copy an oss object. - * (the caller should make sure srcPath is a file and dstPath is valid) - * - * @param srcKey source key - * @param dataLen data size of the object to copy - * @param dstKey destination key - * @return true if successfully copied, or false if upload is aborted - */ - private boolean multipartCopy(String srcKey, long dataLen, String dstKey) { - int partNum = (int)(dataLen / uploadPartSize); - if (dataLen % uploadPartSize != 0) { - partNum++; - } - InitiateMultipartUploadRequest initiateMultipartUploadRequest = - new InitiateMultipartUploadRequest(bucketName, dstKey); - ObjectMetadata meta = new ObjectMetadata(); - if (!serverSideEncryptionAlgorithm.isEmpty()) { - meta.setServerSideEncryption(serverSideEncryptionAlgorithm); - } - initiateMultipartUploadRequest.setObjectMetadata(meta); - InitiateMultipartUploadResult initiateMultipartUploadResult = - ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); - String uploadId = initiateMultipartUploadResult.getUploadId(); - List partETags = new ArrayList(); - try { - for (int i = 0; i < partNum; i++) { - long skipBytes = uploadPartSize * i; - long size = (uploadPartSize < dataLen - skipBytes) ? - uploadPartSize : dataLen - skipBytes; - UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest(); - partCopyRequest.setSourceBucketName(bucketName); - partCopyRequest.setSourceKey(srcKey); - partCopyRequest.setBucketName(bucketName); - partCopyRequest.setKey(dstKey); - partCopyRequest.setUploadId(uploadId); - partCopyRequest.setPartSize(size); - partCopyRequest.setBeginIndex(skipBytes); - partCopyRequest.setPartNumber(i + 1); - UploadPartCopyResult partCopyResult = - ossClient.uploadPartCopy(partCopyRequest); - statistics.incrementWriteOps(1); - partETags.add(partCopyResult.getPartETag()); - } - CompleteMultipartUploadRequest completeMultipartUploadRequest = - new CompleteMultipartUploadRequest(bucketName, dstKey, - uploadId, partETags); - CompleteMultipartUploadResult completeMultipartUploadResult = - ossClient.completeMultipartUpload(completeMultipartUploadRequest); - LOG.debug(completeMultipartUploadResult.getETag()); - return true; - } catch (OSSException | ClientException e) { - AbortMultipartUploadRequest abortMultipartUploadRequest = - new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); - ossClient.abortMultipartUpload(abortMultipartUploadRequest); - return false; - } + return store.copyFile(srcKey, dstKey); } /** * Copy a directory from source path to destination path. * (the caller should make sure srcPath is a directory, and dstPath is valid) * - * @param srcPath source path - * @param dstPath destination path - * @return true if successfully copied + * @param srcPath source path. + * @param dstPath destination path. + * @return true if directory is successfully copied. */ private boolean copyDirectory(Path srcPath, Path dstPath) { String srcKey = pathToKey(srcPath); @@ -835,21 +524,18 @@ public class AliyunOSSFileSystem extends FileSystem { return false; } - ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName); - listObjectsRequest.setPrefix(srcKey); - listObjectsRequest.setMaxKeys(maxKeys); - - ObjectListing objects = ossClient.listObjects(listObjectsRequest); + ObjectListing objects = store.listObjects(srcKey, maxKeys, null, null); statistics.incrementReadOps(1); // Copy files from src folder to dst while (true) { for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { String newKey = dstKey.concat(objectSummary.getKey().substring(srcKey.length())); - copyFile(objectSummary.getKey(), newKey); + store.copyFile(objectSummary.getKey(), newKey); } if (objects.isTruncated()) { - listObjectsRequest.setMarker(objects.getNextMarker()); + objects = store.listObjects(srcKey, maxKeys, null, + objects.getNextMarker()); statistics.incrementReadOps(1); } else { break; @@ -863,4 +549,7 @@ public class AliyunOSSFileSystem extends FileSystem { this.workingDir = dir; } + public AliyunOSSFileSystemStore getStore() { + return store; + } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java new file mode 100644 index 00000000000..b3cd1bdd144 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -0,0 +1,486 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.comm.Protocol; +import com.aliyun.oss.model.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Core implementation of Aliyun OSS Filesystem for Hadoop. + * Provides the bridging logic between Hadoop's abstract filesystem and + * Aliyun OSS. + */ +public class AliyunOSSFileSystemStore { + public static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSFileSystemStore.class); + private FileSystem.Statistics statistics; + private OSSClient ossClient; + private String bucketName; + private long uploadPartSize; + private long multipartThreshold; + private long partSize; + private int maxKeys; + private String serverSideEncryptionAlgorithm; + + public void initialize(URI uri, Configuration conf, + FileSystem.Statistics stat) throws IOException { + statistics = stat; + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY, + MAXIMUM_CONNECTIONS_DEFAULT)); + boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY, + SECURE_CONNECTIONS_DEFAULT); + clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); + clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY, + MAX_ERROR_RETRIES_DEFAULT)); + clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY, + ESTABLISH_TIMEOUT_DEFAULT)); + clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY, + SOCKET_TIMEOUT_DEFAULT)); + + String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); + int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); + if (!proxyHost.isEmpty()) { + clientConf.setProxyHost(proxyHost); + if (proxyPort >= 0) { + clientConf.setProxyPort(proxyPort); + } else { + if (secureConnections) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + clientConf.setProxyPort(443); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + clientConf.setProxyPort(80); + } + } + String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY); + String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " + + PROXY_PASSWORD_KEY + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + clientConf.setProxyUsername(proxyUsername); + clientConf.setProxyPassword(proxyPassword); + clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY)); + clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY)); + } else if (proxyPort >= 0) { + String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " + + PROXY_HOST_KEY; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + + String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); + CredentialsProvider provider = + AliyunOSSUtils.getCredentialsProvider(uri, conf); + ossClient = new OSSClient(endPoint, provider, clientConf); + uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, + MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); + partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) { + partSize = MIN_MULTIPART_UPLOAD_PART_SIZE; + } + serverSideEncryptionAlgorithm = + conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, ""); + + if (uploadPartSize < 5 * 1024 * 1024) { + LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB"); + uploadPartSize = 5 * 1024 * 1024; + } + + if (multipartThreshold < 5 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB"); + multipartThreshold = 5 * 1024 * 1024; + } + + if (multipartThreshold > 1024 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB"); + multipartThreshold = 1024 * 1024 * 1024; + } + + String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); + if (!cannedACLName.isEmpty()) { + CannedAccessControlList cannedACL = + CannedAccessControlList.valueOf(cannedACLName); + ossClient.setBucketAcl(bucketName, cannedACL); + } + + maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + bucketName = uri.getHost(); + } + + /** + * Delete an object, and update write operation statistics. + * + * @param key key to blob to delete. + */ + public void deleteObject(String key) { + ossClient.deleteObject(bucketName, key); + statistics.incrementWriteOps(1); + } + + /** + * Delete a list of keys, and update write operation statistics. + * + * @param keysToDelete collection of keys to delete. + */ + public void deleteObjects(List keysToDelete) { + DeleteObjectsRequest deleteRequest = + new DeleteObjectsRequest(bucketName); + deleteRequest.setKeys(keysToDelete); + ossClient.deleteObjects(deleteRequest); + statistics.incrementWriteOps(keysToDelete.size()); + } + + /** + * Delete a directory from Aliyun OSS. + * + * @param key directory key to delete. + */ + public void deleteDirs(String key) { + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(key); + listRequest.setMaxKeys(maxKeys); + + while (true) { + ObjectListing objects = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + List keysToDelete = new ArrayList(); + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + keysToDelete.add(objectSummary.getKey()); + } + deleteObjects(keysToDelete); + if (objects.isTruncated()) { + listRequest.setMarker(objects.getNextMarker()); + } else { + break; + } + } + } + + /** + * Return metadata of a given object key. + * + * @param key object key. + * @return return null if key does not exist. + */ + public ObjectMetadata getObjectMetadata(String key) { + try { + return ossClient.getObjectMetadata(bucketName, key); + } catch (OSSException osse) { + return null; + } finally { + statistics.incrementReadOps(1); + } + } + + /** + * Upload an empty file as an OSS object, using single upload. + * + * @param key object key. + * @throws IOException if failed to upload object. + */ + public void storeEmptyFile(String key) throws IOException { + ObjectMetadata dirMeta = new ObjectMetadata(); + byte[] buffer = new byte[0]; + ByteArrayInputStream in = new ByteArrayInputStream(buffer); + dirMeta.setContentLength(0); + try { + ossClient.putObject(bucketName, key, in, dirMeta); + } finally { + in.close(); + } + } + + /** + * Copy an object from source key to destination key. + * + * @param srcKey source key. + * @param dstKey destination key. + * @return true if file is successfully copied. + */ + public boolean copyFile(String srcKey, String dstKey) { + ObjectMetadata objectMeta = + ossClient.getObjectMetadata(bucketName, srcKey); + long contentLength = objectMeta.getContentLength(); + if (contentLength <= multipartThreshold) { + return singleCopy(srcKey, dstKey); + } else { + return multipartCopy(srcKey, contentLength, dstKey); + } + } + + /** + * Use single copy to copy an OSS object. + * (The caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcKey source key. + * @param dstKey destination key. + * @return true if object is successfully copied. + */ + private boolean singleCopy(String srcKey, String dstKey) { + CopyObjectResult copyResult = + ossClient.copyObject(bucketName, srcKey, bucketName, dstKey); + LOG.debug(copyResult.getETag()); + return true; + } + + /** + * Use multipart copy to copy an OSS object. + * (The caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcKey source key. + * @param contentLength data size of the object to copy. + * @param dstKey destination key. + * @return true if success, or false if upload is aborted. + */ + private boolean multipartCopy(String srcKey, long contentLength, + String dstKey) { + long realPartSize = + AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize); + int partNum = (int) (contentLength / realPartSize); + if (contentLength % realPartSize != 0) { + partNum++; + } + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, dstKey); + ObjectMetadata meta = new ObjectMetadata(); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + String uploadId = initiateMultipartUploadResult.getUploadId(); + List partETags = new ArrayList(); + try { + for (int i = 0; i < partNum; i++) { + long skipBytes = realPartSize * i; + long size = (realPartSize < contentLength - skipBytes) ? + realPartSize : contentLength - skipBytes; + UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest(); + partCopyRequest.setSourceBucketName(bucketName); + partCopyRequest.setSourceKey(srcKey); + partCopyRequest.setBucketName(bucketName); + partCopyRequest.setKey(dstKey); + partCopyRequest.setUploadId(uploadId); + partCopyRequest.setPartSize(size); + partCopyRequest.setBeginIndex(skipBytes); + partCopyRequest.setPartNumber(i + 1); + UploadPartCopyResult partCopyResult = + ossClient.uploadPartCopy(partCopyRequest); + statistics.incrementWriteOps(1); + partETags.add(partCopyResult.getPartETag()); + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, dstKey, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + return true; + } catch (OSSException | ClientException e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + return false; + } + } + + /** + * Upload a file as an OSS object, using single upload. + * + * @param key object key. + * @param file local file to upload. + * @throws IOException if failed to upload object. + */ + public void uploadObject(String key, File file) throws IOException { + File object = file.getAbsoluteFile(); + FileInputStream fis = new FileInputStream(object); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(object.length()); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + try { + PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta); + LOG.debug(result.getETag()); + statistics.incrementWriteOps(1); + } finally { + fis.close(); + } + } + + /** + * Upload a file as an OSS object, using multipart upload. + * + * @param key object key. + * @param file local file to upload. + * @throws IOException if failed to upload object. + */ + public void multipartUploadObject(String key, File file) throws IOException { + File object = file.getAbsoluteFile(); + long dataLen = object.length(); + long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize); + int partNum = (int) (dataLen / realPartSize); + if (dataLen % realPartSize != 0) { + partNum += 1; + } + + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, key); + ObjectMetadata meta = new ObjectMetadata(); + if (!serverSideEncryptionAlgorithm.isEmpty()) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + List partETags = new ArrayList(); + String uploadId = initiateMultipartUploadResult.getUploadId(); + + try { + for (int i = 0; i < partNum; i++) { + // TODO: Optimize this, avoid opening the object multiple times + FileInputStream fis = new FileInputStream(object); + try { + long skipBytes = realPartSize * i; + AliyunOSSUtils.skipFully(fis, skipBytes); + long size = (realPartSize < dataLen - skipBytes) ? + realPartSize : dataLen - skipBytes; + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(key); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(fis); + uploadPartRequest.setPartSize(size); + uploadPartRequest.setPartNumber(i + 1); + UploadPartResult uploadPartResult = + ossClient.uploadPart(uploadPartRequest); + statistics.incrementWriteOps(1); + partETags.add(uploadPartResult.getPartETag()); + } finally { + fis.close(); + } + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, key, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + } catch (OSSException | ClientException e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, key, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + } + } + + /** + * list objects. + * + * @param prefix prefix. + * @param maxListingLength max no. of entries + * @param delimiter delimiter. + * @param marker last key in any previous search. + * @return a list of matches. + */ + public ObjectListing listObjects(String prefix, int maxListingLength, + String delimiter, String marker) { + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(prefix); + listRequest.setDelimiter(delimiter); + listRequest.setMaxKeys(maxListingLength); + listRequest.setMarker(marker); + + ObjectListing listing = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + return listing; + } + + /** + * Retrieve a part of an object. + * + * @param key the object name that is being retrieved from the Aliyun OSS. + * @param byteStart start position. + * @param byteEnd end position. + * @return This method returns null if the key is not found. + */ + public InputStream retrieve(String key, long byteStart, long byteEnd) { + try { + GetObjectRequest request = new GetObjectRequest(bucketName, key); + request.setRange(byteStart, byteEnd); + return ossClient.getObject(request).getObjectContent(); + } catch (OSSException | ClientException e) { + return null; + } + } + + /** + * Close OSS client properly. + */ + public void close() { + if (ossClient != null) { + ossClient.shutdown(); + ossClient = null; + } + } + + /** + * Clean up all objects matching the prefix. + * + * @param prefix Aliyun OSS object prefix. + */ + public void purge(String prefix) { + String key; + try { + ObjectListing objects = listObjects(prefix, maxKeys, null, null); + for (OSSObjectSummary object : objects.getObjectSummaries()) { + key = object.getKey(); + ossClient.deleteObject(bucketName, key); + } + + for (String dir: objects.getCommonPrefixes()) { + deleteDirs(dir); + } + } catch (OSSException | ClientException e) { + LOG.error("Failed to purge " + prefix); + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java index b12e3f0ca57..69265fb81d4 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -27,12 +27,10 @@ import java.io.InputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem.Statistics; -import com.aliyun.oss.OSSClient; -import com.aliyun.oss.model.GetObjectRequest; - /** * The input stream for OSS blob system. * The class uses multi-part downloading to read data from the object content @@ -40,27 +38,23 @@ import com.aliyun.oss.model.GetObjectRequest; */ public class AliyunOSSInputStream extends FSInputStream { public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class); - private static final int MAX_RETRIES = 10; private final long downloadPartSize; - - private String bucketName; - private String key; - private OSSClient ossClient; + private AliyunOSSFileSystemStore store; + private final String key; private Statistics statistics; private boolean closed; private InputStream wrappedStream = null; - private long dataLen; + private long contentLength; private long position; private long partRemaining; - public AliyunOSSInputStream(Configuration conf, OSSClient client, - String bucketName, String key, Long dataLen, Statistics statistics) - throws IOException { - this.bucketName = bucketName; + public AliyunOSSInputStream(Configuration conf, + AliyunOSSFileSystemStore store, String key, Long contentLength, + Statistics statistics) throws IOException { + this.store = store; this.key = key; - ossClient = client; this.statistics = statistics; - this.dataLen = dataLen; + this.contentLength = contentLength; downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY, MULTIPART_DOWNLOAD_SIZE_DEFAULT); reopen(0); @@ -75,18 +69,17 @@ public class AliyunOSSInputStream extends FSInputStream { * @throws IOException if failed to reopen */ private synchronized void reopen(long pos) throws IOException { - - long partLen; + long partSize; if (pos < 0) { - throw new EOFException("Cannot seek at negtive position:" + pos); - } else if (pos > dataLen) { - throw new EOFException("Cannot seek after EOF, fileLen:" + dataLen + - " position:" + pos); - } else if (pos + downloadPartSize > dataLen) { - partLen = dataLen - pos; + throw new EOFException("Cannot seek at negative position:" + pos); + } else if (pos > contentLength) { + throw new EOFException("Cannot seek after EOF, contentLength:" + + contentLength + " position:" + pos); + } else if (pos + downloadPartSize > contentLength) { + partSize = contentLength - pos; } else { - partLen = downloadPartSize; + partSize = downloadPartSize; } if (wrappedStream != null) { @@ -96,21 +89,19 @@ public class AliyunOSSInputStream extends FSInputStream { wrappedStream.close(); } - GetObjectRequest request = new GetObjectRequest(bucketName, key); - request.setRange(pos, pos + partLen - 1); - wrappedStream = ossClient.getObject(request).getObjectContent(); + wrappedStream = store.retrieve(key, pos, pos + partSize -1); if (wrappedStream == null) { throw new IOException("Null IO stream"); } position = pos; - partRemaining = partLen; + partRemaining = partSize; } @Override public synchronized int read() throws IOException { checkNotClosed(); - if (partRemaining <= 0 && position < dataLen) { + if (partRemaining <= 0 && position < contentLength) { reopen(position); } @@ -139,13 +130,14 @@ public class AliyunOSSInputStream extends FSInputStream { /** - * Check whether the input stream is closed. + * Verify that the input stream is open. Non blocking; this gives + * the last state of the volatile {@link #closed} field. * - * @throws IOException if stream is closed + * @throws IOException if the connection is closed. */ private void checkNotClosed() throws IOException { if (closed) { - throw new IOException("Stream is closed!"); + throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); } } @@ -164,7 +156,7 @@ public class AliyunOSSInputStream extends FSInputStream { int bytesRead = 0; // Not EOF, and read not done - while (position < dataLen && bytesRead < len) { + while (position < contentLength && bytesRead < len) { if (partRemaining == 0) { reopen(position); } @@ -219,7 +211,7 @@ public class AliyunOSSInputStream extends FSInputStream { public synchronized int available() throws IOException { checkNotClosed(); - long remaining = dataLen - position; + long remaining = contentLength - position; if (remaining > Integer.MAX_VALUE) { return Integer.MAX_VALUE; } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java index 1e16df9edb7..c952d0ae858 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -22,15 +22,10 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*; import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; -import com.aliyun.oss.ClientException; -import com.aliyun.oss.OSSException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -38,18 +33,6 @@ import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.util.Progressable; -import com.aliyun.oss.OSSClient; -import com.aliyun.oss.model.AbortMultipartUploadRequest; -import com.aliyun.oss.model.CompleteMultipartUploadRequest; -import com.aliyun.oss.model.CompleteMultipartUploadResult; -import com.aliyun.oss.model.InitiateMultipartUploadRequest; -import com.aliyun.oss.model.InitiateMultipartUploadResult; -import com.aliyun.oss.model.ObjectMetadata; -import com.aliyun.oss.model.PartETag; -import com.aliyun.oss.model.PutObjectResult; -import com.aliyun.oss.model.UploadPartRequest; -import com.aliyun.oss.model.UploadPartResult; - /** * The output stream for OSS blob system. * Data will be buffered on local disk, then uploaded to OSS in @@ -57,36 +40,24 @@ import com.aliyun.oss.model.UploadPartResult; */ public class AliyunOSSOutputStream extends OutputStream { public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class); - private String bucketName; - private String key; + private AliyunOSSFileSystemStore store; + private final String key; private Statistics statistics; private Progressable progress; - private String serverSideEncryptionAlgorithm; - private long partSize; private long partSizeThreshold; private LocalDirAllocator dirAlloc; private boolean closed; private File tmpFile; private BufferedOutputStream backupStream; - private OSSClient ossClient; - public AliyunOSSOutputStream(Configuration conf, OSSClient client, - String bucketName, String key, Progressable progress, - Statistics statistics, String serverSideEncryptionAlgorithm) - throws IOException { - this.bucketName = bucketName; + public AliyunOSSOutputStream(Configuration conf, + AliyunOSSFileSystemStore store, String key, Progressable progress, + Statistics statistics) throws IOException { + this.store = store; this.key = key; // The caller cann't get any progress information this.progress = progress; - ossClient = client; this.statistics = statistics; - this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; - - partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, - MULTIPART_UPLOAD_SIZE_DEFAULT); - if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) { - partSize = MIN_MULTIPART_UPLOAD_PART_SIZE; - } partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); @@ -113,9 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream { long dataLen = tmpFile.length(); try { if (dataLen <= partSizeThreshold) { - uploadObject(); + store.uploadObject(key, tmpFile); } else { - multipartUploadObject(); + store.multipartUploadObject(key, tmpFile); } } finally { if (!tmpFile.delete()) { @@ -124,91 +95,7 @@ public class AliyunOSSOutputStream extends OutputStream { } } - /** - * Upload temporary file as an OSS object, using single upload. - * - * @throws IOException - */ - private void uploadObject() throws IOException { - File object = tmpFile.getAbsoluteFile(); - FileInputStream fis = new FileInputStream(object); - ObjectMetadata meta = new ObjectMetadata(); - meta.setContentLength(object.length()); - if (!serverSideEncryptionAlgorithm.isEmpty()) { - meta.setServerSideEncryption(serverSideEncryptionAlgorithm); - } - try { - PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta); - LOG.debug(result.getETag()); - statistics.incrementWriteOps(1); - } finally { - fis.close(); - } - } - /** - * Upload temporary file as an OSS object, using multipart upload. - * - * @throws IOException - */ - private void multipartUploadObject() throws IOException { - File object = tmpFile.getAbsoluteFile(); - long dataLen = object.length(); - long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize); - int partNum = (int)(dataLen / realPartSize); - if (dataLen % realPartSize != 0) { - partNum += 1; - } - - InitiateMultipartUploadRequest initiateMultipartUploadRequest = - new InitiateMultipartUploadRequest(bucketName, key); - ObjectMetadata meta = new ObjectMetadata(); - // meta.setContentLength(dataLen); - if (!serverSideEncryptionAlgorithm.isEmpty()) { - meta.setServerSideEncryption(serverSideEncryptionAlgorithm); - } - initiateMultipartUploadRequest.setObjectMetadata(meta); - InitiateMultipartUploadResult initiateMultipartUploadResult = - ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); - List partETags = new ArrayList(); - String uploadId = initiateMultipartUploadResult.getUploadId(); - - try { - for (int i = 0; i < partNum; i++) { - // TODO: Optimize this, avoid opening the object multiple times - FileInputStream fis = new FileInputStream(object); - try { - long skipBytes = realPartSize * i; - AliyunOSSUtils.skipFully(fis, skipBytes); - long size = (realPartSize < dataLen - skipBytes) ? - realPartSize : dataLen - skipBytes; - UploadPartRequest uploadPartRequest = new UploadPartRequest(); - uploadPartRequest.setBucketName(bucketName); - uploadPartRequest.setKey(key); - uploadPartRequest.setUploadId(uploadId); - uploadPartRequest.setInputStream(fis); - uploadPartRequest.setPartSize(size); - uploadPartRequest.setPartNumber(i + 1); - UploadPartResult uploadPartResult = - ossClient.uploadPart(uploadPartRequest); - statistics.incrementWriteOps(1); - partETags.add(uploadPartResult.getPartETag()); - } finally { - fis.close(); - } - } - CompleteMultipartUploadRequest completeMultipartUploadRequest = - new CompleteMultipartUploadRequest(bucketName, key, - uploadId, partETags); - CompleteMultipartUploadResult completeMultipartUploadResult = - ossClient.completeMultipartUpload(completeMultipartUploadRequest); - LOG.debug(completeMultipartUploadResult.getETag()); - } catch (OSSException | ClientException e) { - AbortMultipartUploadRequest abortMultipartUploadRequest = - new AbortMultipartUploadRequest(bucketName, key, uploadId); - ossClient.abortMultipartUpload(abortMultipartUploadRequest); - } - } @Override public synchronized void flush() throws IOException { diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index d54dd9c44ae..b96aea7f86e 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -20,142 +20,58 @@ package org.apache.hadoop.fs.aliyun.oss; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.net.URI; -import java.net.URLDecoder; -import java.util.Objects; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.ProviderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT; +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; +import static org.apache.hadoop.fs.aliyun.oss.Constants.ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY; /** * Utility methods for Aliyun OSS code. */ final public class AliyunOSSUtils { + private static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSUtils.class); + private AliyunOSSUtils() { } /** - * User information includes user name and password. - */ - static public class UserInfo { - private final String user; - private final String password; - - public static final UserInfo EMPTY = new UserInfo("", ""); - - public UserInfo(String user, String password) { - this.user = user; - this.password = password; - } - - /** - * Predicate to verify user information is set. - * @return true if the username is defined (not null, not empty). - */ - public boolean hasLogin() { - return StringUtils.isNotEmpty(user); - } - - /** - * Equality test matches user and password. - * @param o other object - * @return true if the objects are considered equivalent. - */ - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - UserInfo that = (UserInfo) o; - return Objects.equals(user, that.user) && - Objects.equals(password, that.password); - } - - @Override - public int hashCode() { - return Objects.hash(user, password); - } - - public String getUser() { - return user; - } - - public String getPassword() { - return password; - } - } - - /** - * Used to get password from configuration, if default value is not available. + * Used to get password from configuration. + * * @param conf configuration that contains password information * @param key the key of the password - * @param val the default value of the key * @return the value for the key * @throws IOException if failed to get password from configuration */ - static public String getPassword(Configuration conf, String key, String val) + static public String getPassword(Configuration conf, String key) throws IOException { - if (StringUtils.isEmpty(val)) { - try { - final char[] pass = conf.getPassword(key); - if (pass != null) { - return (new String(pass)).trim(); - } else { - return ""; - } - } catch (IOException ioe) { - throw new IOException("Cannot find password option " + key, ioe); - } - } else { - return val; - } - } - - /** - * Extract the user information details from a URI. - * @param name URI of the filesystem. - * @return a login tuple, possibly empty. - */ - public static UserInfo extractLoginDetails(URI name) { try { - String authority = name.getAuthority(); - if (authority == null) { - return UserInfo.EMPTY; - } - int loginIndex = authority.indexOf('@'); - if (loginIndex < 0) { - // No user information - return UserInfo.EMPTY; - } - String login = authority.substring(0, loginIndex); - int loginSplit = login.indexOf(':'); - if (loginSplit > 0) { - String user = login.substring(0, loginSplit); - String password = URLDecoder.decode(login.substring(loginSplit + 1), - "UTF-8"); - return new UserInfo(user, password); - } else if (loginSplit == 0) { - // There is no user, just a password. - return UserInfo.EMPTY; + final char[] pass = conf.getPassword(key); + if (pass != null) { + return (new String(pass)).trim(); } else { - return new UserInfo(login, ""); + return ""; } - } catch (UnsupportedEncodingException e) { - // This should never happen; translate it if it does. - throw new RuntimeException(e); + } catch (IOException ioe) { + throw new IOException("Cannot find password option " + key, ioe); } } /** - * Skips the requested number of bytes or fail if there are not enough left. - * This allows for the possibility that {@link InputStream#skip(long)} may not - * skip as many bytes as requested (most likely because of reaching EOF). + * Skip the requested number of bytes or fail if there are no enough bytes + * left. This allows for the possibility that {@link InputStream#skip(long)} + * may not skip as many bytes as requested (most likely because of reaching + * EOF). + * * @param is the input stream to skip. * @param n the number of bytes to skip. * @throws IOException thrown when skipped less number of bytes. @@ -179,12 +95,69 @@ final public class AliyunOSSUtils { * Calculate a proper size of multipart piece. If minPartSize * is too small, the number of multipart pieces may exceed the limit of * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}. + * * @param contentLength the size of file. * @param minPartSize the minimum size of multipart piece. * @return a revisional size of multipart piece. - */ + */ public static long calculatePartSize(long contentLength, long minPartSize) { long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1; return Math.max(minPartSize, tmpPartSize); } + + /** + * Create credential provider specified by configuration, or create default + * credential provider if not specified. + * + * @param name the uri of the file system + * @param conf configuration + * @return a credential provider + * @throws IOException on any problem. Class construction issues may be + * nested inside the IOE. + */ + public static CredentialsProvider getCredentialsProvider(URI name, + Configuration conf) throws IOException { + URI uri = java.net.URI.create( + name.getScheme() + "://" + name.getAuthority()); + CredentialsProvider credentials; + + String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); + if (StringUtils.isEmpty(className)) { + Configuration newConf = + ProviderUtils.excludeIncompatibleCredentialProviders(conf, + AliyunOSSFileSystem.class); + String accessKey = + AliyunOSSUtils.getPassword(newConf, ACCESS_KEY); + String secretKey = + AliyunOSSUtils.getPassword(newConf, SECRET_KEY); + credentials = new DefaultCredentialProvider( + new DefaultCredentials(accessKey, secretKey)); + } else { + try { + LOG.debug("Credential provider class is:" + className); + Class credClass = Class.forName(className); + try { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor( + URI.class, Configuration.class).newInstance(uri, conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } + } catch (ClassNotFoundException e) { + throw new IOException(className + " not found.", e); + } catch (NoSuchMethodException | SecurityException e) { + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), + e); + } catch (ReflectiveOperationException | IllegalArgumentException e) { + throw new IOException(className + " instantiation exception.", e); + } + } + + return credentials; + } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 99022754fbe..243fdd4c0e1 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -72,7 +72,7 @@ public final class Constants { // Number of records to get while paging through a directory listing public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum"; - public static final int MAX_PAGING_KEYS_DEFAULT = 500; + public static final int MAX_PAGING_KEYS_DEFAULT = 1000; // Size of each of or multipart pieces in bytes public static final String MULTIPART_UPLOAD_SIZE_KEY = @@ -109,5 +109,6 @@ public final class Constants { public static final String FS_OSS = "oss"; public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L; + public static final int MAX_RETRIES = 10; } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java index de4e5a93152..f234d508e75 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java @@ -73,11 +73,6 @@ public class TestOSSFileSystemContract extends FileSystemContractBaseTest { // not supported } - /** - * Assert that root directory renames are not allowed. - * - * @throws Exception on failures - */ @Override public void testRootDirAlwaysExists() throws Exception { //this will throw an exception if the path is not found @@ -88,11 +83,6 @@ public class TestOSSFileSystemContract extends FileSystemContractBaseTest { fs.exists(super.path("/"))); } - /** - * Assert that root directory renames are not allowed. - * - * @throws Exception on failures - */ @Override public void testRenameRootDirForbidden() throws Exception { if (!renameSupported()) { diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java new file mode 100644 index 00000000000..6331ed829f8 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.*; +import java.net.URI; +import java.security.DigestInputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; + +/** + * Test the bridging logic between Hadoop's abstract filesystem and + * Aliyun OSS. + */ +public class TestOSSFileSystemStore { + private Configuration conf; + private AliyunOSSFileSystemStore store; + private AliyunOSSFileSystem fs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + fs = new AliyunOSSFileSystem(); + fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf); + store = fs.getStore(); + } + + @After + public void tearDown() throws Exception { + try { + store.purge("test"); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + @BeforeClass + public static void checkSettings() throws Exception { + Configuration conf = new Configuration(); + assumeNotNull(conf.get("fs.oss.accessKeyId")); + assumeNotNull(conf.get("fs.oss.accessKeySecret")); + assumeNotNull(conf.get("test.fs.oss.name")); + } + + protected void writeRenameReadCompare(Path path, long len) + throws IOException, NoSuchAlgorithmException { + // If len > fs.oss.multipart.upload.threshold, + // we'll use a multipart upload copy + MessageDigest digest = MessageDigest.getInstance("MD5"); + OutputStream out = new BufferedOutputStream( + new DigestOutputStream(fs.create(path, false), digest)); + for (long i = 0; i < len; i++) { + out.write('Q'); + } + out.flush(); + out.close(); + + assertTrue("Exists", fs.exists(path)); + + Path copyPath = path.suffix(".copy"); + fs.rename(path, copyPath); + + assertTrue("Copy exists", fs.exists(copyPath)); + + // Download file from Aliyun OSS and compare the digest against the original + MessageDigest digest2 = MessageDigest.getInstance("MD5"); + InputStream in = new BufferedInputStream( + new DigestInputStream(fs.open(copyPath), digest2)); + long copyLen = 0; + while (in.read() != -1) { + copyLen++; + } + in.close(); + + assertEquals("Copy length matches original", len, copyLen); + assertArrayEquals("Digests match", digest.digest(), digest2.digest()); + } + + @Test + public void testSmallUpload() throws IOException, NoSuchAlgorithmException { + // Regular upload, regular copy + writeRenameReadCompare(new Path("/test/small"), 16384); + } + + @Test + public void testLargeUpload() + throws IOException, NoSuchAlgorithmException { + // Multipart upload, multipart copy + writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java index 8214b9f6bec..f90a8bb5947 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.aliyun.oss.contract; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils; diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java new file mode 100644 index 00000000000..eb0c5e0cdd2 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Contract test suite covering Aliyun OSS integration with DistCp. + */ +public class TestOSSContractDispCp extends AbstractContractDistCpTest { + + private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB + + @Override + protected Configuration createConfiguration() { + Configuration newConf = super.createConfiguration(); + newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING); + newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING); + return newConf; + } + + @Override + protected OSSContract createContract(Configuration conf) { + return new OSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java new file mode 100644 index 00000000000..cc21a2e2ce2 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test getFileStatus and related listing operations. + */ +public class TestOSSContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java new file mode 100644 index 00000000000..cbc262c735d --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Root dir operations against an Aliyun OSS bucket. + */ +public class TestOSSContractRootDir extends + AbstractContractRootDirectoryTest { + + private static final Logger LOG = + LoggerFactory.getLogger(TestOSSContractRootDir.class); + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OSSContract(conf); + } + + @Override + public void testListEmptyRootDirectory() throws IOException { + for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) { + try { + super.testListEmptyRootDirectory(); + break; + } catch (AssertionError | FileNotFoundException e) { + if (attempt < maxAttempts) { + LOG.info("Attempt {} of {} for empty root directory test failed. " + + "Attempting retry.", attempt, maxAttempts); + try { + Thread.sleep(1000); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + fail("Test interrupted."); + break; + } + } else { + LOG.error( + "Empty root directory test failed {} attempts. Failing test.", + maxAttempts); + throw e; + } + } + } + } +} From e671a0f52b5488b8453e1a3258ea5e6477995648 Mon Sep 17 00:00:00 2001 From: Mingfei Date: Sun, 28 Aug 2016 10:37:52 +0800 Subject: [PATCH 07/13] HADOOP-13481. User documents for Aliyun OSS FileSystem. Contributed by Genmao Yu. --- .../hadoop/fs/aliyun/oss/Constants.java | 3 +- .../markdown/tools/hadoop-aliyun/index.md | 299 ++++++++++++++++++ 2 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 243fdd4c0e1..e0c05ed740f 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -95,8 +95,7 @@ public final class Constants { // Comma separated list of directories public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir"; - // private | public-read | public-read-write | authenticated-read | - // log-delivery-write | bucket-owner-read | bucket-owner-full-control + // private | public-read | public-read-write public static final String CANNED_ACL_KEY = "fs.oss.acl.default"; public static final String CANNED_ACL_DEFAULT = ""; diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md new file mode 100644 index 00000000000..4095e06fa57 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md @@ -0,0 +1,299 @@ + + +# Hadoop-Aliyun module: Integration with Aliyun Web Services + + + +## Overview + +The `hadoop-aliyun` module provides support for Aliyun integration with +[Aliyun Object Storage Service (Aliyun OSS)](https://www.aliyun.com/product/oss). +The generated JAR file, `hadoop-aliyun.jar` also declares a transitive +dependency on all external artifacts which are needed for this support — enabling +downstream applications to easily use this support. + +To make it part of Apache Hadoop's default classpath, simply make sure +that HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aliyun' in the list. + +### Features + +* Read and write data stored in Aliyun OSS. +* Present a hierarchical file system view by implementing the standard Hadoop +[`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface. +* Can act as a source of data in a MapReduce job, or a sink. + +### Warning #1: Object Stores are not filesystems. + +Aliyun OSS is an example of "an object store". In order to achieve scalability +and especially high availability, Aliyun OSS has relaxed some of the constraints +which classic "POSIX" filesystems promise. + + + +Specifically + +1. Atomic operations: `delete()` and `rename()` are implemented by recursive +file-by-file operations. They take time at least proportional to the number of files, +during which time partial updates may be visible. `delete()` and `rename()` +can not guarantee atomicity. If the operations are interrupted, the filesystem +is left in an intermediate state. +2. File owner and group are persisted, but the permissions model is not enforced. +Authorization occurs at the level of the entire Aliyun account via +[Aliyun Resource Access Management (Aliyun RAM)](https://www.aliyun.com/product/ram). +3. Directory last access time is not tracked. +4. The append operation is not supported. + +### Warning #2: Directory last access time is not tracked, +features of Hadoop relying on this can have unexpected behaviour. E.g. the +AggregatedLogDeletionService of YARN will not remove the appropriate logfiles. + +### Warning #3: Your Aliyun credentials are valuable + +Your Aliyun credentials not only pay for services, they offer read and write +access to the data. Anyone with the account can not only read your datasets +—they can delete them. + +Do not inadvertently share these credentials through means such as +1. Checking in to SCM any configuration files containing the secrets. +2. Logging them to a console, as they invariably end up being seen. +3. Defining filesystem URIs with the credentials in the URL, such as +`oss://accessKeyId:accessKeySecret@directory/file`. They will end up in +logs and error messages. +4. Including the secrets in bug reports. + +If you do any of these: change your credentials immediately! + +### Warning #4: The Aliyun OSS client provided by Aliyun E-MapReduce are different from this implementation + +Specifically: on Aliyun E-MapReduce, `oss://` is also supported but with +a different implementation. If you are using Aliyun E-MapReduce, +follow these instructions —and be aware that all issues related to Aliyun +OSS integration in E-MapReduce can only be addressed by Aliyun themselves: +please raise your issues with them. + +## OSS + +### Authentication properties + + + fs.oss.accessKeyId + Aliyun access key ID + + + + fs.oss.accessKeySecret + Aliyun access key secret + + + + fs.oss.credentials.provider + + Class name of a credentials provider that implements + com.aliyun.oss.common.auth.CredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + + + +### Other properties + + + fs.oss.endpoint + Aliyun OSS endpoint to connect to. An up-to-date list is + provided in the Aliyun OSS Documentation. + + + + + fs.oss.proxy.host + Hostname of the (optinal) proxy server for Aliyun OSS connection + + + + fs.oss.proxy.port + Proxy server port + + + + fs.oss.proxy.username + Username for authenticating with proxy server + + + + fs.oss.proxy.password + Password for authenticating with proxy server. + + + + fs.oss.proxy.domain + Domain for authenticating with proxy server. + + + + fs.oss.proxy.workstation + Workstation for authenticating with proxy server. + + + + fs.oss.attempts.maximum + 20 + How many times we should retry commands on transient errors. + + + + fs.oss.connection.establish.timeout + 50000 + Connection setup timeout in milliseconds. + + + + fs.oss.connection.timeout + 200000 + Socket connection timeout in milliseconds. + + + + fs.oss.paging.maximum + 500 + How many keys to request from Aliyun OSS when doing directory listings at a time. + + + + + fs.oss.multipart.upload.size + 10485760 + Size of each of multipart pieces in bytes. + + + + fs.oss.multipart.upload.threshold + 20971520 + Minimum size in bytes before we start a multipart uploads or copy. + + + + fs.oss.multipart.download.size + 102400/value> + Size in bytes in each request from ALiyun OSS. + + + + fs.oss.buffer.dir + Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS + + + + fs.oss.buffer.dir + Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS + + + + fs.oss.acl.default + + Set a canned ACL for bucket. Value may be private, public-read, public-read-write. + + + + + fs.oss.server-side-encryption-algorithm + + Specify a server-side encryption algorithm for oss: file system. + Unset by default, and the only other currently allowable value is AES256. + + + + + fs.oss.connection.maximum + 32 + Number of simultaneous connections to oss. + + + + fs.oss.connection.secure.enabled + true + Connect to oss over ssl or not, true by default. + + +## Testing the hadoop-aliyun Module + +To test `oss://` filesystem client, two files which pass in authentication +details to the test runner are needed. + +1. `auth-keys.xml` +2. `core-site.xml` + +Those two configuration files must be put into +`hadoop-tools/hadoop-aliyun/src/test/resources`. + +### `core-site.xml` + +This file pre-exists and sources the configurations created in `auth-keys.xml`. + +For most cases, no modification is needed, unless a specific, non-default property +needs to be set during the testing. + +### `auth-keys.xml` + +This file triggers the testing of Aliyun OSS module. Without this file, +*none of the tests in this module will be executed* + +It contains the access key Id/secret and proxy information that are needed to +connect to Aliyun OSS, and an OSS bucket URL should be also provided. + +1. `test.fs.oss.name` : the URL of the bucket for Aliyun OSS tests + +The contents of the bucket will be cleaned during the testing process, so +do not use the bucket for any purpose other than testing. + +### Run Hadoop contract tests +Create file `contract-test-options.xml` under `/test/resources`. If a +specific file `fs.contract.test.fs.oss` test path is not defined, those +tests will be skipped. Credentials are also needed to run any of those +tests, they can be copied from `auth-keys.xml` or through direct +XInclude inclusion. Here is an example of `contract-test-options.xml`: + + + + + + + + + fs.contract.test.fs.oss + oss://spark-tests + + + + fs.oss.impl + org.apache.hadoop.fs.aliyun.AliyunOSSFileSystem + + + + fs.oss.endpoint + oss-cn-hangzhou.aliyuncs.com + + + + fs.oss.buffer.dir + /tmp/oss + + + + fs.oss.multipart.download.size + 102400 + + From a1940464a498d1e662e5c3843f2d31ce63ec726b Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Sun, 18 Sep 2016 19:10:48 +0800 Subject: [PATCH 08/13] HADOOP-13610. Clean up AliyunOss integration tests. Contributed by Genmao Yu --- ...TestUtils.java => AliyunOSSTestUtils.java} | 19 ++++++++----------- ...a => TestAliyunOSSFileSystemContract.java} | 16 ++++++---------- ...java => TestAliyunOSSFileSystemStore.java} | 6 +++--- ...eam.java => TestAliyunOSSInputStream.java} | 11 ++++++----- ...am.java => TestAliyunOSSOutputStream.java} | 9 +++++---- ...=> TestAliyunOSSTemporaryCredentials.java} | 7 ++++--- ...SSContract.java => AliyunOSSContract.java} | 18 +++++++----------- ....java => TestAliyunOSSContractCreate.java} | 6 +++--- ....java => TestAliyunOSSContractDelete.java} | 6 +++--- ....java => TestAliyunOSSContractDispCp.java} | 6 +++--- ...> TestAliyunOSSContractGetFileStatus.java} | 4 ++-- ...r.java => TestAliyunOSSContractMkdir.java} | 6 +++--- ...en.java => TestAliyunOSSContractOpen.java} | 6 +++--- ....java => TestAliyunOSSContractRename.java} | 6 +++--- ...java => TestAliyunOSSContractRootDir.java} | 6 +++--- ...ek.java => TestAliyunOSSContractSeek.java} | 6 +++--- .../contract/{oss.xml => aliyun-oss.xml} | 0 17 files changed, 65 insertions(+), 73 deletions(-) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{OSSTestUtils.java => AliyunOSSTestUtils.java} (82%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestOSSFileSystemContract.java => TestAliyunOSSFileSystemContract.java} (95%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestOSSFileSystemStore.java => TestAliyunOSSFileSystemStore.java} (96%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestOSSInputStream.java => TestAliyunOSSInputStream.java} (93%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestOSSOutputStream.java => TestAliyunOSSOutputStream.java} (92%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestOSSTemporaryCredentials.java => TestAliyunOSSTemporaryCredentials.java} (92%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{OSSContract.java => AliyunOSSContract.java} (70%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractCreate.java => TestAliyunOSSContractCreate.java} (87%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractDelete.java => TestAliyunOSSContractDelete.java} (87%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractDispCp.java => TestAliyunOSSContractDispCp.java} (88%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractGetFileStatus.java => TestAliyunOSSContractGetFileStatus.java} (93%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractMkdir.java => TestAliyunOSSContractMkdir.java} (87%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractOpen.java => TestAliyunOSSContractOpen.java} (87%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractRename.java => TestAliyunOSSContractRename.java} (87%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractRootDir.java => TestAliyunOSSContractRootDir.java} (93%) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestOSSContractSeek.java => TestAliyunOSSContractSeek.java} (88%) rename hadoop-tools/hadoop-aliyun/src/test/resources/contract/{oss.xml => aliyun-oss.xml} (100%) diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java similarity index 82% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java index 37ed8319174..84dba6a2d6d 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java @@ -24,15 +24,13 @@ import org.junit.internal.AssumptionViolatedException; import java.io.IOException; import java.net.URI; -import java.util.Date; -import java.util.Random; /** - * Utility class for OSS Tests. + * Utility class for Aliyun OSS Tests. */ -public final class OSSTestUtils { +public final class AliyunOSSTestUtils { - private OSSTestUtils() { + private AliyunOSSTestUtils() { } /** @@ -48,7 +46,7 @@ public final class OSSTestUtils { public static AliyunOSSFileSystem createTestFileSystem(Configuration conf) throws IOException { String fsname = conf.getTrimmed( - TestOSSFileSystemContract.TEST_FS_OSS_NAME, ""); + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, ""); boolean liveTest = !StringUtils.isEmpty(fsname); URI testURI = null; @@ -59,7 +57,7 @@ public final class OSSTestUtils { if (!liveTest) { throw new AssumptionViolatedException("No test filesystem in " - + TestOSSFileSystemContract.TEST_FS_OSS_NAME); + + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME); } AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem(); ossfs.initialize(testURI, conf); @@ -72,9 +70,8 @@ public final class OSSTestUtils { * @return root test path */ public static String generateUniqueTestPath() { - Long time = new Date().getTime(); - Random rand = new Random(); - return "/test_" + Long.toString(time) + "_" - + Long.toString(Math.abs(rand.nextLong())); + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? "/test" : + "/" + testUniqueForkId + "/test"; } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java similarity index 95% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java index f234d508e75..ad8ef6ee3a4 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java @@ -22,32 +22,28 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystemContractBaseTest; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; /** - * Tests a live OSS system. + * Tests a live Aliyun OSS system. * * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from * TestCase which uses the old Junit3 runner that doesn't ignore assumptions * properly making it impossible to skip the tests if we don't have a valid * bucket. */ -public class TestOSSFileSystemContract extends FileSystemContractBaseTest { - - protected static final Logger LOG = - LoggerFactory.getLogger(TestOSSFileSystemContract.class); - +public class TestAliyunOSSFileSystemContract + extends FileSystemContractBaseTest { public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; - private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); @Override public void setUp() throws Exception { Configuration conf = new Configuration(); - fs = OSSTestUtils.createTestFileSystem(conf); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); super.setUp(); } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java similarity index 96% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java index 6331ed829f8..c8498042b6e 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java @@ -41,7 +41,7 @@ import static org.junit.Assume.assumeNotNull; * Test the bridging logic between Hadoop's abstract filesystem and * Aliyun OSS. */ -public class TestOSSFileSystemStore { +public class TestAliyunOSSFileSystemStore { private Configuration conf; private AliyunOSSFileSystemStore store; private AliyunOSSFileSystem fs; @@ -67,8 +67,8 @@ public class TestOSSFileSystemStore { @BeforeClass public static void checkSettings() throws Exception { Configuration conf = new Configuration(); - assumeNotNull(conf.get("fs.oss.accessKeyId")); - assumeNotNull(conf.get("fs.oss.accessKeySecret")); + assumeNotNull(conf.get(Constants.ACCESS_KEY)); + assumeNotNull(conf.get(Constants.SECRET_KEY)); assumeNotNull(conf.get("test.fs.oss.name")); } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java similarity index 93% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java index 411cd576f75..892eda062e4 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java @@ -37,14 +37,15 @@ import static org.junit.Assert.assertTrue; * Tests basic functionality for AliyunOSSInputStream, including seeking and * reading files. */ -public class TestOSSInputStream { +public class TestAliyunOSSInputStream { private FileSystem fs; - protected static final Logger LOG = - LoggerFactory.getLogger(TestOSSInputStream.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestAliyunOSSInputStream.class); - private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); @Rule public Timeout testTimeout = new Timeout(30 * 60 * 1000); @@ -52,7 +53,7 @@ public class TestOSSInputStream { @Before public void setUp() throws Exception { Configuration conf = new Configuration(); - fs = OSSTestUtils.createTestFileSystem(conf); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); } @After diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java similarity index 92% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java index b33ab99cfc3..6b87d9ca466 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java @@ -33,9 +33,10 @@ import java.io.IOException; /** * Tests regular and multi-part upload functionality for AliyunOSSOutputStream. */ -public class TestOSSOutputStream { +public class TestAliyunOSSOutputStream { private FileSystem fs; - private static String testRootPath = OSSTestUtils.generateUniqueTestPath(); + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); @Rule public Timeout testTimeout = new Timeout(30 * 60 * 1000); @@ -45,7 +46,7 @@ public class TestOSSOutputStream { Configuration conf = new Configuration(); conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024); conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024); - fs = OSSTestUtils.createTestFileSystem(conf); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); } @After @@ -56,7 +57,7 @@ public class TestOSSOutputStream { } protected Path getTestPath() { - return new Path(testRootPath + "/testoss"); + return new Path(testRootPath + "/test-aliyun-oss"); } @Test diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java similarity index 92% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java index ca2c0bc203d..e106aae184e 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java @@ -21,7 +21,7 @@ package org.apache.hadoop.fs.aliyun.oss; import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.InvalidCredentialsException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.aliyun.oss.contract.OSSContract; +import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.junit.Test; @@ -38,11 +38,12 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; * should only be used against transient filesystems where you don't care about * the data. */ -public class TestOSSTemporaryCredentials extends AbstractFSContractTestBase { +public class TestAliyunOSSTemporaryCredentials + extends AbstractFSContractTestBase { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } @Test diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java similarity index 70% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java index f90a8bb5947..624c606c6b8 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java @@ -20,21 +20,16 @@ package org.apache.hadoop.fs.aliyun.oss.contract; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; /** - * The contract of OSS: only enabled if the test bucket is provided. + * The contract of Aliyun OSS: only enabled if the test bucket is provided. */ -public class OSSContract extends AbstractBondedFSContract { +public class AliyunOSSContract extends AbstractBondedFSContract { - public static final String CONTRACT_XML = "contract/oss.xml"; - public static final String CONTRACT_TEST_OSS_FS_NAME = - "fs.contract.test.fs.oss"; + public static final String CONTRACT_XML = "contract/aliyun-oss.xml"; - private static String testPath = OSSTestUtils.generateUniqueTestPath(); - - public OSSContract(Configuration conf) { + public AliyunOSSContract(Configuration conf) { super(conf); //insert the base features addConfResource(CONTRACT_XML); @@ -47,7 +42,8 @@ public class OSSContract extends AbstractBondedFSContract { @Override public Path getTestPath() { - Path path = new Path(testPath); - return path; + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? super.getTestPath() : + new Path("/" + testUniqueForkId, "test"); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java similarity index 87% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java index ce927a97b39..88dd8cd2267 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java @@ -23,13 +23,13 @@ import org.apache.hadoop.fs.contract.AbstractContractCreateTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract creating tests. + * Aliyun OSS contract creating tests. */ -public class TestOSSContractCreate extends AbstractContractCreateTest { +public class TestAliyunOSSContractCreate extends AbstractContractCreateTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java similarity index 87% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java index 6a1eb40e3a1..1658d806831 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java @@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract deleting tests. + * Aliyun OSS contract deleting tests. */ -public class TestOSSContractDelete extends AbstractContractDeleteTest { +public class TestAliyunOSSContractDelete extends AbstractContractDeleteTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java similarity index 88% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java index eb0c5e0cdd2..4b482fca338 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java @@ -25,7 +25,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*; /** * Contract test suite covering Aliyun OSS integration with DistCp. */ -public class TestOSSContractDispCp extends AbstractContractDistCpTest { +public class TestAliyunOSSContractDispCp extends AbstractContractDistCpTest { private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB @@ -38,7 +38,7 @@ public class TestOSSContractDispCp extends AbstractContractDistCpTest { } @Override - protected OSSContract createContract(Configuration conf) { - return new OSSContract(conf); + protected AliyunOSSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java similarity index 93% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java index cc21a2e2ce2..c69124d9243 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java @@ -24,12 +24,12 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; /** * Test getFileStatus and related listing operations. */ -public class TestOSSContractGetFileStatus +public class TestAliyunOSSContractGetFileStatus extends AbstractContractGetFileStatusTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java similarity index 87% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java index 1dcb7f030de..6cb754975ca 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java @@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract directory tests. + * Aliyun OSS contract directory tests. */ -public class TestOSSContractMkdir extends AbstractContractMkdirTest { +public class TestAliyunOSSContractMkdir extends AbstractContractMkdirTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java similarity index 87% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java index ee0c055decb..099aba6296f 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java @@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractOpenTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract opening file tests. + * Aliyun OSS contract opening file tests. */ -public class TestOSSContractOpen extends AbstractContractOpenTest { +public class TestAliyunOSSContractOpen extends AbstractContractOpenTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java similarity index 87% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java index 634fcf12fa6..e15b3ba30de 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java @@ -23,13 +23,13 @@ import org.apache.hadoop.fs.contract.AbstractContractRenameTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract renaming tests. + * Aliyun OSS contract renaming tests. */ -public class TestOSSContractRename extends AbstractContractRenameTest { +public class TestAliyunOSSContractRename extends AbstractContractRenameTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java similarity index 93% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java index cbc262c735d..9faae374521 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java @@ -29,15 +29,15 @@ import java.io.IOException; /** * Root dir operations against an Aliyun OSS bucket. */ -public class TestOSSContractRootDir extends +public class TestAliyunOSSContractRootDir extends AbstractContractRootDirectoryTest { private static final Logger LOG = - LoggerFactory.getLogger(TestOSSContractRootDir.class); + LoggerFactory.getLogger(TestAliyunOSSContractRootDir.class); @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } @Override diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java similarity index 88% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java index 40ea772c943..b247ab10c7c 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java @@ -23,12 +23,12 @@ import org.apache.hadoop.fs.contract.AbstractContractSeekTest; import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * OSS contract seeking tests. + * Aliyun OSS contract seeking tests. */ -public class TestOSSContractSeek extends AbstractContractSeekTest { +public class TestAliyunOSSContractSeek extends AbstractContractSeekTest { @Override protected AbstractFSContract createContract(Configuration conf) { - return new OSSContract(conf); + return new AliyunOSSContract(conf); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml similarity index 100% rename from hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml rename to hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml From 9cd47602576cd01a905e27642b685905a88eee72 Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Tue, 20 Sep 2016 11:39:02 +0800 Subject: [PATCH 09/13] HADOOP-13609. Refine credential provider related codes for AliyunOss integration. Contributed by Genmao Yu --- .../aliyun/oss/AliyunCredentialsProvider.java | 87 +++++++++++++++++++ .../fs/aliyun/oss/AliyunOSSFileSystem.java | 4 +- .../aliyun/oss/AliyunOSSFileSystemStore.java | 20 ++++- .../fs/aliyun/oss/AliyunOSSInputStream.java | 4 +- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 4 +- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 14 +-- .../hadoop/fs/aliyun/oss/Constants.java | 4 +- .../TemporaryAliyunCredentialsProvider.java | 64 -------------- ...ntials.java => TestAliyunCredentials.java} | 37 +++++--- .../oss/TestAliyunOSSFileSystemStore.java | 4 +- 10 files changed, 144 insertions(+), 98 deletions(-) create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java delete mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestAliyunOSSTemporaryCredentials.java => TestAliyunCredentials.java} (69%) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java new file mode 100644 index 00000000000..b46c67aa5e7 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import com.aliyun.oss.common.auth.InvalidCredentialsException; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Support session credentials for authenticating with Aliyun. + */ +public class AliyunCredentialsProvider implements CredentialsProvider { + private Credentials credentials = null; + + public AliyunCredentialsProvider(Configuration conf) + throws IOException { + String accessKeyId; + String accessKeySecret; + String securityToken; + try { + accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID); + accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET); + } catch (IOException e) { + throw new InvalidCredentialsException(e); + } + + try { + securityToken = AliyunOSSUtils.getValueWithKey(conf, SECURITY_TOKEN); + } catch (IOException e) { + securityToken = null; + } + + if (StringUtils.isEmpty(accessKeyId) + || StringUtils.isEmpty(accessKeySecret)) { + throw new InvalidCredentialsException( + "AccessKeyId and AccessKeySecret should not be null or empty."); + } + + if (StringUtils.isNotEmpty(securityToken)) { + credentials = new DefaultCredentials(accessKeyId, accessKeySecret, + securityToken); + } else { + credentials = new DefaultCredentials(accessKeyId, accessKeySecret); + } + } + + @Override + public void setCredentials(Credentials creds) { + if (creds == null) { + throw new InvalidCredentialsException("Credentials should not be null."); + } + + credentials = creds; + } + + @Override + public Credentials getCredentials() { + if (credentials == null) { + throw new InvalidCredentialsException("Invalid credentials"); + } + + return credentials; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index ad321bd2039..d40fcd0decd 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.aliyun.oss; -import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -43,6 +41,8 @@ import com.aliyun.oss.model.ObjectMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + /** * Implementation of {@link FileSystem} for * Aliyun OSS, used to access OSS blob system in a filesystem style. diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index b3cd1bdd144..d53f930b252 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -23,7 +23,25 @@ import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSException; import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.comm.Protocol; -import com.aliyun.oss.model.*; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CannedAccessControlList; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadResult; +import com.aliyun.oss.model.CopyObjectResult; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.PartETag; +import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; +import com.aliyun.oss.model.UploadPartRequest; +import com.aliyun.oss.model.UploadPartResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.slf4j.Logger; diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java index 69265fb81d4..b87a3a753bc 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.aliyun.oss; -import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -31,6 +29,8 @@ import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem.Statistics; +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + /** * The input stream for OSS blob system. * The class uses multi-part downloading to read data from the object content diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java index c952d0ae858..c75ee187bc9 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.aliyun.oss; -import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -33,6 +31,8 @@ import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.util.Progressable; +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + /** * The output stream for OSS blob system. * Data will be buffered on local disk, then uploaded to OSS in diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index b96aea7f86e..fe70d533a53 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -23,8 +23,6 @@ import java.io.InputStream; import java.net.URI; import com.aliyun.oss.common.auth.CredentialsProvider; -import com.aliyun.oss.common.auth.DefaultCredentialProvider; -import com.aliyun.oss.common.auth.DefaultCredentials; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.ProviderUtils; @@ -32,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.aliyun.oss.Constants.*; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY; /** * Utility methods for Aliyun OSS code. @@ -52,7 +49,7 @@ final public class AliyunOSSUtils { * @return the value for the key * @throws IOException if failed to get password from configuration */ - static public String getPassword(Configuration conf, String key) + public static String getValueWithKey(Configuration conf, String key) throws IOException { try { final char[] pass = conf.getPassword(key); @@ -126,12 +123,7 @@ final public class AliyunOSSUtils { Configuration newConf = ProviderUtils.excludeIncompatibleCredentialProviders(conf, AliyunOSSFileSystem.class); - String accessKey = - AliyunOSSUtils.getPassword(newConf, ACCESS_KEY); - String secretKey = - AliyunOSSUtils.getPassword(newConf, SECRET_KEY); - credentials = new DefaultCredentialProvider( - new DefaultCredentials(accessKey, secretKey)); + credentials = new AliyunCredentialsProvider(newConf); } else { try { LOG.debug("Credential provider class is:" + className); @@ -139,7 +131,7 @@ final public class AliyunOSSUtils { try { credentials = (CredentialsProvider)credClass.getDeclaredConstructor( - URI.class, Configuration.class).newInstance(uri, conf); + Configuration.class).newInstance(conf); } catch (NoSuchMethodException | SecurityException e) { credentials = (CredentialsProvider)credClass.getDeclaredConstructor() diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index e0c05ed740f..04a2ccd6c54 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -31,8 +31,8 @@ public final class Constants { "fs.oss.credentials.provider"; // OSS access verification - public static final String ACCESS_KEY = "fs.oss.accessKeyId"; - public static final String SECRET_KEY = "fs.oss.accessKeySecret"; + public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId"; + public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; public static final String SECURITY_TOKEN = "fs.oss.securityToken"; // Number of simultaneous connections to oss diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java deleted file mode 100644 index ec8e7fe8386..00000000000 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.aliyun.oss; - -import com.aliyun.oss.common.auth.Credentials; -import com.aliyun.oss.common.auth.CredentialsProvider; -import com.aliyun.oss.common.auth.DefaultCredentials; -import com.aliyun.oss.common.auth.InvalidCredentialsException; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; - -import java.net.URI; - -import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - -/** - * Support session credentials for authenticating with ALiyun. - */ -public class TemporaryAliyunCredentialsProvider implements CredentialsProvider { - public static final String NAME - = "org.apache.hadoop.fs.aliyun.oss.TemporaryAliyunCredentialsProvider"; - private final String accessKeyId; - private final String accessKeySecret; - private final String securityToken; - - public TemporaryAliyunCredentialsProvider(URI uri, Configuration conf) { - this.accessKeyId = conf.get(ACCESS_KEY, null); - this.accessKeySecret = conf.get(SECRET_KEY, null); - this.securityToken = conf.get(SECURITY_TOKEN, null); - } - - @Override - public void setCredentials(Credentials creds) { - - } - - @Override - public Credentials getCredentials() { - if (!StringUtils.isEmpty(accessKeyId) - && !StringUtils.isEmpty(accessKeySecret) - && !StringUtils.isEmpty(securityToken)) { - return new DefaultCredentials(accessKeyId, accessKeySecret, - securityToken); - } - throw new InvalidCredentialsException( - "AccessKeyId, AccessKeySecret or SecurityToken is unset"); - } -} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java similarity index 69% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java index e106aae184e..e08a4dccfca 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java @@ -26,10 +26,10 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.junit.Test; -import java.net.URI; +import java.io.IOException; -import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY; -import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY; +import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID; +import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; /** @@ -38,8 +38,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; * should only be used against transient filesystems where you don't care about * the data. */ -public class TestAliyunOSSTemporaryCredentials - extends AbstractFSContractTestBase { +public class TestAliyunCredentials extends AbstractFSContractTestBase { @Override protected AbstractFSContract createContract(Configuration conf) { @@ -47,19 +46,33 @@ public class TestAliyunOSSTemporaryCredentials } @Test - public void testTemporaryCredentialValidation() throws Throwable { + public void testCredentialMissingAccessKeyId() throws Throwable { Configuration conf = new Configuration(); - conf.set(ACCESS_KEY, "accessKeyId"); - conf.set(SECRET_KEY, "accessKeySecret"); - conf.set(SECURITY_TOKEN, ""); - URI uri = getFileSystem().getUri(); - TemporaryAliyunCredentialsProvider provider - = new TemporaryAliyunCredentialsProvider(uri, conf); + conf.set(ACCESS_KEY_ID, ""); + conf.set(ACCESS_KEY_SECRET, "accessKeySecret"); + conf.set(SECURITY_TOKEN, "token"); + validateCredential(conf); + } + + @Test + public void testCredentialMissingAccessKeySecret() throws Throwable { + Configuration conf = new Configuration(); + conf.set(ACCESS_KEY_ID, "accessKeyId"); + conf.set(ACCESS_KEY_SECRET, ""); + conf.set(SECURITY_TOKEN, "token"); + validateCredential(conf); + } + + private void validateCredential(Configuration conf) { try { + AliyunCredentialsProvider provider + = new AliyunCredentialsProvider(conf); Credentials credentials = provider.getCredentials(); fail("Expected a CredentialInitializationException, got " + credentials); } catch (InvalidCredentialsException expected) { // expected + } catch (IOException e) { + fail("Unexpected exception."); } } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java index c8498042b6e..dee4ccfdb1c 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java @@ -67,8 +67,8 @@ public class TestAliyunOSSFileSystemStore { @BeforeClass public static void checkSettings() throws Exception { Configuration conf = new Configuration(); - assumeNotNull(conf.get(Constants.ACCESS_KEY)); - assumeNotNull(conf.get(Constants.SECRET_KEY)); + assumeNotNull(conf.get(Constants.ACCESS_KEY_ID)); + assumeNotNull(conf.get(Constants.ACCESS_KEY_SECRET)); assumeNotNull(conf.get("test.fs.oss.name")); } From 08b37603d9c0be67c4e0790c1ad266551ef21f5e Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Tue, 20 Sep 2016 15:12:02 +0800 Subject: [PATCH 10/13] HADOOP-13591. Unit test failure in TestOSSContractGetFileStatus and TestOSSContractRootDir. Contributed by Genmao Yu --- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 88 ++++++++----------- .../aliyun/oss/AliyunOSSFileSystemStore.java | 36 +++++--- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 24 +++-- .../fs/aliyun/oss/AliyunOSSTestUtils.java | 2 +- .../test/resources/contract/aliyun-oss.xml | 10 +++ 5 files changed, 89 insertions(+), 71 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index d40fcd0decd..81e038d25f9 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -24,6 +24,8 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -124,9 +126,6 @@ public class AliyunOSSFileSystem extends FileSystem { Path f = status.getPath(); String key = pathToKey(f); if (status.isDirectory()) { - if (!key.endsWith("/")) { - key += "/"; - } if (!recursive) { FileStatus[] statuses = listStatus(status.getPath()); // Check whether it is an empty directory or not @@ -135,6 +134,7 @@ public class AliyunOSSFileSystem extends FileSystem { ": It is not empty!"); } else { // Delete empty directory without '-r' + key = AliyunOSSUtils.maybeAddTrailingSlash(key); store.deleteObject(key); } } else { @@ -149,15 +149,9 @@ public class AliyunOSSFileSystem extends FileSystem { } private void createFakeDirectoryIfNecessary(Path f) throws IOException { - try { - Path pPath = f.getParent(); - FileStatus pStatus = getFileStatus(pPath); - if (pStatus.isFile()) { - throw new IOException("Path " + pPath + - " is assumed to be a directory!"); - } - } catch (FileNotFoundException fnfe) { - // Make sure the parent directory exists + String key = pathToKey(f); + if (StringUtils.isNotEmpty(key) && !exists(f)) { + LOG.debug("Creating new fake directory at {}", f); mkdir(pathToKey(f.getParent())); } } @@ -175,14 +169,14 @@ public class AliyunOSSFileSystem extends FileSystem { ObjectMetadata meta = store.getObjectMetadata(key); // If key not found and key does not end with "/" if (meta == null && !key.endsWith("/")) { - // Case: dir + "/" + // In case of 'dir + "/"' key += "/"; meta = store.getObjectMetadata(key); } if (meta == null) { - ObjectListing listing = store.listObjects(key, 1, "/", null); - if (!listing.getObjectSummaries().isEmpty() || - !listing.getCommonPrefixes().isEmpty()) { + ObjectListing listing = store.listObjects(key, 1, null, false); + if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) || + CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) { return new FileStatus(0, true, 1, 0, 0, qualifiedPath); } else { throw new FileNotFoundException(path + ": No such file or directory!"); @@ -251,7 +245,7 @@ public class AliyunOSSFileSystem extends FileSystem { */ private boolean objectRepresentsDirectory(final String name, final long size) { - return !name.isEmpty() && name.endsWith("/") && size == 0L; + return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L; } /** @@ -265,10 +259,6 @@ public class AliyunOSSFileSystem extends FileSystem { path = new Path(workingDir, path); } - if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) { - return ""; - } - return path.toUri().getPath().substring(1); } @@ -287,26 +277,23 @@ public class AliyunOSSFileSystem extends FileSystem { final FileStatus fileStatus = getFileStatus(path); if (fileStatus.isDirectory()) { - if (!key.endsWith("/")) { - key = key + "/"; - } - if (LOG.isDebugEnabled()) { LOG.debug("listStatus: doing listObjects for directory " + key); } - ObjectListing objects = store.listObjects(key, maxKeys, "/", null); + ObjectListing objects = store.listObjects(key, maxKeys, null, false); while (true) { statistics.incrementReadOps(1); for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { - Path keyPath = keyToPath(objectSummary.getKey()) - .makeQualified(uri, workingDir); - if (keyPath.equals(path)) { + String objKey = objectSummary.getKey(); + if (objKey.equals(key + "/")) { if (LOG.isDebugEnabled()) { - LOG.debug("Ignoring: " + keyPath); + LOG.debug("Ignoring: " + objKey); } continue; } else { + Path keyPath = keyToPath(objectSummary.getKey()) + .makeQualified(uri, workingDir); if (LOG.isDebugEnabled()) { LOG.debug("Adding: fi: " + keyPath); } @@ -317,10 +304,13 @@ public class AliyunOSSFileSystem extends FileSystem { } for (String prefix : objects.getCommonPrefixes()) { - Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir); - if (keyPath.equals(path)) { + if (prefix.equals(key + "/")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring: " + prefix); + } continue; } else { + Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir); if (LOG.isDebugEnabled()) { LOG.debug("Adding: rd: " + keyPath); } @@ -332,8 +322,8 @@ public class AliyunOSSFileSystem extends FileSystem { if (LOG.isDebugEnabled()) { LOG.debug("listStatus: list truncated - getting next batch"); } - objects = store.listObjects(key, maxKeys, "/", - objects.getNextMarker()); + String nextMarker = objects.getNextMarker(); + objects = store.listObjects(key, maxKeys, nextMarker, false); statistics.incrementReadOps(1); } else { break; @@ -358,10 +348,12 @@ public class AliyunOSSFileSystem extends FileSystem { */ private boolean mkdir(final String key) throws IOException { String dirName = key; - if (!key.endsWith("/")) { - dirName += "/"; + if (StringUtils.isNotEmpty(key)) { + if (!key.endsWith("/")) { + dirName += "/"; + } + store.storeEmptyFile(dirName); } - store.storeEmptyFile(dirName); return true; } @@ -506,16 +498,11 @@ public class AliyunOSSFileSystem extends FileSystem { * @param dstPath destination path. * @return true if directory is successfully copied. */ - private boolean copyDirectory(Path srcPath, Path dstPath) { - String srcKey = pathToKey(srcPath); - String dstKey = pathToKey(dstPath); - - if (!srcKey.endsWith("/")) { - srcKey = srcKey + "/"; - } - if (!dstKey.endsWith("/")) { - dstKey = dstKey + "/"; - } + private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException { + String srcKey = AliyunOSSUtils + .maybeAddTrailingSlash(pathToKey(srcPath)); + String dstKey = AliyunOSSUtils + .maybeAddTrailingSlash(pathToKey(dstPath)); if (dstKey.startsWith(srcKey)) { if (LOG.isDebugEnabled()) { @@ -524,7 +511,8 @@ public class AliyunOSSFileSystem extends FileSystem { return false; } - ObjectListing objects = store.listObjects(srcKey, maxKeys, null, null); + store.storeEmptyFile(dstKey); + ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true); statistics.incrementReadOps(1); // Copy files from src folder to dst while (true) { @@ -534,8 +522,8 @@ public class AliyunOSSFileSystem extends FileSystem { store.copyFile(objectSummary.getKey(), newKey); } if (objects.isTruncated()) { - objects = store.listObjects(srcKey, maxKeys, null, - objects.getNextMarker()); + String nextMarker = objects.getNextMarker(); + objects = store.listObjects(srcKey, maxKeys, nextMarker, true); statistics.incrementReadOps(1); } else { break; diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index d53f930b252..be87fa98094 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -42,6 +42,8 @@ import com.aliyun.oss.model.UploadPartCopyRequest; import com.aliyun.oss.model.UploadPartCopyResult; import com.aliyun.oss.model.UploadPartRequest; import com.aliyun.oss.model.UploadPartResult; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.slf4j.Logger; @@ -89,7 +91,7 @@ public class AliyunOSSFileSystemStore { String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); - if (!proxyHost.isEmpty()) { + if (StringUtils.isNotEmpty(proxyHost)) { clientConf.setProxyHost(proxyHost); if (proxyPort >= 0) { clientConf.setProxyPort(proxyPort); @@ -123,7 +125,7 @@ public class AliyunOSSFileSystemStore { String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); CredentialsProvider provider = - AliyunOSSUtils.getCredentialsProvider(uri, conf); + AliyunOSSUtils.getCredentialsProvider(conf); ossClient = new OSSClient(endPoint, provider, clientConf); uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_UPLOAD_SIZE_DEFAULT); @@ -153,7 +155,7 @@ public class AliyunOSSFileSystemStore { } String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); - if (!cannedACLName.isEmpty()) { + if (StringUtils.isNotEmpty(cannedACLName)) { CannedAccessControlList cannedACL = CannedAccessControlList.valueOf(cannedACLName); ossClient.setBucketAcl(bucketName, cannedACL); @@ -179,11 +181,13 @@ public class AliyunOSSFileSystemStore { * @param keysToDelete collection of keys to delete. */ public void deleteObjects(List keysToDelete) { - DeleteObjectsRequest deleteRequest = - new DeleteObjectsRequest(bucketName); - deleteRequest.setKeys(keysToDelete); - ossClient.deleteObjects(deleteRequest); - statistics.incrementWriteOps(keysToDelete.size()); + if (CollectionUtils.isNotEmpty(keysToDelete)) { + DeleteObjectsRequest deleteRequest = + new DeleteObjectsRequest(bucketName); + deleteRequest.setKeys(keysToDelete); + ossClient.deleteObjects(deleteRequest); + statistics.incrementWriteOps(keysToDelete.size()); + } } /** @@ -192,8 +196,10 @@ public class AliyunOSSFileSystemStore { * @param key directory key to delete. */ public void deleteDirs(String key) { + key = AliyunOSSUtils.maybeAddTrailingSlash(key); ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); listRequest.setPrefix(key); + listRequest.setDelimiter(null); listRequest.setMaxKeys(maxKeys); while (true) { @@ -299,7 +305,7 @@ public class AliyunOSSFileSystemStore { InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, dstKey); ObjectMetadata meta = new ObjectMetadata(); - if (!serverSideEncryptionAlgorithm.isEmpty()) { + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { meta.setServerSideEncryption(serverSideEncryptionAlgorithm); } initiateMultipartUploadRequest.setObjectMetadata(meta); @@ -353,7 +359,7 @@ public class AliyunOSSFileSystemStore { FileInputStream fis = new FileInputStream(object); ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(object.length()); - if (!serverSideEncryptionAlgorithm.isEmpty()) { + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { meta.setServerSideEncryption(serverSideEncryptionAlgorithm); } try { @@ -384,7 +390,7 @@ public class AliyunOSSFileSystemStore { InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key); ObjectMetadata meta = new ObjectMetadata(); - if (!serverSideEncryptionAlgorithm.isEmpty()) { + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { meta.setServerSideEncryption(serverSideEncryptionAlgorithm); } initiateMultipartUploadRequest.setObjectMetadata(meta); @@ -435,12 +441,14 @@ public class AliyunOSSFileSystemStore { * * @param prefix prefix. * @param maxListingLength max no. of entries - * @param delimiter delimiter. * @param marker last key in any previous search. + * @param recursive whether to list directory recursively. * @return a list of matches. */ public ObjectListing listObjects(String prefix, int maxListingLength, - String delimiter, String marker) { + String marker, boolean recursive) { + String delimiter = recursive ? null : "/"; + prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix); ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); listRequest.setPrefix(prefix); listRequest.setDelimiter(delimiter); @@ -488,7 +496,7 @@ public class AliyunOSSFileSystemStore { public void purge(String prefix) { String key; try { - ObjectListing objects = listObjects(prefix, maxKeys, null, null); + ObjectListing objects = listObjects(prefix, maxKeys, null, true); for (OSSObjectSummary object : objects.getObjectSummaries()) { key = object.getKey(); ossClient.deleteObject(bucketName, key); diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index fe70d533a53..cae9749a614 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -20,7 +20,6 @@ package org.apache.hadoop.fs.aliyun.oss; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import com.aliyun.oss.common.auth.CredentialsProvider; import org.apache.commons.lang.StringUtils; @@ -106,16 +105,13 @@ final public class AliyunOSSUtils { * Create credential provider specified by configuration, or create default * credential provider if not specified. * - * @param name the uri of the file system * @param conf configuration * @return a credential provider * @throws IOException on any problem. Class construction issues may be * nested inside the IOE. */ - public static CredentialsProvider getCredentialsProvider(URI name, - Configuration conf) throws IOException { - URI uri = java.net.URI.create( - name.getScheme() + "://" + name.getAuthority()); + public static CredentialsProvider getCredentialsProvider(Configuration conf) + throws IOException { CredentialsProvider credentials; String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); @@ -152,4 +148,20 @@ final public class AliyunOSSUtils { return credentials; } + + /** + * Turns a path (relative or otherwise) into an OSS key, adding a trailing + * "/" if the path is not the root and does not already have a "/" + * at the end. + * + * @param key OSS key or "" + * @return the with a trailing "/", or, if it is the root key, "". + */ + public static String maybeAddTrailingSlash(String key) { + if (StringUtils.isNotEmpty(key) && !key.endsWith("/")) { + return key + '/'; + } else { + return key; + } + } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java index 84dba6a2d6d..901cb2bd082 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java @@ -48,7 +48,7 @@ public final class AliyunOSSTestUtils { String fsname = conf.getTrimmed( TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, ""); - boolean liveTest = !StringUtils.isEmpty(fsname); + boolean liveTest = StringUtils.isNotEmpty(fsname); URI testURI = null; if (liveTest) { testURI = URI.create(fsname); diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml index 2bc34b754b9..7bbbf4646b6 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml @@ -98,6 +98,16 @@ true + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.supports-getfilestatus + true + + fs.oss.multipart.download.size 102400 From 22af6f8db3a44cd51514b4851b99adcfad42751d Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Wed, 21 Sep 2016 14:02:44 +0800 Subject: [PATCH 11/13] HADOOP-13624. Rename TestAliyunOSSContractDispCp. Contributed by Genmao Yu --- .../hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java | 6 +++++- .../hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java | 6 +++++- .../hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java | 5 ++++- ...ContractDispCp.java => TestAliyunOSSContractDistCp.java} | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/{TestAliyunOSSContractDispCp.java => TestAliyunOSSContractDistCp.java} (96%) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index be87fa98094..9792a7816c3 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -49,7 +49,11 @@ import org.apache.hadoop.fs.FileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.util.ArrayList; import java.util.List; diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java index dee4ccfdb1c..7f4bac25642 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java @@ -25,7 +25,11 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URI; import java.security.DigestInputStream; import java.security.DigestOutputStream; diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java index 892eda062e4..37af28ff06a 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java @@ -24,7 +24,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.io.IOUtils; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java similarity index 96% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java index 4b482fca338..18d09d54149 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java @@ -25,7 +25,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*; /** * Contract test suite covering Aliyun OSS integration with DistCp. */ -public class TestAliyunOSSContractDispCp extends AbstractContractDistCpTest { +public class TestAliyunOSSContractDistCp extends AbstractContractDistCpTest { private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB From 26d5df390cf976dcc1d17fc68d0fed789dc34e84 Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Fri, 23 Sep 2016 08:44:28 +0800 Subject: [PATCH 12/13] HADOOP-13634. Some configuration in doc has been outdated. Contributed by Genmao Yu --- .../src/site/markdown/tools/hadoop-aliyun/index.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md index 4095e06fa57..88c83b5724f 100644 --- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md +++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md @@ -167,7 +167,7 @@ please raise your issues with them. fs.oss.paging.maximum - 500 + 1000 How many keys to request from Aliyun OSS when doing directory listings at a time. @@ -195,11 +195,6 @@ please raise your issues with them. Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS - - fs.oss.buffer.dir - Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS - - fs.oss.acl.default From c31b5e61b1f09949548116309218a2b3e9c0beda Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Sat, 8 Oct 2016 17:35:59 +0600 Subject: [PATCH 13/13] HADOOP-13701. AbstractContractRootDirectoryTest can fail when handling delete "/". Contributed by Genmao Yu --- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 81e038d25f9..3b266c83934 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; @@ -53,6 +54,7 @@ public class AliyunOSSFileSystem extends FileSystem { private static final Logger LOG = LoggerFactory.getLogger(AliyunOSSFileSystem.class); private URI uri; + private String bucket; private Path workingDir; private AliyunOSSFileSystemStore store; private int maxKeys; @@ -124,11 +126,20 @@ public class AliyunOSSFileSystem extends FileSystem { private boolean innerDelete(FileStatus status, boolean recursive) throws IOException { Path f = status.getPath(); + String p = f.toUri().getPath(); + FileStatus[] statuses; + // indicating root directory "/". + if (p.equals("/")) { + statuses = listStatus(status.getPath()); + boolean isEmptyDir = statuses.length <= 0; + return rejectRootDirectoryDelete(isEmptyDir, recursive); + } + String key = pathToKey(f); if (status.isDirectory()) { if (!recursive) { - FileStatus[] statuses = listStatus(status.getPath()); // Check whether it is an empty directory or not + statuses = listStatus(status.getPath()); if (statuses.length > 0) { throw new IOException("Cannot remove directory " + f + ": It is not empty!"); @@ -148,6 +159,31 @@ public class AliyunOSSFileSystem extends FileSystem { return true; } + /** + * Implements the specific logic to reject root directory deletion. + * The caller must return the result of this call, rather than + * attempt to continue with the delete operation: deleting root + * directories is never allowed. This method simply implements + * the policy of when to return an exit code versus raise an exception. + * @param isEmptyDir empty directory or not + * @param recursive recursive flag from command + * @return a return code for the operation + * @throws PathIOException if the operation was explicitly rejected. + */ + private boolean rejectRootDirectoryDelete(boolean isEmptyDir, + boolean recursive) throws IOException { + LOG.info("oss delete the {} root directory of {}", bucket, recursive); + if (isEmptyDir) { + return true; + } + if (recursive) { + return false; + } else { + // reject + throw new PathIOException(bucket, "Cannot delete root path"); + } + } + private void createFakeDirectoryIfNecessary(Path f) throws IOException { String key = pathToKey(f); if (StringUtils.isNotEmpty(key) && !exists(f)) { @@ -226,6 +262,7 @@ public class AliyunOSSFileSystem extends FileSystem { public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); + bucket = name.getHost(); uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority()); workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(uri, null);