getOperationStatistics() {
+ return durationStats.getDurationStatistics();
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html
new file mode 100644
index 00000000000..0ccca32b4f5
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html
@@ -0,0 +1,81 @@
+
+
+
+
+
+ Swift Filesystem Client for Apache Hadoop
+
+
+
+
+ Swift Filesystem Client for Apache Hadoop
+
+
+Introduction
+
+This package provides support in Apache Hadoop for the OpenStack Swift
+ Key-Value store, allowing client applications -including MR Jobs- to
+ read and write data in Swift.
+
+
+Design Goals
+
+ Give clients access to SwiftFS files, similar to S3n:
+ maybe: support a Swift Block store -- at least until Swift's
+ support for >5GB files has stabilized.
+
+ Support for data-locality if the Swift FS provides file location information
+ Support access to multiple Swift filesystems in the same client/task.
+ Authenticate using the Keystone APIs.
+ Avoid dependency on unmaintained libraries.
+
+
+
+Supporting multiple Swift Filesystems
+
+The goal of supporting multiple swift filesystems simultaneously changes how
+clusters are named and authenticated. In Hadoop's S3 and S3N filesystems, the "bucket" into
+which objects are stored is directly named in the URL, such as
+s3n://bucket/object1. The Hadoop configuration contains a
+single set of login credentials for S3 (username and key), which are used to
+authenticate the HTTP operations.
+
+For swift, we need to know not only which "container" name, but which credentials
+to use to authenticate with it -and which URL to use for authentication.
+
+This has led to a different design pattern from S3, as instead of simple bucket names,
+the hostname of an S3 container is two-level, the name of the service provider
+being the second path: swift://bucket.service/
+
+The service portion of this domainame is used as a reference into
+the client settings -and so identify the service provider of that container.
+
+
+Testing
+
+
+ The client code can be tested against public or private Swift instances; the
+ public services are (at the time of writing -January 2013-), Rackspace and
+ HP Cloud. Testing against both instances is how interoperability
+ can be verified.
+
+
+
+
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java
new file mode 100644
index 00000000000..701f5100aaa
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.hadoop.fs.BufferedFSInputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+
+import java.io.IOException;
+
+/**
+ * Add stricter compliance with the evolving FS specifications
+ */
+public class StrictBufferedFSInputStream extends BufferedFSInputStream {
+
+ public StrictBufferedFSInputStream(FSInputStream in,
+ int size) {
+ super(in, size);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ if (pos < 0) {
+ throw new IOException("Negative position");
+ }
+ if (in == null) {
+ throw new SwiftConnectionClosedException("Stream closed");
+ }
+ super.seek(pos);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
new file mode 100644
index 00000000000..d010d08cdc5
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * A subclass of {@link FileStatus} that contains the
+ * Swift-specific rules of when a file is considered to be a directory.
+ */
+public class SwiftFileStatus extends FileStatus {
+
+ public SwiftFileStatus() {
+ }
+
+ public SwiftFileStatus(long length,
+ boolean isdir,
+ int block_replication,
+ long blocksize, long modification_time, Path path) {
+ super(length, isdir, block_replication, blocksize, modification_time, path);
+ }
+
+ public SwiftFileStatus(long length,
+ boolean isdir,
+ int block_replication,
+ long blocksize,
+ long modification_time,
+ long access_time,
+ FsPermission permission,
+ String owner, String group, Path path) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, path);
+ }
+
+ //HDFS2+ only
+
+ public SwiftFileStatus(long length,
+ boolean isdir,
+ int block_replication,
+ long blocksize,
+ long modification_time,
+ long access_time,
+ FsPermission permission,
+ String owner, String group, Path symlink, Path path) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path);
+ }
+
+ /**
+ * Declare that the path represents a directory, which in the
+ * SwiftNativeFileSystem means "is a directory or a 0 byte file"
+ *
+ * @return true if the status is considered to be a file
+ */
+ @Override
+ public boolean isDir() {
+ return super.isDirectory() || getLen() == 0;
+ }
+
+ /**
+ * A entry is a file if it is not a directory.
+ * By implementing it and not marking as an override this
+ * subclass builds and runs in both Hadoop versions.
+ * @return the opposite value to {@link #isDir()}
+ */
+ @Override
+ public boolean isFile() {
+ return !isDir();
+ }
+
+ /**
+ * Directory test
+ * @return true if the file is considered to be a directory
+ */
+ public boolean isDirectory() {
+ return isDir();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getSimpleName());
+ sb.append("{ ");
+ sb.append("path=").append(getPath());
+ sb.append("; isDirectory=").append(isDir());
+ sb.append("; length=").append(getLen());
+ sb.append("; blocksize=").append(getBlockSize());
+ sb.append("; modification_time=").append(getModificationTime());
+ sb.append("}");
+ return sb.toString();
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
new file mode 100644
index 00000000000..f4a4bd8a1e9
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
@@ -0,0 +1,714 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftPathExistsException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftUnsupportedFeatureException;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Swift file system implementation. Extends Hadoop FileSystem
+ */
+public class SwiftNativeFileSystem extends FileSystem {
+
+ /** filesystem prefix: {@value} */
+ public static final String SWIFT = "swift";
+ private static final Log LOG =
+ LogFactory.getLog(SwiftNativeFileSystem.class);
+
+ /**
+ * path to user work directory for storing temporary files
+ */
+ private Path workingDir;
+
+ /**
+ * Swift URI
+ */
+ private URI uri;
+
+ /**
+ * reference to swiftFileSystemStore
+ */
+ private SwiftNativeFileSystemStore store;
+
+ /**
+ * Default constructor for Hadoop
+ */
+ public SwiftNativeFileSystem() {
+ // set client in initialize()
+ }
+
+ /**
+ * This constructor used for testing purposes
+ */
+ public SwiftNativeFileSystem(SwiftNativeFileSystemStore store) {
+ this.store = store;
+ }
+
+ /**
+ * This is for testing
+ * @return the inner store class
+ */
+ public SwiftNativeFileSystemStore getStore() {
+ return store;
+ }
+
+ @Override
+ public String getScheme() {
+ return SWIFT;
+ }
+
+ /**
+ * default class initialization
+ *
+ * @param fsuri path to Swift
+ * @param conf Hadoop configuration
+ * @throws IOException
+ */
+ @Override
+ public void initialize(URI fsuri, Configuration conf) throws IOException {
+ super.initialize(fsuri, conf);
+
+ setConf(conf);
+ if (store == null) {
+ store = new SwiftNativeFileSystemStore();
+ }
+ this.uri = fsuri;
+ String username = System.getProperty("user.name");
+ this.workingDir = new Path("/user", username)
+ .makeQualified(uri, new Path(username));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initializing SwiftNativeFileSystem against URI " + uri
+ + " and working dir " + workingDir);
+ }
+ store.initialize(uri, conf);
+ LOG.debug("SwiftFileSystem initialized");
+ }
+
+ /**
+ * @return path to Swift
+ */
+ @Override
+ public URI getUri() {
+
+ return uri;
+ }
+
+ @Override
+ public String toString() {
+ return "Swift FileSystem " + store;
+ }
+
+ /**
+ * Path to user working directory
+ *
+ * @return Hadoop path
+ */
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ /**
+ * @param dir user working directory
+ */
+ @Override
+ public void setWorkingDirectory(Path dir) {
+ workingDir = makeAbsolute(dir);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SwiftFileSystem.setWorkingDirectory to " + dir);
+ }
+ }
+
+ /**
+ * Return a file status object that represents the path.
+ *
+ * @param path The path we want information from
+ * @return a FileStatus object
+ */
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ Path absolutePath = makeAbsolute(path);
+ return store.getObjectMetadata(absolutePath);
+ }
+
+ /**
+ * The blocksize of this filesystem is set by the property
+ * SwiftProtocolConstants.SWIFT_BLOCKSIZE;the default is the value of
+ * SwiftProtocolConstants.DEFAULT_SWIFT_BLOCKSIZE;
+ * @return the blocksize for this FS.
+ */
+ @Override
+ public long getDefaultBlockSize() {
+ return store.getBlocksize();
+ }
+
+ /**
+ * The blocksize for this filesystem.
+ * @see #getDefaultBlockSize()
+ * @param f path of file
+ * @return the blocksize for the path
+ */
+ @Override
+ public long getDefaultBlockSize(Path f) {
+ return store.getBlocksize();
+ }
+
+ @Override
+ public long getBlockSize(Path path) throws IOException {
+ return store.getBlocksize();
+ }
+
+ @Override
+ public boolean isFile(Path f) throws IOException {
+ try {
+ FileStatus fileStatus = getFileStatus(f);
+ return !SwiftUtils.isDirectory(fileStatus);
+ } catch (FileNotFoundException e) {
+ return false; // f does not exist
+ }
+ }
+
+ @Override
+ public boolean isDirectory(Path f) throws IOException {
+
+ try {
+ FileStatus fileStatus = getFileStatus(f);
+ return SwiftUtils.isDirectory(fileStatus);
+ } catch (FileNotFoundException e) {
+ return false; // f does not exist
+ }
+ }
+
+ /**
+ * Return an array containing hostnames, offset and size of
+ * portions of the given file. For a nonexistent
+ * file or regions, null will be returned.
+ *
+ * This call is most helpful with DFS, where it returns
+ * hostnames of machines that contain the given file.
+ *
+ * The FileSystem will simply return an elt containing 'localhost'.
+ */
+ @Override
+ public BlockLocation[] getFileBlockLocations(FileStatus file,
+ long start,
+ long len) throws IOException {
+ //argument checks
+ if (file == null) {
+ return null;
+ }
+
+ if (start < 0 || len < 0) {
+ throw new IllegalArgumentException("Negative start or len parameter" +
+ " to getFileBlockLocations");
+ }
+ if (file.getLen() <= start) {
+ return new BlockLocation[0];
+ }
+
+ // Check if requested file in Swift is more than 5Gb. In this case
+ // each block has its own location -which may be determinable
+ // from the Swift client API, depending on the remote server
+ final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(),
+ false,
+ true);
+ List locations = new ArrayList();
+ if (listOfFileBlocks.length > 1) {
+ for (FileStatus fileStatus : listOfFileBlocks) {
+ if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
+ .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
+ continue;
+ }
+ locations.addAll(store.getObjectLocation(fileStatus.getPath()));
+ }
+ } else {
+ locations = store.getObjectLocation(file.getPath());
+ }
+
+ if (locations.isEmpty()) {
+ LOG.debug("No locations returned for " + file.getPath());
+ //no locations were returned for the object
+ //fall back to the superclass
+
+ String[] name = {SwiftProtocolConstants.BLOCK_LOCATION};
+ String[] host = { "localhost" };
+ String[] topology={SwiftProtocolConstants.TOPOLOGY_PATH};
+ return new BlockLocation[] {
+ new BlockLocation(name, host, topology,0, file.getLen())
+ };
+ }
+
+ final String[] names = new String[locations.size()];
+ final String[] hosts = new String[locations.size()];
+ int i = 0;
+ for (URI location : locations) {
+ hosts[i] = location.getHost();
+ names[i] = location.getAuthority();
+ i++;
+ }
+ return new BlockLocation[]{
+ new BlockLocation(names, hosts, 0, file.getLen())
+ };
+ }
+
+ /**
+ * Create the parent directories.
+ * As an optimization, the entire hierarchy of parent
+ * directories is Not polled. Instead
+ * the tree is walked up from the last to the first,
+ * creating directories until one that exists is found.
+ *
+ * This strategy means if a file is created in an existing directory,
+ * one quick poll sufficies.
+ *
+ * There is a big assumption here: that all parent directories of an existing
+ * directory also exists.
+ * @param path path to create.
+ * @param permission to apply to files
+ * @return true if the operation was successful
+ * @throws IOException on a problem
+ */
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SwiftFileSystem.mkdirs: " + path);
+ }
+ Path directory = makeAbsolute(path);
+
+ //build a list of paths to create
+ List paths = new ArrayList();
+ while (shouldCreate(directory)) {
+ //this directory needs creation, add to the list
+ paths.add(0, directory);
+ //now see if the parent needs to be created
+ directory = directory.getParent();
+ }
+
+ //go through the list of directories to create
+ for (Path p : paths) {
+ if (isNotRoot(p)) {
+ //perform a mkdir operation without any polling of
+ //the far end first
+ forceMkdir(p);
+ }
+ }
+
+ //if an exception was not thrown, this operation is considered
+ //a success
+ return true;
+ }
+
+ private boolean isNotRoot(Path absolutePath) {
+ return !isRoot(absolutePath);
+ }
+
+ private boolean isRoot(Path absolutePath) {
+ return absolutePath.getParent() == null;
+ }
+
+ /**
+ * internal implementation of directory creation.
+ *
+ * @param path path to file
+ * @return boolean file is created; false: no need to create
+ * @throws IOException if specified path is file instead of directory
+ */
+ private boolean mkdir(Path path) throws IOException {
+ Path directory = makeAbsolute(path);
+ boolean shouldCreate = shouldCreate(directory);
+ if (shouldCreate) {
+ forceMkdir(directory);
+ }
+ return shouldCreate;
+ }
+
+ /**
+ * Should mkdir create this directory?
+ * If the directory is root : false
+ * If the entry exists and is a directory: false
+ * If the entry exists and is a file: exception
+ * else: true
+ * @param directory path to query
+ * @return true iff the directory should be created
+ * @throws IOException IO problems
+ * @throws SwiftNotDirectoryException if the path references a file
+ */
+ private boolean shouldCreate(Path directory) throws IOException {
+ FileStatus fileStatus;
+ boolean shouldCreate;
+ if (isRoot(directory)) {
+ //its the base dir, bail out immediately
+ return false;
+ }
+ try {
+ //find out about the path
+ fileStatus = getFileStatus(directory);
+
+ if (!SwiftUtils.isDirectory(fileStatus)) {
+ //if it's a file, raise an error
+ throw new SwiftNotDirectoryException(directory,
+ String.format(": can't mkdir since it exists and is not a directory: %s",
+ fileStatus));
+ } else {
+ //path exists, and it is a directory
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("skipping mkdir(" + directory + ") as it exists already");
+ }
+ shouldCreate = false;
+ }
+ } catch (FileNotFoundException e) {
+ shouldCreate = true;
+ }
+ return shouldCreate;
+ }
+
+ /**
+ * mkdir of a directory -irrespective of what was there underneath.
+ * There are no checks for the directory existing, there not
+ * being a path there, etc. etc. Those are assumed to have
+ * taken place already
+ * @param absolutePath path to create
+ * @throws IOException IO problems
+ */
+ private void forceMkdir(Path absolutePath) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Making dir '" + absolutePath + "' in Swift");
+ }
+ //file is not found: it must be created
+ store.createDirectory(absolutePath);
+ }
+
+ /**
+ * List the statuses of the files/directories in the given path if the path is
+ * a directory.
+ *
+ * @param path given path
+ * @return the statuses of the files/directories in the given path
+ * @throws IOException
+ */
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SwiftFileSystem.listStatus for: " + path);
+ }
+ return store.listSubPaths(makeAbsolute(path), false, true);
+ }
+
+ /**
+ * This optional operation is not supported
+ */
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
+ throws IOException {
+ LOG.debug("SwiftFileSystem.append");
+ throw new SwiftUnsupportedFeatureException("Not supported: append()");
+ }
+
+ /**
+ * @param permission Currently ignored.
+ */
+ @Override
+ public FSDataOutputStream create(Path file, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress)
+ throws IOException {
+ LOG.debug("SwiftFileSystem.create");
+
+ FileStatus fileStatus = null;
+ Path absolutePath = makeAbsolute(file);
+ try {
+ fileStatus = getFileStatus(absolutePath);
+ } catch (FileNotFoundException e) {
+ //the file isn't there.
+ }
+
+ if (fileStatus != null) {
+ //the path exists -action depends on whether or not it is a directory,
+ //and what the overwrite policy is.
+
+ //What is clear at this point is that if the entry exists, there's
+ //no need to bother creating any parent entries
+ if (fileStatus.isDirectory()) {
+ //here someone is trying to create a file over a directory
+
+/* we can't throw an exception here as there is no easy way to distinguish
+ a file from the dir
+
+ throw new SwiftPathExistsException("Cannot create a file over a directory:"
+ + file);
+ */
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Overwriting either an empty file or a directory");
+ }
+ }
+ if (overwrite) {
+ //overwrite set -> delete the object.
+ store.delete(absolutePath, true);
+ } else {
+ throw new SwiftPathExistsException("Path exists: " + file);
+ }
+ } else {
+ // destination does not exist -trigger creation of the parent
+ Path parent = file.getParent();
+ if (parent != null) {
+ if (!mkdirs(parent)) {
+ throw new SwiftOperationFailedException(
+ "Mkdirs failed to create " + parent);
+ }
+ }
+ }
+
+ SwiftNativeOutputStream out = createSwiftOutputStream(file);
+ return new FSDataOutputStream(out, statistics);
+ }
+
+ /**
+ * Create the swift output stream
+ * @param path path to write to
+ * @return the new file
+ * @throws IOException
+ */
+ protected SwiftNativeOutputStream createSwiftOutputStream(Path path) throws
+ IOException {
+ long partSizeKB = getStore().getPartsizeKB();
+ return new SwiftNativeOutputStream(getConf(),
+ getStore(),
+ path.toUri().toString(),
+ partSizeKB);
+ }
+
+ /**
+ * Opens an FSDataInputStream at the indicated Path.
+ *
+ * @param path the file name to open
+ * @param bufferSize the size of the buffer to be used.
+ * @return the input stream
+ * @throws FileNotFoundException if the file is not found
+ * @throws IOException any IO problem
+ */
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ int bufferSizeKB = getStore().getBufferSizeKB();
+ long readBlockSize = bufferSizeKB * 1024L;
+ return open(path, bufferSize, readBlockSize);
+ }
+
+ /**
+ * Low-level operation to also set the block size for this operation
+ * @param path the file name to open
+ * @param bufferSize the size of the buffer to be used.
+ * @param readBlockSize how big should the read blockk/buffer size be?
+ * @return the input stream
+ * @throws FileNotFoundException if the file is not found
+ * @throws IOException any IO problem
+ */
+ public FSDataInputStream open(Path path,
+ int bufferSize,
+ long readBlockSize) throws IOException {
+ if (readBlockSize <= 0) {
+ throw new SwiftConfigurationException("Bad remote buffer size");
+ }
+ Path absolutePath = makeAbsolute(path);
+ return new FSDataInputStream(
+ new StrictBufferedFSInputStream(
+ new SwiftNativeInputStream(store,
+ statistics,
+ absolutePath,
+ readBlockSize),
+ bufferSize));
+ }
+
+ /**
+ * Renames Path src to Path dst. On swift this uses copy-and-delete
+ * and is not atomic .
+ *
+ * @param src path
+ * @param dst path
+ * @return true if directory renamed, false otherwise
+ * @throws IOException on problems
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+
+ try {
+ store.rename(makeAbsolute(src), makeAbsolute(dst));
+ //success
+ return true;
+ } catch (SwiftOperationFailedException e) {
+ //downgrade to a failure
+ return false;
+ } catch (FileNotFoundException e) {
+ //downgrade to a failure
+ return false;
+ }
+ }
+
+
+ /**
+ * Delete a file or directory
+ *
+ * @param path the path to delete.
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception if the
+ * directory is not empty
+ * case of a file the recursive can be set to either true or false.
+ * @return true if the object was deleted
+ * @throws IOException IO problems
+ */
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ try {
+ return store.delete(path, recursive);
+ } catch (FileNotFoundException e) {
+ //base path was not found.
+ return false;
+ }
+ }
+
+ /**
+ * Delete a file.
+ * This method is abstract in Hadoop 1.x; in 2.x+ it is non-abstract
+ * and deprecated
+ */
+ @Override
+ public boolean delete(Path f) throws IOException {
+ return delete(f, true);
+ }
+
+ /**
+ * Makes path absolute
+ *
+ * @param path path to file
+ * @return absolute path
+ */
+ protected Path makeAbsolute(Path path) {
+ if (path.isAbsolute()) {
+ return path;
+ }
+ return new Path(workingDir, path);
+ }
+
+ /**
+ * Get the current operation statistics
+ * @return a snapshot of the statistics
+ */
+ public List getOperationStatistics() {
+ return store.getOperationStatistics();
+ }
+
+ /**
+ * Low level method to do a deep listing of all entries, not stopping
+ * at the next directory entry. This is to let tests be confident that
+ * recursive deletes &c really are working.
+ * @param path path to recurse down
+ * @param newest ask for the newest data, potentially slower than not.
+ * @return a potentially empty array of file status
+ * @throws IOException any problem
+ */
+ @InterfaceAudience.Private
+ public FileStatus[] listRawFileStatus(Path path, boolean newest) throws IOException {
+ return store.listSubPaths(makeAbsolute(path), true, newest);
+ }
+
+ /**
+ * Get the number of partitions written by an output stream
+ * This is for testing
+ * @param outputStream output stream
+ * @return the #of partitions written by that stream
+ */
+ @InterfaceAudience.Private
+ public static int getPartitionsWritten(FSDataOutputStream outputStream) {
+ SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+ return snos.getPartitionsWritten();
+ }
+
+ private static SwiftNativeOutputStream getSwiftNativeOutputStream(
+ FSDataOutputStream outputStream) {
+ OutputStream wrappedStream = outputStream.getWrappedStream();
+ return (SwiftNativeOutputStream) wrappedStream;
+ }
+
+ /**
+ * Get the size of partitions written by an output stream
+ * This is for testing
+ *
+ * @param outputStream output stream
+ * @return partition size in bytes
+ */
+ @InterfaceAudience.Private
+ public static long getPartitionSize(FSDataOutputStream outputStream) {
+ SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+ return snos.getFilePartSize();
+ }
+
+ /**
+ * Get the the number of bytes written to an output stream
+ * This is for testing
+ *
+ * @param outputStream output stream
+ * @return partition size in bytes
+ */
+ @InterfaceAudience.Private
+ public static long getBytesWritten(FSDataOutputStream outputStream) {
+ SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+ return snos.getBytesWritten();
+ }
+
+ /**
+ * Get the the number of bytes uploaded by an output stream
+ * to the swift cluster.
+ * This is for testing
+ *
+ * @param outputStream output stream
+ * @return partition size in bytes
+ */
+ @InterfaceAudience.Private
+ public static long getBytesUploaded(FSDataOutputStream outputStream) {
+ SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+ return snos.getBytesUploaded();
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
new file mode 100644
index 00000000000..e30b47ba556
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
@@ -0,0 +1,977 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.http.HttpBodyContent;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.JSONUtil;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.codehaus.jackson.map.type.CollectionType;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * File system store implementation.
+ * Makes REST requests, parses data from responses
+ */
+public class SwiftNativeFileSystemStore {
+ private static final Pattern URI_PATTERN = Pattern.compile("\"\\S+?\"");
+ private static final String PATTERN = "EEE, d MMM yyyy hh:mm:ss zzz";
+ private static final Log LOG =
+ LogFactory.getLog(SwiftNativeFileSystemStore.class);
+ private URI uri;
+ private SwiftRestClient swiftRestClient;
+
+ /**
+ * Initalize the filesystem store -this creates the REST client binding.
+ *
+ * @param fsURI URI of the filesystem, which is used to map to the filesystem-specific
+ * options in the configuration file
+ * @param configuration configuration
+ * @throws IOException on any failure.
+ */
+ public void initialize(URI fsURI, Configuration configuration) throws IOException {
+ this.uri = fsURI;
+ this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
+ }
+
+ @Override
+ public String toString() {
+ return "SwiftNativeFileSystemStore with "
+ + swiftRestClient;
+ }
+
+ /**
+ * Get the default blocksize of this (bound) filesystem
+ * @return the blocksize returned for all FileStatus queries,
+ * which is used by the MapReduce splitter.
+ */
+ public long getBlocksize() {
+ return 1024L * swiftRestClient.getBlocksizeKB();
+ }
+
+ public long getPartsizeKB() {
+ return swiftRestClient.getPartSizeKB();
+ }
+
+ public int getBufferSizeKB() {
+ return swiftRestClient.getBufferSizeKB();
+ }
+
+ public int getThrottleDelay() {
+ return swiftRestClient.getThrottleDelay();
+ }
+ /**
+ * Upload a file/input stream of a specific length.
+ *
+ * @param path destination path in the swift filesystem
+ * @param inputStream input data. This is closed afterwards, always
+ * @param length length of the data
+ * @throws IOException on a problem
+ */
+ public void uploadFile(Path path, InputStream inputStream, long length)
+ throws IOException {
+ swiftRestClient.upload(toObjectPath(path), inputStream, length);
+ }
+
+ /**
+ * Upload part of a larger file.
+ *
+ * @param path destination path
+ * @param partNumber item number in the path
+ * @param inputStream input data
+ * @param length length of the data
+ * @throws IOException on a problem
+ */
+ public void uploadFilePart(Path path, int partNumber,
+ InputStream inputStream, long length)
+ throws IOException {
+
+ String stringPath = path.toUri().toString();
+ String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
+ partNumber);
+ if (stringPath.endsWith("/")) {
+ stringPath = stringPath.concat(partitionFilename);
+ } else {
+ stringPath = stringPath.concat("/").concat(partitionFilename);
+ }
+
+ swiftRestClient.upload(
+ new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
+ inputStream,
+ length);
+ }
+
+ /**
+ * Tell the Swift server to expect a multi-part upload by submitting
+ * a 0-byte file with the X-Object-Manifest header
+ *
+ * @param path path of final final
+ * @throws IOException
+ */
+ public void createManifestForPartUpload(Path path) throws IOException {
+ String pathString = toObjectPath(path).toString();
+ if (!pathString.endsWith("/")) {
+ pathString = pathString.concat("/");
+ }
+ if (pathString.startsWith("/")) {
+ pathString = pathString.substring(1);
+ }
+
+ swiftRestClient.upload(toObjectPath(path),
+ new ByteArrayInputStream(new byte[0]),
+ 0,
+ new Header(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString));
+ }
+
+ /**
+ * Get the metadata of an object
+ *
+ * @param path path
+ * @return file metadata. -or null if no headers were received back from the server.
+ * @throws IOException on a problem
+ * @throws FileNotFoundException if there is nothing at the end
+ */
+ public SwiftFileStatus getObjectMetadata(Path path) throws IOException {
+ return getObjectMetadata(path, true);
+ }
+
+ /**
+ * Get the HTTP headers, in case you really need the low-level
+ * metadata
+ * @param path path to probe
+ * @param newest newest or oldest?
+ * @return the header list
+ * @throws IOException IO problem
+ * @throws FileNotFoundException if there is nothing at the end
+ */
+ public Header[] getObjectHeaders(Path path, boolean newest)
+ throws IOException, FileNotFoundException {
+ SwiftObjectPath objectPath = toObjectPath(path);
+ return stat(objectPath, newest);
+ }
+
+ /**
+ * Get the metadata of an object
+ *
+ * @param path path
+ * @param newest flag to say "set the newest header", otherwise take any entry
+ * @return file metadata. -or null if no headers were received back from the server.
+ * @throws IOException on a problem
+ * @throws FileNotFoundException if there is nothing at the end
+ */
+ public SwiftFileStatus getObjectMetadata(Path path, boolean newest)
+ throws IOException, FileNotFoundException {
+
+ SwiftObjectPath objectPath = toObjectPath(path);
+ final Header[] headers = stat(objectPath, newest);
+ //no headers is treated as a missing file
+ if (headers.length == 0) {
+ throw new FileNotFoundException("Not Found " + path.toUri());
+ }
+
+ boolean isDir = false;
+ long length = 0;
+ long lastModified = 0 ;
+ for (Header header : headers) {
+ String headerName = header.getName();
+ if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
+ headerName.equals(SwiftProtocolConstants.X_CONTAINER_BYTES_USED)) {
+ length = 0;
+ isDir = true;
+ }
+ if (SwiftProtocolConstants.HEADER_CONTENT_LENGTH.equals(headerName)) {
+ length = Long.parseLong(header.getValue());
+ }
+ if (SwiftProtocolConstants.HEADER_LAST_MODIFIED.equals(headerName)) {
+ final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(PATTERN);
+ try {
+ lastModified = simpleDateFormat.parse(header.getValue()).getTime();
+ } catch (ParseException e) {
+ throw new SwiftException("Failed to parse " + header.toString(), e);
+ }
+ }
+ }
+ if (lastModified == 0) {
+ lastModified = System.currentTimeMillis();
+ }
+
+ Path correctSwiftPath = getCorrectSwiftPath(path);
+ return new SwiftFileStatus(length,
+ isDir,
+ 1,
+ getBlocksize(),
+ lastModified,
+ correctSwiftPath);
+ }
+
+ private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
+ IOException {
+ Header[] headers;
+ if (newest) {
+ headers = swiftRestClient.headRequest("getObjectMetadata-newest",
+ objectPath, SwiftRestClient.NEWEST);
+ } else {
+ headers = swiftRestClient.headRequest("getObjectMetadata",
+ objectPath);
+ }
+ return headers;
+ }
+
+ /**
+ * Get the object as an input stream
+ *
+ * @param path object path
+ * @return the input stream -this must be closed to terminate the connection
+ * @throws IOException IO problems
+ * @throws FileNotFoundException path doesn't resolve to an object
+ */
+ public HttpBodyContent getObject(Path path) throws IOException {
+ return swiftRestClient.getData(toObjectPath(path),
+ SwiftRestClient.NEWEST);
+ }
+
+ /**
+ * Get the input stream starting from a specific point.
+ *
+ * @param path path to object
+ * @param byteRangeStart starting point
+ * @param length no. of bytes
+ * @return an input stream that must be closed
+ * @throws IOException IO problems
+ */
+ public HttpBodyContent getObject(Path path, long byteRangeStart, long length)
+ throws IOException {
+ return swiftRestClient.getData(
+ toObjectPath(path), byteRangeStart, length);
+ }
+
+ /**
+ * List a directory.
+ * This is O(n) for the number of objects in this path.
+ *
+ *
+ *
+ * @param path working path
+ * @param listDeep ask for all the data
+ * @param newest ask for the newest data
+ * @return Collection of file statuses
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if the path does not exist
+ */
+ private List listDirectory(SwiftObjectPath path,
+ boolean listDeep,
+ boolean newest) throws IOException {
+ final byte[] bytes;
+ final ArrayList files = new ArrayList();
+ final Path correctSwiftPath = getCorrectSwiftPath(path);
+ try {
+ bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep);
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("" +
+ "File/Directory not found " + path);
+ }
+ if (SwiftUtils.isRootDir(path)) {
+ return Collections.emptyList();
+ } else {
+ throw e;
+ }
+ } catch (SwiftInvalidResponseException e) {
+ //bad HTTP error code
+ if (e.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
+ //this can come back on a root list if the container is empty
+ if (SwiftUtils.isRootDir(path)) {
+ return Collections.emptyList();
+ } else {
+ //NO_CONTENT returned on something other than the root directory;
+ //see if it is there, and convert to empty list or not found
+ //depending on whether the entry exists.
+ FileStatus stat = getObjectMetadata(correctSwiftPath, newest);
+
+ if (stat.isDirectory()) {
+ //it's an empty directory. state that
+ return Collections.emptyList();
+ } else {
+ //it's a file -return that as the status
+ files.add(stat);
+ return files;
+ }
+ }
+ } else {
+ //a different status code: rethrow immediately
+ throw e;
+ }
+ }
+
+ final CollectionType collectionType = JSONUtil.getJsonMapper().getTypeFactory().
+ constructCollectionType(List.class, SwiftObjectFileStatus.class);
+
+ final List fileStatusList =
+ JSONUtil.toObject(new String(bytes), collectionType);
+
+ //this can happen if user lists file /data/files/file
+ //in this case swift will return empty array
+ if (fileStatusList.isEmpty()) {
+ SwiftFileStatus objectMetadata = getObjectMetadata(correctSwiftPath,
+ newest);
+ if (objectMetadata.isFile()) {
+ files.add(objectMetadata);
+ }
+
+ return files;
+ }
+
+ for (SwiftObjectFileStatus status : fileStatusList) {
+ if (status.getName() != null) {
+ files.add(new SwiftFileStatus(status.getBytes(),
+ status.getBytes() == 0,
+ 1,
+ getBlocksize(),
+ status.getLast_modified().getTime(),
+ getCorrectSwiftPath(new Path(status.getName()))));
+ }
+ }
+
+ return files;
+ }
+
+ /**
+ * List all elements in this directory
+ *
+ *
+ *
+ * @param path path to work with
+ * @param recursive do a recursive get
+ * @param newest ask for the newest, or can some out of date data work?
+ * @return the file statuses, or an empty array if there are no children
+ * @throws IOException on IO problems
+ * @throws FileNotFoundException if the path is nonexistent
+ */
+ public FileStatus[] listSubPaths(Path path,
+ boolean recursive,
+ boolean newest) throws IOException {
+ final Collection fileStatuses;
+ fileStatuses = listDirectory(toDirPath(path), recursive, newest);
+ return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
+ }
+
+ /**
+ * Create a directory
+ *
+ * @param path path
+ * @throws IOException
+ */
+ public void createDirectory(Path path) throws IOException {
+ innerCreateDirectory(toDirPath(path));
+ }
+
+ /**
+ * The inner directory creation option. This only creates
+ * the dir at the given path, not any parent dirs.
+ * @param swiftObjectPath swift object path at which a 0-byte blob should be
+ * put
+ * @throws IOException IO problems
+ */
+ private void innerCreateDirectory(SwiftObjectPath swiftObjectPath)
+ throws IOException {
+
+ swiftRestClient.putRequest(swiftObjectPath);
+ }
+
+ private SwiftObjectPath toDirPath(Path path) throws
+ SwiftConfigurationException {
+ return SwiftObjectPath.fromPath(uri, path, false);
+ }
+
+ private SwiftObjectPath toObjectPath(Path path) throws
+ SwiftConfigurationException {
+ return SwiftObjectPath.fromPath(uri, path);
+ }
+
+ /**
+ * Try to find the specific server(s) on which the data lives
+ * @param path path to probe
+ * @return a possibly empty list of locations
+ * @throws IOException on problems determining the locations
+ */
+ public List getObjectLocation(Path path) throws IOException {
+ final byte[] objectLocation;
+ objectLocation = swiftRestClient.getObjectLocation(toObjectPath(path));
+ if (objectLocation == null || objectLocation.length == 0) {
+ //no object location, return an empty list
+ return new LinkedList();
+ }
+ return extractUris(new String(objectLocation), path);
+ }
+
+ /**
+ * deletes object from Swift
+ *
+ * @param path path to delete
+ * @return true if the path was deleted by this specific operation.
+ * @throws IOException on a failure
+ */
+ public boolean deleteObject(Path path) throws IOException {
+ SwiftObjectPath swiftObjectPath = toObjectPath(path);
+ if (!SwiftUtils.isRootDir(swiftObjectPath)) {
+ return swiftRestClient.delete(swiftObjectPath);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not deleting root directory entry");
+ }
+ return true;
+ }
+ }
+
+ /**
+ * deletes a directory from Swift. This is not recursive
+ *
+ * @param path path to delete
+ * @return true if the path was deleted by this specific operation -or
+ * the path was root and not acted on.
+ * @throws IOException on a failure
+ */
+ public boolean rmdir(Path path) throws IOException {
+ return deleteObject(path);
+ }
+
+ /**
+ * Does the object exist
+ *
+ * @param path object path
+ * @return true if the metadata of an object could be retrieved
+ * @throws IOException IO problems other than FileNotFound, which
+ * is downgraded to an object does not exist return code
+ */
+ public boolean objectExists(Path path) throws IOException {
+ return objectExists(toObjectPath(path));
+ }
+
+ /**
+ * Does the object exist
+ *
+ * @param path swift object path
+ * @return true if the metadata of an object could be retrieved
+ * @throws IOException IO problems other than FileNotFound, which
+ * is downgraded to an object does not exist return code
+ */
+ public boolean objectExists(SwiftObjectPath path) throws IOException {
+ try {
+ Header[] headers = swiftRestClient.headRequest("objectExists",
+ path,
+ SwiftRestClient.NEWEST);
+ //no headers is treated as a missing file
+ return headers.length != 0;
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Rename through copy-and-delete. this is a consequence of the
+ * Swift filesystem using the path as the hash
+ * into the Distributed Hash Table, "the ring" of filenames.
+ *
+ * Because of the nature of the operation, it is not atomic.
+ *
+ * @param src source file/dir
+ * @param dst destination
+ * @throws IOException IO failure
+ * @throws SwiftOperationFailedException if the rename failed
+ * @throws FileNotFoundException if the source directory is missing, or
+ * the parent directory of the destination
+ */
+ public void rename(Path src, Path dst)
+ throws FileNotFoundException, SwiftOperationFailedException, IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("mv " + src + " " + dst);
+ }
+ boolean renamingOnToSelf = src.equals(dst);
+
+ SwiftObjectPath srcObject = toObjectPath(src);
+ SwiftObjectPath destObject = toObjectPath(dst);
+
+ if (SwiftUtils.isRootDir(srcObject)) {
+ throw new SwiftOperationFailedException("cannot rename root dir");
+ }
+
+ final SwiftFileStatus srcMetadata;
+ srcMetadata = getObjectMetadata(src);
+ SwiftFileStatus dstMetadata;
+ try {
+ dstMetadata = getObjectMetadata(dst);
+ } catch (FileNotFoundException e) {
+ //destination does not exist.
+ LOG.debug("Destination does not exist");
+ dstMetadata = null;
+ }
+
+ //check to see if the destination parent directory exists
+ Path srcParent = src.getParent();
+ Path dstParent = dst.getParent();
+ //skip the overhead of a HEAD call if the src and dest share the same
+ //parent dir (in which case the dest dir exists), or the destination
+ //directory is root, in which case it must also exist
+ if (dstParent != null && !dstParent.equals(srcParent)) {
+ try {
+ getObjectMetadata(dstParent);
+ } catch (FileNotFoundException e) {
+ //destination parent doesn't exist; bail out
+ LOG.debug("destination parent directory " + dstParent + " doesn't exist");
+ throw e;
+ }
+ }
+
+ boolean destExists = dstMetadata != null;
+ boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
+ //calculate the destination
+ SwiftObjectPath destPath;
+
+ //enum the child entries and everything underneath
+ List childStats = listDirectory(srcObject, true, true);
+ boolean srcIsFile = !srcMetadata.isDir();
+ if (srcIsFile) {
+
+ //source is a simple file OR a partitioned file
+ // outcomes:
+ // #1 dest exists and is file: fail
+ // #2 dest exists and is dir: destination path becomes under dest dir
+ // #3 dest does not exist: use dest as name
+ if (destExists) {
+
+ if (destIsDir) {
+ //outcome #2 -move to subdir of dest
+ destPath = toObjectPath(new Path(dst, src.getName()));
+ } else {
+ //outcome #1 dest it's a file: fail if differeent
+ if (!renamingOnToSelf) {
+ throw new SwiftOperationFailedException(
+ "cannot rename a file over one that already exists");
+ } else {
+ //is mv self self where self is a file. this becomes a no-op
+ LOG.debug("Renaming file onto self: no-op => success");
+ return;
+ }
+ }
+ } else {
+ //outcome #3 -new entry
+ destPath = toObjectPath(dst);
+ }
+ int childCount = childStats.size();
+ //here there is one of:
+ // - a single object ==> standard file
+ // ->
+ if (childCount == 0) {
+ copyThenDeleteObject(srcObject, destPath);
+ } else {
+ //do the copy
+ SwiftUtils.debug(LOG, "Source file appears to be partitioned." +
+ " copying file and deleting children");
+
+ copyObject(srcObject, destPath);
+ for (FileStatus stat : childStats) {
+ SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
+ deleteObject(stat.getPath());
+ }
+
+ swiftRestClient.delete(srcObject);
+ }
+ } else {
+
+ //here the source exists and is a directory
+ // outcomes (given we know the parent dir exists if we get this far)
+ // #1 destination is a file: fail
+ // #2 destination is a directory: create a new dir under that one
+ // #3 destination doesn't exist: create a new dir with that name
+ // #3 and #4 are only allowed if the dest path is not == or under src
+
+
+ if (destExists && !destIsDir) {
+ // #1 destination is a file: fail
+ throw new SwiftOperationFailedException(
+ "the source is a directory, but not the destination");
+ }
+ Path targetPath;
+ if (destExists) {
+ // #2 destination is a directory: create a new dir under that one
+ targetPath = new Path(dst, src.getName());
+ } else {
+ // #3 destination doesn't exist: create a new dir with that name
+ targetPath = dst;
+ }
+ SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
+ //final check for any recursive operations
+ if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
+ //you can't rename a directory onto itself
+ throw new SwiftOperationFailedException(
+ "cannot move a directory under itself");
+ }
+
+
+ LOG.info("mv " + srcObject + " " + targetPath);
+
+ logDirectory("Directory to copy ", srcObject, childStats);
+
+ // iterative copy of everything under the directory.
+ // by listing all children this can be done iteratively
+ // rather than recursively -everything in this list is either a file
+ // or a 0-byte-len file pretending to be a directory.
+ String srcURI = src.toUri().toString();
+ int prefixStripCount = srcURI.length() + 1;
+ for (FileStatus fileStatus : childStats) {
+ Path copySourcePath = fileStatus.getPath();
+ String copySourceURI = copySourcePath.toUri().toString();
+
+ String copyDestSubPath = copySourceURI.substring(prefixStripCount);
+
+ Path copyDestPath = new Path(targetPath, copyDestSubPath);
+ if (LOG.isTraceEnabled()) {
+ //trace to debug some low-level rename path problems; retained
+ //in case they ever come back.
+ LOG.trace("srcURI=" + srcURI
+ + "; copySourceURI=" + copySourceURI
+ + "; copyDestSubPath=" + copyDestSubPath
+ + "; copyDestPath=" + copyDestPath);
+ }
+ SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
+
+ try {
+ copyThenDeleteObject(toObjectPath(copySourcePath),
+ copyDestination);
+ } catch (FileNotFoundException e) {
+ LOG.info("Skipping rename of " + copySourcePath);
+ }
+ //add a throttle delay
+ throttle();
+ }
+ //now rename self. If missing, create the dest directory and warn
+ if (!SwiftUtils.isRootDir(srcObject)) {
+ try {
+ copyThenDeleteObject(srcObject,
+ targetObjectPath);
+ } catch (FileNotFoundException e) {
+ //create the destination directory
+ LOG.warn("Source directory deleted during rename", e);
+ innerCreateDirectory(destObject);
+ }
+ }
+ }
+ }
+
+ /**
+ * Debug action to dump directory statuses to the debug log
+ *
+ * @param message explanation
+ * @param objectPath object path (can be null)
+ * @param statuses listing output
+ */
+ private void logDirectory(String message, SwiftObjectPath objectPath,
+ Iterable statuses) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(message + ": listing of " + objectPath);
+ for (FileStatus fileStatus : statuses) {
+ LOG.debug(fileStatus.getPath());
+ }
+ }
+ }
+
+ public void copy(Path srcKey, Path dstKey) throws IOException {
+ SwiftObjectPath srcObject = toObjectPath(srcKey);
+ SwiftObjectPath destObject = toObjectPath(dstKey);
+ swiftRestClient.copyObject(srcObject, destObject);
+ }
+
+
+ /**
+ * Copy an object then, if the copy worked, delete it.
+ * If the copy failed, the source object is not deleted.
+ *
+ * @param srcObject source object path
+ * @param destObject destination object path
+ * @throws IOException IO problems
+
+ */
+ private void copyThenDeleteObject(SwiftObjectPath srcObject,
+ SwiftObjectPath destObject) throws
+ IOException {
+
+
+ //do the copy
+ copyObject(srcObject, destObject);
+ //getting here means the copy worked
+ swiftRestClient.delete(srcObject);
+ }
+ /**
+ * Copy an object
+ * @param srcObject source object path
+ * @param destObject destination object path
+ * @throws IOException IO problems
+ */
+ private void copyObject(SwiftObjectPath srcObject,
+ SwiftObjectPath destObject) throws
+ IOException {
+ if (srcObject.isEqualToOrParentOf(destObject)) {
+ throw new SwiftException(
+ "Can't copy " + srcObject + " onto " + destObject);
+ }
+ //do the copy
+ boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
+ if (!copySucceeded) {
+ throw new SwiftException("Copy of " + srcObject + " to "
+ + destObject + "failed");
+ }
+ }
+
+ /**
+ * Take a Hadoop path and return one which uses the URI prefix and authority
+ * of this FS. It doesn't make a relative path absolute
+ * @param path path in
+ * @return path with a URI bound to this FS
+ * @throws SwiftException URI cannot be created.
+ */
+ public Path getCorrectSwiftPath(Path path) throws
+ SwiftException {
+ try {
+ final URI fullUri = new URI(uri.getScheme(),
+ uri.getAuthority(),
+ path.toUri().getPath(),
+ null,
+ null);
+
+ return new Path(fullUri);
+ } catch (URISyntaxException e) {
+ throw new SwiftException("Specified path " + path + " is incorrect", e);
+ }
+ }
+
+ /**
+ * Builds a hadoop-Path from a swift path, inserting the URI authority
+ * of this FS instance
+ * @param path swift object path
+ * @return Hadoop path
+ * @throws SwiftException if the URI couldn't be created.
+ */
+ private Path getCorrectSwiftPath(SwiftObjectPath path) throws
+ SwiftException {
+ try {
+ final URI fullUri = new URI(uri.getScheme(),
+ uri.getAuthority(),
+ path.getObject(),
+ null,
+ null);
+
+ return new Path(fullUri);
+ } catch (URISyntaxException e) {
+ throw new SwiftException("Specified path " + path + " is incorrect", e);
+ }
+ }
+
+
+ /**
+ * extracts URIs from json
+ * @param json json to parse
+ * @param path path (used in exceptions)
+ * @return URIs
+ * @throws SwiftOperationFailedException on any problem parsing the JSON
+ */
+ public static List extractUris(String json, Path path) throws
+ SwiftOperationFailedException {
+ final Matcher matcher = URI_PATTERN.matcher(json);
+ final List result = new ArrayList();
+ while (matcher.find()) {
+ final String s = matcher.group();
+ final String uri = s.substring(1, s.length() - 1);
+ try {
+ URI createdUri = URI.create(uri);
+ result.add(createdUri);
+ } catch (IllegalArgumentException e) {
+ //failure to create the URI, which means this is bad JSON. Convert
+ //to an exception with useful text
+ throw new SwiftOperationFailedException(
+ String.format(
+ "could not convert \"%s\" into a URI." +
+ " source: %s " +
+ " first JSON: %s",
+ uri, path, json.substring(0, 256)));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Insert a throttled wait if the throttle delay >0
+ * @throws InterruptedIOException if interrupted during sleep
+ */
+ public void throttle() throws InterruptedIOException {
+ int throttleDelay = getThrottleDelay();
+ if (throttleDelay > 0) {
+ try {
+ Thread.sleep(throttleDelay);
+ } catch (InterruptedException e) {
+ //convert to an IOE
+ throw (InterruptedIOException) new InterruptedIOException(e.toString())
+ .initCause(e);
+ }
+ }
+ }
+
+ /**
+ * Get the current operation statistics
+ * @return a snapshot of the statistics
+ */
+ public List getOperationStatistics() {
+ return swiftRestClient.getOperationStatistics();
+ }
+
+
+ /**
+ * Delete the entire tree. This is an internal one with slightly different
+ * behavior: if an entry is missing, a {@link FileNotFoundException} is
+ * raised. This lets the caller distinguish a file not found with
+ * other reasons for failure, so handles race conditions in recursive
+ * directory deletes better.
+ *
+ * The problem being addressed is: caller A requests a recursive directory
+ * of directory /dir ; caller B requests a delete of a file /dir/file,
+ * between caller A enumerating the files contents, and requesting a delete
+ * of /dir/file. We want to recognise the special case
+ * "directed file is no longer there" and not convert that into a failure
+ *
+ * @param absolutePath the path to delete.
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception if the
+ * directory is not empty
+ * case of a file the recursive can be set to either true or false.
+ * @return true if the object was deleted
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if a file/dir being deleted is not there -
+ * this includes entries below the specified path, (if the path is a dir
+ * and recursive is true)
+ */
+ public boolean delete(Path absolutePath, boolean recursive) throws IOException {
+ Path swiftPath = getCorrectSwiftPath(absolutePath);
+ SwiftUtils.debug(LOG, "Deleting path '%s' recursive=%b",
+ absolutePath,
+ recursive);
+ boolean askForNewest = true;
+ SwiftFileStatus fileStatus = getObjectMetadata(swiftPath, askForNewest);
+
+ //ask for the file/dir status, but don't demand the newest, as we
+ //don't mind if the directory has changed
+ //list all entries under this directory.
+ //this will throw FileNotFoundException if the file isn't there
+ FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest);
+ if (statuses == null) {
+ //the directory went away during the non-atomic stages of the operation.
+ // Return false as it was not this thread doing the deletion.
+ SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'",
+ absolutePath,
+ recursive);
+ return false;
+ }
+ int filecount = statuses.length;
+ SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
+ absolutePath,
+ filecount);
+
+ if (filecount == 0) {
+ //it's an empty directory or a path
+ rmdir(absolutePath);
+ return true;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ SwiftUtils.debug(LOG, SwiftUtils.fileStatsToString(statuses, "\n"));
+ }
+
+ if (filecount == 1 && swiftPath.equals(statuses[0].getPath())) {
+ // 1 entry => simple file and it is the target
+ //simple file: delete it
+ SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
+ deleteObject(absolutePath);
+ return true;
+ }
+
+ //>1 entry implies directory with children. Run through them,
+ // but first check for the recursive flag and reject it *unless it looks
+ // like a partitioned file (len > 0 && has children)
+ if (!fileStatus.isDir()) {
+ LOG.debug("Multiple child entries but entry has data: assume partitioned");
+ } else if (!recursive) {
+ //if there are children, unless this is a recursive operation, fail immediately
+ throw new SwiftOperationFailedException("Directory " + fileStatus
+ + " is not empty: "
+ + SwiftUtils.fileStatsToString(
+ statuses, "; "));
+ }
+
+ //delete the entries. including ourself.
+ for (FileStatus entryStatus : statuses) {
+ Path entryPath = entryStatus.getPath();
+ try {
+ boolean deleted = deleteObject(entryPath);
+ if (!deleted) {
+ SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
+ entryPath);
+ }
+ } catch (FileNotFoundException e) {
+ //the path went away -race conditions.
+ //do not fail, as the outcome is still OK.
+ SwiftUtils.debug(LOG, "Path '%s' is no longer present; continuing",
+ entryPath);
+ }
+ throttle();
+ }
+ //now delete self
+ SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
+ deleteObject(absolutePath);
+
+ return true;
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java
new file mode 100644
index 00000000000..6574762e406
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java
@@ -0,0 +1,379 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.http.HttpBodyContent;
+import org.apache.hadoop.fs.swift.http.HttpInputStreamWithRelease;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * The input stream from remote Swift blobs.
+ * The class attempts to be buffer aware, and react to a forward seek operation
+ * by trying to scan ahead through the current block of data to find it.
+ * This accelerates some operations that do a lot of seek()/read() actions,
+ * including work (such as in the MR engine) that do a seek() immediately after
+ * an open().
+ */
+class SwiftNativeInputStream extends FSInputStream {
+
+ private static final Log LOG = LogFactory.getLog(SwiftNativeInputStream.class);
+
+ /**
+ * range requested off the server: {@value}
+ */
+ private final long bufferSize;
+
+ /**
+ * File nativeStore instance
+ */
+ private final SwiftNativeFileSystemStore nativeStore;
+
+ /**
+ * Hadoop statistics. Used to get info about number of reads, writes, etc.
+ */
+ private final FileSystem.Statistics statistics;
+
+ /**
+ * Data input stream
+ */
+ private HttpInputStreamWithRelease httpStream;
+
+ /**
+ * File path
+ */
+ private final Path path;
+
+ /**
+ * Current position
+ */
+ private long pos = 0;
+
+ /**
+ * Length of the file picked up at start time
+ */
+ private long contentLength = -1;
+
+ /**
+ * Why the stream is closed
+ */
+ private String reasonClosed = "unopened";
+
+ /**
+ * Offset in the range requested last
+ */
+ private long rangeOffset = 0;
+
+ public SwiftNativeInputStream(SwiftNativeFileSystemStore storeNative,
+ FileSystem.Statistics statistics, Path path, long bufferSize)
+ throws IOException {
+ this.nativeStore = storeNative;
+ this.statistics = statistics;
+ this.path = path;
+ if (bufferSize <= 0) {
+ throw new IllegalArgumentException("Invalid buffer size");
+ }
+ this.bufferSize = bufferSize;
+ //initial buffer fill
+ this.httpStream = storeNative.getObject(path).getInputStream();
+ //fillBuffer(0);
+ }
+
+ /**
+ * Move to a new position within the file relative to where the pointer is now.
+ * Always call from a synchronized clause
+ * @param offset offset
+ */
+ private synchronized void incPos(int offset) {
+ pos += offset;
+ rangeOffset += offset;
+ SwiftUtils.trace(LOG, "Inc: pos=%d bufferOffset=%d", pos, rangeOffset);
+ }
+
+ /**
+ * Update the start of the buffer; always call from a sync'd clause
+ * @param seekPos position sought.
+ * @param contentLength content length provided by response (may be -1)
+ */
+ private synchronized void updateStartOfBufferPosition(long seekPos,
+ long contentLength) {
+ //reset the seek pointer
+ pos = seekPos;
+ //and put the buffer offset to 0
+ rangeOffset = 0;
+ this.contentLength = contentLength;
+ SwiftUtils.trace(LOG, "Move: pos=%d; bufferOffset=%d; contentLength=%d",
+ pos,
+ rangeOffset,
+ contentLength);
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ verifyOpen();
+ int result = -1;
+ try {
+ result = httpStream.read();
+ } catch (IOException e) {
+ String msg = "IOException while reading " + path
+ + ": " +e + ", attempting to reopen.";
+ LOG.debug(msg, e);
+ if (reopenBuffer()) {
+ result = httpStream.read();
+ }
+ }
+ if (result != -1) {
+ incPos(1);
+ }
+ if (statistics != null && result != -1) {
+ statistics.incrementBytesRead(1);
+ }
+ return result;
+ }
+
+ @Override
+ public synchronized int read(byte[] b, int off, int len) throws IOException {
+ SwiftUtils.debug(LOG, "read(buffer, %d, %d)", off, len);
+ SwiftUtils.validateReadArgs(b, off, len);
+ int result = -1;
+ try {
+ verifyOpen();
+ result = httpStream.read(b, off, len);
+ } catch (IOException e) {
+ //other IO problems are viewed as transient and re-attempted
+ LOG.info("Received IOException while reading '" + path +
+ "', attempting to reopen: " + e);
+ LOG.debug("IOE on read()" + e, e);
+ if (reopenBuffer()) {
+ result = httpStream.read(b, off, len);
+ }
+ }
+ if (result > 0) {
+ incPos(result);
+ if (statistics != null) {
+ statistics.incrementBytesRead(result);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Re-open the buffer
+ * @return true iff more data could be added to the buffer
+ * @throws IOException if not
+ */
+ private boolean reopenBuffer() throws IOException {
+ innerClose("reopening buffer to trigger refresh");
+ boolean success = false;
+ try {
+ fillBuffer(pos);
+ success = true;
+ } catch (EOFException eof) {
+ //the EOF has been reached
+ this.reasonClosed = "End of file";
+ }
+ return success;
+ }
+
+ /**
+ * close the stream. After this the stream is not usable -unless and until
+ * it is re-opened (which can happen on some of the buffer ops)
+ * This method is thread-safe and idempotent.
+ *
+ * @throws IOException on IO problems.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ innerClose("closed");
+ }
+
+ private void innerClose(String reason) throws IOException {
+ try {
+ if (httpStream != null) {
+ reasonClosed = reason;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Closing HTTP input stream : " + reason);
+ }
+ httpStream.close();
+ }
+ } finally {
+ httpStream = null;
+ }
+ }
+
+ /**
+ * Assume that the connection is not closed: throws an exception if it is
+ * @throws SwiftConnectionClosedException
+ */
+ private void verifyOpen() throws SwiftConnectionClosedException {
+ if (httpStream == null) {
+ throw new SwiftConnectionClosedException(reasonClosed);
+ }
+ }
+
+ @Override
+ public synchronized String toString() {
+ return "SwiftNativeInputStream" +
+ " position=" + pos
+ + " buffer size = " + bufferSize
+ + " "
+ + (httpStream != null ? httpStream.toString()
+ : (" no input stream: " + reasonClosed));
+ }
+
+ /**
+ * Treats any finalize() call without the input stream being closed
+ * as a serious problem, logging at error level
+ * @throws Throwable n/a
+ */
+ @Override
+ protected void finalize() throws Throwable {
+ if (httpStream != null) {
+ LOG.error(
+ "Input stream is leaking handles by not being closed() properly: "
+ + httpStream.toString());
+ }
+ }
+
+ /**
+ * Read through the specified number of bytes.
+ * The implementation iterates a byte a time, which may seem inefficient
+ * compared to the read(bytes[]) method offered by input streams.
+ * However, if you look at the code that implements that method, it comes
+ * down to read() one char at a time -only here the return value is discarded.
+ *
+ *
+ * This is a no-op if the stream is closed
+ * @param bytes number of bytes to read.
+ * @throws IOException IO problems
+ * @throws SwiftException if a read returned -1.
+ */
+ private int chompBytes(long bytes) throws IOException {
+ int count = 0;
+ if (httpStream != null) {
+ int result;
+ for (long i = 0; i < bytes; i++) {
+ result = httpStream.read();
+ if (result < 0) {
+ throw new SwiftException("Received error code while chomping input");
+ }
+ count ++;
+ incPos(1);
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Seek to an offset. If the data is already in the buffer, move to it
+ * @param targetPos target position
+ * @throws IOException on any problem
+ */
+ @Override
+ public synchronized void seek(long targetPos) throws IOException {
+ if (targetPos < 0) {
+ throw new IOException("Negative Seek offset not supported");
+ }
+ //there's some special handling of near-local data
+ //as the seek can be omitted if it is in/adjacent
+ long offset = targetPos - pos;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Seek to " + targetPos + "; current pos =" + pos
+ + "; offset="+offset);
+ }
+ if (offset == 0) {
+ LOG.debug("seek is no-op");
+ return;
+ }
+
+ if (offset < 0) {
+ LOG.debug("seek is backwards");
+ } else if ((rangeOffset + offset < bufferSize)) {
+ //if the seek is in range of that requested, scan forwards
+ //instead of closing and re-opening a new HTTP connection
+ SwiftUtils.debug(LOG,
+ "seek is within current stream"
+ + "; pos= %d ; targetPos=%d; "
+ + "offset= %d ; bufferOffset=%d",
+ pos, targetPos, offset, rangeOffset);
+ try {
+ LOG.debug("chomping ");
+ chompBytes(offset);
+ } catch (IOException e) {
+ //this is assumed to be recoverable with a seek -or more likely to fail
+ LOG.debug("while chomping ",e);
+ }
+ if (targetPos - pos == 0) {
+ LOG.trace("chomping successful");
+ return;
+ }
+ LOG.trace("chomping failed");
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Seek is beyond buffer size of " + bufferSize);
+ }
+ }
+
+ innerClose("seeking to " + targetPos);
+ fillBuffer(targetPos);
+ }
+
+ /**
+ * Fill the buffer from the target position
+ * If the target position == current position, the
+ * read still goes ahead; this is a way of handling partial read failures
+ * @param targetPos target position
+ * @throws IOException IO problems on the read
+ */
+ private void fillBuffer(long targetPos) throws IOException {
+ long length = targetPos + bufferSize;
+ SwiftUtils.debug(LOG, "Fetching %d bytes starting at %d", length, targetPos);
+ HttpBodyContent blob = nativeStore.getObject(path, targetPos, length);
+ httpStream = blob.getInputStream();
+ updateStartOfBufferPosition(targetPos, blob.getContentLength());
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ return pos;
+ }
+
+ /**
+ * This FS doesn't explicitly support multiple data sources, so
+ * return false here.
+ * @param targetPos the desired target position
+ * @return true if a new source of the data has been set up
+ * as the source of future reads
+ * @throws IOException IO problems
+ */
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java
new file mode 100644
index 00000000000..4603ded264d
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Output stream, buffers data on local disk.
+ * Writes to Swift on the close() method, unless the
+ * file is significantly large that it is being written as partitions.
+ * In this case, the first partition is written on the first write that puts
+ * data over the partition, as may later writes. The close() then causes
+ * the final partition to be written, along with a partition manifest.
+ */
+class SwiftNativeOutputStream extends OutputStream {
+ public static final int ATTEMPT_LIMIT = 3;
+ private long filePartSize;
+ private static final Log LOG =
+ LogFactory.getLog(SwiftNativeOutputStream.class);
+ private Configuration conf;
+ private String key;
+ private File backupFile;
+ private OutputStream backupStream;
+ private SwiftNativeFileSystemStore nativeStore;
+ private boolean closed;
+ private int partNumber;
+ private long blockOffset;
+ private long bytesWritten;
+ private long bytesUploaded;
+ private boolean partUpload = false;
+ final byte[] oneByte = new byte[1];
+
+ /**
+ * Create an output stream
+ * @param conf configuration to use
+ * @param nativeStore native store to write through
+ * @param key the key to write
+ * @param partSizeKB the partition size
+ * @throws IOException
+ */
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ public SwiftNativeOutputStream(Configuration conf,
+ SwiftNativeFileSystemStore nativeStore,
+ String key,
+ long partSizeKB) throws IOException {
+ this.conf = conf;
+ this.key = key;
+ this.backupFile = newBackupFile();
+ this.nativeStore = nativeStore;
+ this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile));
+ this.partNumber = 1;
+ this.blockOffset = 0;
+ this.filePartSize = 1024L * partSizeKB;
+ }
+
+ private File newBackupFile() throws IOException {
+ File dir = new File(conf.get("hadoop.tmp.dir"));
+ if (!dir.mkdirs() && !dir.exists()) {
+ throw new SwiftException("Cannot create Swift buffer directory: " + dir);
+ }
+ File result = File.createTempFile("output-", ".tmp", dir);
+ result.deleteOnExit();
+ return result;
+ }
+
+ /**
+ * Flush the local backing stream.
+ * This does not trigger a flush of data to the remote blobstore.
+ * @throws IOException
+ */
+ @Override
+ public void flush() throws IOException {
+ backupStream.flush();
+ }
+
+ /**
+ * check that the output stream is open
+ *
+ * @throws SwiftException if it is not
+ */
+ private synchronized void verifyOpen() throws SwiftException {
+ if (closed) {
+ throw new SwiftException("Output stream is closed");
+ }
+ }
+
+ /**
+ * Close the stream. This will trigger the upload of all locally cached
+ * data to the remote blobstore.
+ * @throws IOException IO problems uploading the data.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+
+ try {
+ closed = true;
+ //formally declare as closed.
+ backupStream.close();
+ backupStream = null;
+ Path keypath = new Path(key);
+ if (partUpload) {
+ partUpload(true);
+ nativeStore.createManifestForPartUpload(keypath);
+ } else {
+ uploadOnClose(keypath);
+ }
+ } finally {
+ delete(backupFile);
+ backupFile = null;
+ }
+ assert backupStream == null: "backup stream has been reopened";
+ }
+
+ /**
+ * Upload a file when closed, either in one go, or, if the file is
+ * already partitioned, by uploading the remaining partition and a manifest.
+ * @param keypath key as a path
+ * @throws IOException IO Problems
+ */
+ private void uploadOnClose(Path keypath) throws IOException {
+ boolean uploadSuccess = false;
+ int attempt = 0;
+ while (!uploadSuccess) {
+ try {
+ ++attempt;
+ bytesUploaded += uploadFileAttempt(keypath, attempt);
+ uploadSuccess = true;
+ } catch (IOException e) {
+ LOG.info("Upload failed " + e, e);
+ if (attempt > ATTEMPT_LIMIT) {
+ throw e;
+ }
+ }
+ }
+}
+
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ private long uploadFileAttempt(Path keypath, int attempt) throws IOException {
+ long uploadLen = backupFile.length();
+ SwiftUtils.debug(LOG, "Closing write of file %s;" +
+ " localfile=%s of length %d - attempt %d",
+ key,
+ backupFile,
+ uploadLen,
+ attempt);
+
+ nativeStore.uploadFile(keypath,
+ new FileInputStream(backupFile),
+ uploadLen);
+ return uploadLen;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ if(!closed) {
+ LOG.warn("stream not closed");
+ }
+ if (backupFile != null) {
+ LOG.warn("Leaking backing file " + backupFile);
+ }
+ }
+
+ private void delete(File file) {
+ if (file != null) {
+ SwiftUtils.debug(LOG, "deleting %s", file);
+ if (!file.delete()) {
+ LOG.warn("Could not delete " + file);
+ }
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ //insert to a one byte array
+ oneByte[0] = (byte) b;
+ //then delegate to the array writing routine
+ write(oneByte, 0, 1);
+ }
+
+ @Override
+ public synchronized void write(byte[] buffer, int offset, int len) throws
+ IOException {
+ //validate args
+ if (offset < 0 || len < 0 || (offset + len) > buffer.length) {
+ throw new IndexOutOfBoundsException("Invalid offset/length for write");
+ }
+ //validate the output stream
+ verifyOpen();
+ SwiftUtils.debug(LOG, " write(offset=%d, len=%d)", offset, len);
+
+ // if the size of file is greater than the partition limit
+ while (blockOffset + len >= filePartSize) {
+ // - then partition the blob and upload as many partitions
+ // are needed.
+ //how many bytes to write for this partition.
+ int subWriteLen = (int) (filePartSize - blockOffset);
+ if (subWriteLen < 0 || subWriteLen > len) {
+ throw new SwiftInternalStateException("Invalid subwrite len: "
+ + subWriteLen
+ + " -buffer len: " + len);
+ }
+ writeToBackupStream(buffer, offset, subWriteLen);
+ //move the offset along and length down
+ offset += subWriteLen;
+ len -= subWriteLen;
+ //now upload the partition that has just been filled up
+ // (this also sets blockOffset=0)
+ partUpload(false);
+ }
+ //any remaining data is now written
+ writeToBackupStream(buffer, offset, len);
+ }
+
+ /**
+ * Write to the backup stream.
+ * Guarantees:
+ *
+ * backupStream is open
+ * blockOffset + len < filePartSize
+ *
+ * @param buffer buffer to write
+ * @param offset offset in buffer
+ * @param len length of write.
+ * @throws IOException backup stream write failing
+ */
+ private void writeToBackupStream(byte[] buffer, int offset, int len) throws
+ IOException {
+ assert len >= 0 : "remainder to write is negative";
+ SwiftUtils.debug(LOG," writeToBackupStream(offset=%d, len=%d)", offset, len);
+ if (len == 0) {
+ //no remainder -downgrade to noop
+ return;
+ }
+
+ //write the new data out to the backup stream
+ backupStream.write(buffer, offset, len);
+ //increment the counters
+ blockOffset += len;
+ bytesWritten += len;
+ }
+
+ /**
+ * Upload a single partition. This deletes the local backing-file,
+ * and re-opens it to create a new one.
+ * @param closingUpload is this the final upload of an upload
+ * @throws IOException on IO problems
+ */
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ private void partUpload(boolean closingUpload) throws IOException {
+ if (backupStream != null) {
+ backupStream.close();
+ }
+
+ if (closingUpload && partUpload && backupFile.length() == 0) {
+ //skipping the upload if
+ // - it is close time
+ // - the final partition is 0 bytes long
+ // - one part has already been written
+ SwiftUtils.debug(LOG, "skipping upload of 0 byte final partition");
+ delete(backupFile);
+ } else {
+ partUpload = true;
+ boolean uploadSuccess = false;
+ int attempt = 0;
+ while(!uploadSuccess) {
+ try {
+ ++attempt;
+ bytesUploaded += uploadFilePartAttempt(attempt);
+ uploadSuccess = true;
+ } catch (IOException e) {
+ LOG.info("Upload failed " + e, e);
+ if (attempt > ATTEMPT_LIMIT) {
+ throw e;
+ }
+ }
+ }
+ delete(backupFile);
+ partNumber++;
+ blockOffset = 0;
+ if (!closingUpload) {
+ //if not the final upload, create a new output stream
+ backupFile = newBackupFile();
+ backupStream =
+ new BufferedOutputStream(new FileOutputStream(backupFile));
+ }
+ }
+ }
+
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ private long uploadFilePartAttempt(int attempt) throws IOException {
+ long uploadLen = backupFile.length();
+ SwiftUtils.debug(LOG, "Uploading part %d of file %s;" +
+ " localfile=%s of length %d - attempt %d",
+ partNumber,
+ key,
+ backupFile,
+ uploadLen,
+ attempt);
+ nativeStore.uploadFilePart(new Path(key),
+ partNumber,
+ new FileInputStream(backupFile),
+ uploadLen);
+ return uploadLen;
+ }
+
+ /**
+ * Get the file partition size
+ * @return the partition size
+ */
+ long getFilePartSize() {
+ return filePartSize;
+ }
+
+ /**
+ * Query the number of partitions written
+ * This is intended for testing
+ * @return the of partitions already written to the remote FS
+ */
+ synchronized int getPartitionsWritten() {
+ return partNumber - 1;
+ }
+
+ /**
+ * Get the number of bytes written to the output stream.
+ * This should always be less than or equal to bytesUploaded.
+ * @return the number of bytes written to this stream
+ */
+ long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ /**
+ * Get the number of bytes uploaded to remote Swift cluster.
+ * bytesUploaded -bytesWritten = the number of bytes left to upload
+ * @return the number of bytes written to the remote endpoint
+ */
+ long getBytesUploaded() {
+ return bytesUploaded;
+ }
+
+ @Override
+ public String toString() {
+ return "SwiftNativeOutputStream{" +
+ ", key='" + key + '\'' +
+ ", backupFile=" + backupFile +
+ ", closed=" + closed +
+ ", filePartSize=" + filePartSize +
+ ", partNumber=" + partNumber +
+ ", blockOffset=" + blockOffset +
+ ", partUpload=" + partUpload +
+ ", nativeStore=" + nativeStore +
+ ", bytesWritten=" + bytesWritten +
+ ", bytesUploaded=" + bytesUploaded +
+ '}';
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
new file mode 100644
index 00000000000..ca8adc6244c
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import java.util.Date;
+
+/**
+ * Java mapping of Swift JSON file status.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+
+class SwiftObjectFileStatus {
+ private long bytes;
+ private String content_type;
+ private String hash;
+ private Date last_modified;
+ private String name;
+ private String subdir;
+
+ SwiftObjectFileStatus() {
+ }
+
+ SwiftObjectFileStatus(long bytes, String content_type, String hash,
+ Date last_modified, String name) {
+ this.bytes = bytes;
+ this.content_type = content_type;
+ this.hash = hash;
+ this.last_modified = last_modified;
+ this.name = name;
+ }
+
+ public long getBytes() {
+ return bytes;
+ }
+
+ public void setBytes(long bytes) {
+ this.bytes = bytes;
+ }
+
+ public String getContent_type() {
+ return content_type;
+ }
+
+ public void setContent_type(String content_type) {
+ this.content_type = content_type;
+ }
+
+ public String getHash() {
+ return hash;
+ }
+
+ public void setHash(String hash) {
+ this.hash = hash;
+ }
+
+ public Date getLast_modified() {
+ return last_modified;
+ }
+
+ public void setLast_modified(Date last_modified) {
+ this.last_modified = last_modified;
+ }
+
+ public String getName() {
+ return pathToRootPath(name);
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSubdir() {
+ return pathToRootPath(subdir);
+ }
+
+ public void setSubdir(String subdir) {
+ this.subdir = subdir;
+ }
+
+ /**
+ * If path doesn't starts with '/'
+ * method will concat '/'
+ *
+ * @param path specified path
+ * @return root path string
+ */
+ private String pathToRootPath(String path) {
+ if (path == null) {
+ return null;
+ }
+
+ if (path.startsWith("/")) {
+ return path;
+ }
+
+ return "/".concat(path);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java
new file mode 100644
index 00000000000..3071f946824
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+public class Duration {
+
+ private final long started;
+ private long finished;
+
+ public Duration() {
+ started = time();
+ finished = started;
+ }
+
+ private long time() {
+ return System.currentTimeMillis();
+ }
+
+ public void finished() {
+ finished = time();
+ }
+
+ public String getDurationString() {
+ return humanTime(value());
+ }
+
+ public static String humanTime(long time) {
+ long seconds = (time / 1000);
+ long minutes = (seconds / 60);
+ return String.format("%d:%02d:%03d", minutes, seconds % 60, time % 1000);
+ }
+
+ @Override
+ public String toString() {
+ return getDurationString();
+ }
+
+ public long value() {
+ return finished -started;
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java
new file mode 100644
index 00000000000..b6d54f7139b
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+/**
+ * Build ongoing statistics from duration data
+ */
+public class DurationStats {
+
+ final String operation;
+ int n;
+ long sum;
+ long min;
+ long max;
+ double mean, m2;
+
+ /**
+ * Construct statistics for a given operation.
+ * @param operation operation
+ */
+ public DurationStats(String operation) {
+ this.operation = operation;
+ reset();
+ }
+
+ /**
+ * construct from anothr stats entry;
+ * all value are copied.
+ * @param that the source statistics
+ */
+ public DurationStats(DurationStats that) {
+ operation = that.operation;
+ n = that.n;
+ sum = that.sum;
+ min = that.min;
+ max = that.max;
+ mean = that.mean;
+ m2 = that.m2;
+ }
+
+ /**
+ * Add a duration
+ * @param duration the new duration
+ */
+ public void add(Duration duration) {
+ add(duration.value());
+ }
+
+ /**
+ * Add a number
+ * @param x the number
+ */
+ public void add(long x) {
+ n++;
+ sum += x;
+ double delta = x - mean;
+ mean += delta / n;
+ m2 += delta * (x - mean);
+ if (x < min) {
+ min = x;
+ }
+ if (x > max) {
+ max = x;
+ }
+ }
+
+ /**
+ * Reset the data
+ */
+ public void reset() {
+ n = 0;
+ sum = 0;
+ sum = 0;
+ min = 10000000;
+ max = 0;
+ mean = 0;
+ m2 = 0;
+ }
+
+ /**
+ * Get the number of entries sampled
+ * @return the number of durations added
+ */
+ public int getCount() {
+ return n;
+ }
+
+ /**
+ * Get the sum of all durations
+ * @return all the durations
+ */
+ public long getSum() {
+ return sum;
+ }
+
+ /**
+ * Get the arithmetic mean of the aggregate statistics
+ * @return the arithmetic mean
+ */
+ public double getArithmeticMean() {
+ return mean;
+ }
+
+ /**
+ * Variance, sigma^2
+ * @return variance, or, if no samples are there, 0.
+ */
+ public double getVariance() {
+ return n > 0 ? (m2 / (n - 1)) : 0;
+ }
+
+ /**
+ * Get the std deviation, sigma
+ * @return the stddev, 0 may mean there are no samples.
+ */
+ public double getDeviation() {
+ double variance = getVariance();
+ return (variance > 0) ? Math.sqrt(variance) : 0;
+ }
+
+ /**
+ * Covert to a useful string
+ * @return a human readable summary
+ */
+ @Override
+ public String toString() {
+ return String.format(
+ "%s count=%d total=%.3fs mean=%.3fs stddev=%.3fs min=%.3fs max=%.3fs",
+ operation,
+ n,
+ sum / 1000.0,
+ mean / 1000.0,
+ getDeviation() / 1000000.0,
+ min / 1000.0,
+ max / 1000.0);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java
new file mode 100644
index 00000000000..58f8f0b641d
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Build a duration stats table to which you can add statistics.
+ * Designed to be multithreaded
+ */
+public class DurationStatsTable {
+
+ private Map statsTable
+ = new HashMap(6);
+
+ /**
+ * Add an operation
+ * @param operation operation name
+ * @param duration duration
+ */
+ public void add(String operation, Duration duration, boolean success) {
+ DurationStats durationStats;
+ String key = operation;
+ if (!success) {
+ key += "-FAIL";
+ }
+ synchronized (this) {
+ durationStats = statsTable.get(key);
+ if (durationStats == null) {
+ durationStats = new DurationStats(key);
+ statsTable.put(key, durationStats);
+ }
+ }
+ synchronized (durationStats) {
+ durationStats.add(duration);
+ }
+ }
+
+ /**
+ * Get the current duration statistics
+ * @return a snapshot of the statistics
+ */
+ public synchronized List getDurationStatistics() {
+ List results = new ArrayList(statsTable.size());
+ for (DurationStats stat: statsTable.values()) {
+ results.add(new DurationStats(stat));
+ }
+ return results;
+ }
+
+ /**
+ * reset the values of the statistics. This doesn't delete them, merely zeroes them.
+ */
+ public synchronized void reset() {
+ for (DurationStats stat : statsTable.values()) {
+ stat.reset();
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java
new file mode 100644
index 00000000000..fd923ce28fe
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.type.CollectionType;
+import org.codehaus.jackson.type.TypeReference;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+
+
+public class JSONUtil {
+ private static ObjectMapper jsonMapper = new ObjectMapper();
+
+ /**
+ * Private constructor.
+ */
+ private JSONUtil() {
+ }
+
+ /**
+ * Converting object to JSON string. If errors appears throw
+ * MeshinException runtime exception.
+ *
+ * @param object The object to convert.
+ * @return The JSON string representation.
+ * @throws IOException IO issues
+ * @throws SwiftJsonMarshallingException failure to generate JSON
+ */
+ public static String toJSON(Object object) throws
+ IOException {
+ Writer json = new StringWriter();
+ try {
+ jsonMapper.writeValue(json, object);
+ return json.toString();
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString(), e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString(), e);
+ }
+ }
+
+ /**
+ * Convert string representation to object. If errors appears throw
+ * Exception runtime exception.
+ *
+ * @param value The JSON string.
+ * @param klazz The class to convert.
+ * @return The Object of the given class.
+ */
+ public static T toObject(String value, Class klazz) throws
+ IOException {
+ try {
+ return jsonMapper.readValue(value, klazz);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ }
+ }
+
+ /**
+ * @param value json string
+ * @param typeReference class type reference
+ * @param type
+ * @return deserialized T object
+ */
+ public static T toObject(String value,
+ final TypeReference typeReference)
+ throws IOException {
+ try {
+ return jsonMapper.readValue(value, typeReference);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException("Error generating response", e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException("Error generating response", e);
+ }
+ }
+
+ /**
+ * @param value json string
+ * @param collectionType class describing how to deserialize collection of objects
+ * @param type
+ * @return deserialized T object
+ */
+ public static T toObject(String value,
+ final CollectionType collectionType)
+ throws IOException {
+ try {
+ return jsonMapper.readValue(value, collectionType);
+ } catch (JsonGenerationException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ } catch (JsonMappingException e) {
+ throw new SwiftJsonMarshallingException(e.toString()
+ + " source: " + value,
+ e);
+ }
+ }
+
+ public static ObjectMapper getJsonMapper() {
+ return jsonMapper;
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java
new file mode 100644
index 00000000000..264f6d9cc18
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+
+import java.net.URI;
+import java.util.regex.Pattern;
+
+/**
+ * Swift hierarchy mapping of (container, path)
+ */
+public final class SwiftObjectPath {
+ private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/");
+
+ /**
+ * Swift container
+ */
+ private final String container;
+
+ /**
+ * swift object
+ */
+ private final String object;
+
+ private final String uriPath;
+
+ /**
+ * Build an instance from a (host, object) pair
+ *
+ * @param container container name
+ * @param object object ref underneath the container
+ */
+ public SwiftObjectPath(String container, String object) {
+
+ this.container = container;
+ this.object = object;
+ uriPath = buildUriPath();
+ }
+
+ public String getContainer() {
+ return container;
+ }
+
+ public String getObject() {
+ return object;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof SwiftObjectPath)) return false;
+ final SwiftObjectPath that = (SwiftObjectPath) o;
+ return this.toUriPath().equals(that.toUriPath());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = container.hashCode();
+ result = 31 * result + object.hashCode();
+ return result;
+ }
+
+ private String buildUriPath() {
+ return SwiftUtils.joinPaths(container, object);
+ }
+
+ public String toUriPath() {
+ return uriPath;
+ }
+
+ @Override
+ public String toString() {
+ return toUriPath();
+ }
+
+ /**
+ * Test for the object matching a path, ignoring the container
+ * value.
+ *
+ * @param path path string
+ * @return true iff the object's name matches the path
+ */
+ public boolean objectMatches(String path) {
+ return object.equals(path);
+ }
+
+
+ /**
+ * Query to see if the possibleChild object is a child path of this.
+ * object.
+ *
+ * The test is done by probing for the path of the this object being
+ * at the start of the second -with a trailing slash, and both
+ * containers being equal
+ *
+ * @param possibleChild possible child dir
+ * @return true iff the possibleChild is under this object
+ */
+ public boolean isEqualToOrParentOf(SwiftObjectPath possibleChild) {
+ String origPath = toUriPath();
+ String path = origPath;
+ if (!path.endsWith("/")) {
+ path = path + "/";
+ }
+ String childPath = possibleChild.toUriPath();
+ return childPath.equals(origPath) || childPath.startsWith(path);
+ }
+
+ /**
+ * Create a path tuple of (container, path), where the container is
+ * chosen from the host of the URI.
+ *
+ * @param uri uri to start from
+ * @param path path underneath
+ * @return a new instance.
+ * @throws SwiftConfigurationException if the URI host doesn't parse into
+ * container.service
+ */
+ public static SwiftObjectPath fromPath(URI uri,
+ Path path)
+ throws SwiftConfigurationException {
+ return fromPath(uri, path, false);
+ }
+
+ /**
+ * Create a path tuple of (container, path), where the container is
+ * chosen from the host of the URI.
+ * A trailing slash can be added to the path. This is the point where
+ * these /-es need to be appended, because when you construct a {@link Path}
+ * instance, {@link Path#normalizePath(String, String)} is called
+ * -which strips off any trailing slash.
+ *
+ * @param uri uri to start from
+ * @param path path underneath
+ * @param addTrailingSlash should a trailing slash be added if there isn't one.
+ * @return a new instance.
+ * @throws SwiftConfigurationException if the URI host doesn't parse into
+ * container.service
+ */
+ public static SwiftObjectPath fromPath(URI uri,
+ Path path,
+ boolean addTrailingSlash)
+ throws SwiftConfigurationException {
+
+ String url =
+ path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), "");
+ //add a trailing slash if needed
+ if (addTrailingSlash && !url.endsWith("/")) {
+ url += "/";
+ }
+
+ String container = uri.getHost();
+ if (container == null) {
+ //no container, not good: replace with ""
+ container = "";
+ } else if (container.contains(".")) {
+ //its a container.service URI. Strip the container
+ container = RestClientBindings.extractContainerName(container);
+ }
+ return new SwiftObjectPath(container, url);
+ }
+
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
new file mode 100644
index 00000000000..7e850e713de
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+/**
+ * Utilities used across test cases
+ */
+public class SwiftTestUtils extends org.junit.Assert {
+
+ private static final Log LOG =
+ LogFactory.getLog(SwiftTestUtils.class);
+
+ public static final String TEST_FS_SWIFT = "test.fs.swift.name";
+ public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ /**
+ * Get the test URI
+ * @param conf configuration
+ * @throws SwiftConfigurationException missing parameter or bad URI
+ */
+ public static URI getServiceURI(Configuration conf) throws
+ SwiftConfigurationException {
+ String instance = conf.get(TEST_FS_SWIFT);
+ if (instance == null) {
+ throw new SwiftConfigurationException(
+ "Missing configuration entry " + TEST_FS_SWIFT);
+ }
+ try {
+ return new URI(instance);
+ } catch (URISyntaxException e) {
+ throw new SwiftConfigurationException("Bad URI: " + instance);
+ }
+ }
+
+ public static boolean hasServiceURI(Configuration conf) {
+ String instance = conf.get(TEST_FS_SWIFT);
+ return instance != null;
+ }
+
+ /**
+ * Assert that a property in the property set matches the expected value
+ * @param props property set
+ * @param key property name
+ * @param expected expected value. If null, the property must not be in the set
+ */
+ public static void assertPropertyEquals(Properties props,
+ String key,
+ String expected) {
+ String val = props.getProperty(key);
+ if (expected == null) {
+ assertNull("Non null property " + key + " = " + val, val);
+ } else {
+ assertEquals("property " + key + " = " + val,
+ expected,
+ val);
+ }
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a finally{} block.
+ * @throws IOException IO problems
+ */
+ public static void writeAndRead(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ fs.mkdirs(path.getParent());
+
+ writeDataset(fs, path, src, len, blocksize, overwrite);
+
+ byte[] dest = readDataset(fs, path, len);
+
+ compareByteArrays(src, dest, len);
+
+ if (delete) {
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertPathDoesNotExist(fs, "Cleanup failed", path);
+ }
+ }
+
+ /**
+ * Write a file.
+ * Optional flags control
+ * whether file overwrite operations should be enabled
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @throws IOException IO problems
+ */
+ public static void writeDataset(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite) throws IOException {
+ assertTrue(
+ "Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ FSDataOutputStream out = fs.create(path,
+ overwrite,
+ fs.getConf()
+ .getInt(IO_FILE_BUFFER_SIZE,
+ 4096),
+ (short) 1,
+ blocksize);
+ out.write(src, 0, len);
+ out.close();
+ assertFileHasLength(fs, path, len);
+ }
+
+ /**
+ * Read the file and convert to a byte dataaset
+ * @param fs filesystem
+ * @param path path to read from
+ * @param len length of data to read
+ * @return the bytes
+ * @throws IOException IO problems
+ */
+ public static byte[] readDataset(FileSystem fs, Path path, int len)
+ throws IOException {
+ FSDataInputStream in = fs.open(path);
+ byte[] dest = new byte[len];
+ try {
+ in.readFully(0, dest);
+ } finally {
+ in.close();
+ }
+ return dest;
+ }
+
+ /**
+ * Assert that tthe array src[0..len] and dest[] are equal
+ * @param src source data
+ * @param dest actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] src,
+ byte[] dest,
+ int len) {
+ assertEquals("Number of bytes read != number written",
+ len, dest.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (src[i] != dest[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = dest[i];
+ byte expected = src[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ public static void cleanupInTeardown(FileSystem fileSystem,
+ String cleanupPath) {
+ cleanup("TEARDOWN", fileSystem, cleanupPath);
+ }
+
+ public static void cleanup(String action,
+ FileSystem fileSystem,
+ String cleanupPath) {
+ noteAction(action);
+ try {
+ if (fileSystem != null) {
+ fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
+ true);
+ }
+ } catch (Exception e) {
+ LOG.error("Error deleting in "+ action + " - " + cleanupPath + ": " + e, e);
+ }
+ }
+
+ public static void noteAction(String action) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("============== "+ action +" =============");
+ }
+ }
+
+ /**
+ * downgrade a failure to a message and a warning, then an
+ * exception for the Junit test runner to mark as failed
+ * @param message text message
+ * @param failure what failed
+ * @throws AssumptionViolatedException always
+ */
+ public static void downgrade(String message, Throwable failure) {
+ LOG.warn("Downgrading test " + message, failure);
+ AssumptionViolatedException ave =
+ new AssumptionViolatedException(failure, null);
+ throw ave;
+ }
+
+ /**
+ * report an overridden test as unsupported
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void unsupported(String message) {
+ throw new AssumptionViolatedException(message);
+ }
+
+ /**
+ * report a test has been skipped for some reason
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void skip(String message) {
+ throw new AssumptionViolatedException(message);
+ }
+
+
+ /**
+ * Make an assertion about the length of a file
+ * @param fs filesystem
+ * @param path path of the file
+ * @param expected expected length
+ * @throws IOException on File IO problems
+ */
+ public static void assertFileHasLength(FileSystem fs, Path path,
+ int expected) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(
+ "Wrong file length of file " + path + " status: " + status,
+ expected,
+ status.getLen());
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a dir -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Write the text to a file, returning the converted byte array
+ * for use in validating the round trip
+ * @param fs filesystem
+ * @param path path of file
+ * @param text text to write
+ * @param overwrite should the operation overwrite any existing file?
+ * @return the read bytes
+ * @throws IOException on IO problems
+ */
+ public static byte[] writeTextFile(FileSystem fs,
+ Path path,
+ String text,
+ boolean overwrite) throws IOException {
+ FSDataOutputStream stream = fs.create(path, overwrite);
+ byte[] bytes = new byte[0];
+ if (text != null) {
+ bytes = toAsciiByteArray(text);
+ stream.write(bytes);
+ }
+ stream.close();
+ return bytes;
+ }
+
+ /**
+ * Touch a file: fails if it is already there
+ * @param fs filesystem
+ * @param path path
+ * @throws IOException IO problems
+ */
+ public static void touch(FileSystem fs,
+ Path path) throws IOException {
+ fs.delete(path, true);
+ writeTextFile(fs, path, null, false);
+ }
+
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive) throws IOException {
+ assertPathExists(fs, "about to be deleted file", file);
+ boolean deleted = fs.delete(file, recursive);
+ String dir = ls(fs, file.getParent());
+ assertTrue("Delete failed on " + file + ": " + dir, deleted);
+ assertPathDoesNotExist(fs, "Deleted file", file);
+ }
+
+ /**
+ * Read in "length" bytes, convert to an ascii string
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read.
+ * @return the bytes read and converted to a string
+ * @throws IOException
+ */
+ public static String readBytesToString(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ FSDataInputStream in = fs.open(path);
+ try {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return toChar(buf);
+ } finally {
+ in.close();
+ }
+ }
+
+ public static String getDefaultWorkingDirectory() {
+ return "/user/" + System.getProperty("user.name");
+ }
+
+ public static String ls(FileSystem fileSystem, Path path) throws IOException {
+ return SwiftUtils.ls(fileSystem, path);
+ }
+
+ public static String dumpStats(String pathname, FileStatus[] stats) {
+ return pathname + SwiftUtils.fileStatsToString(stats,"\n");
+ }
+
+ /**
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param fileSystem filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+ IOException {
+ assertPathExists(fileSystem, "Expected file", filename);
+ FileStatus status = fileSystem.getFileStatus(filename);
+ String fileInfo = filename + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDirectory());
+/* disabled for Hadoop v1 compatibility
+ assertFalse("File claims to be a symlink " + fileInfo,
+ status.isSymlink());
+*/
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileSystem fileSystem, String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ fail(message + ": not found " + path + " in " + path.getParent());
+ ls(fileSystem, path.getParent());
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
new file mode 100644
index 00000000000..e86a307f9d2
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Various utility classes for SwiftFS support
+ */
+public final class SwiftUtils {
+
+ public static final String READ = "read(buffer, offset, length)";
+
+ /**
+ * Join two (non null) paths, inserting a forward slash between them
+ * if needed
+ *
+ * @param path1 first path
+ * @param path2 second path
+ * @return the combined path
+ */
+ public static String joinPaths(String path1, String path2) {
+ StringBuilder result =
+ new StringBuilder(path1.length() + path2.length() + 1);
+ result.append(path1);
+ boolean insertSlash = true;
+ if (path1.endsWith("/")) {
+ insertSlash = false;
+ } else if (path2.startsWith("/")) {
+ insertSlash = false;
+ }
+ if (insertSlash) {
+ result.append("/");
+ }
+ result.append(path2);
+ return result.toString();
+ }
+
+ /**
+ * This test contains the is-directory logic for Swift, so if
+ * changed there is only one place for it.
+ *
+ * @param fileStatus status to examine
+ * @return true if we consider this status to be representative of a
+ * directory.
+ */
+ public static boolean isDirectory(FileStatus fileStatus) {
+ return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
+ }
+
+ /**
+ * Test for the entry being a file that is treated as if it is a
+ * directory
+ *
+ * @param fileStatus status
+ * @return true if it meets the rules for being a directory
+ */
+ public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
+ return fileStatus.getLen() == 0;
+ }
+
+ /**
+ * Predicate: Is a swift object referring to the root direcory?
+ * @param swiftObject object to probe
+ * @return true iff the object refers to the root
+ */
+ public static boolean isRootDir(SwiftObjectPath swiftObject) {
+ return swiftObject.objectMatches("") || swiftObject.objectMatches("/");
+ }
+
+ /**
+ * Sprintf() to the log iff the log is at debug level. If the log
+ * is not at debug level, the printf operation is skipped, so
+ * no time is spent generating the string.
+ * @param log log to use
+ * @param text text message
+ * @param args args arguments to the print statement
+ */
+ public static void debug(Log log, String text, Object... args) {
+ if (log.isDebugEnabled()) {
+ log.debug(String.format(text, args));
+ }
+ }
+
+ /**
+ * Log an exception (in text and trace) iff the log is at debug
+ * @param log Log to use
+ * @param text text message
+ * @param ex exception
+ */
+ public static void debugEx(Log log, String text, Exception ex) {
+ if (log.isDebugEnabled()) {
+ log.debug(text + ex, ex);
+ }
+ }
+
+ /**
+ * Sprintf() to the log iff the log is at trace level. If the log
+ * is not at trace level, the printf operation is skipped, so
+ * no time is spent generating the string.
+ * @param log log to use
+ * @param text text message
+ * @param args args arguments to the print statement
+ */
+ public static void trace(Log log, String text, Object... args) {
+ if (log.isTraceEnabled()) {
+ log.trace(String.format(text, args));
+ }
+ }
+
+ /**
+ * Given a partition number, calculate the partition value.
+ * This is used in the SwiftNativeOutputStream, and is placed
+ * here for tests to be able to calculate the filename of
+ * a partition.
+ * @param partNumber part number
+ * @return a string to use as the filename
+ */
+ public static String partitionFilenameFromNumber(int partNumber) {
+ return String.format("%06d", partNumber);
+ }
+
+ /**
+ * List a a path to string
+ * @param fileSystem filesystem
+ * @param path directory
+ * @return a listing of the filestatuses of elements in the directory, one
+ * to a line, precedeed by the full path of the directory
+ * @throws IOException connectivity problems
+ */
+ public static String ls(FileSystem fileSystem, Path path) throws
+ IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return pathtext + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * Verify that the basic args to a read operation are valid;
+ * throws an exception if not -with meaningful text includeing
+ * @param buffer destination buffer
+ * @param off offset
+ * @param len number of bytes to read
+ * @throws NullPointerException null buffer
+ * @throws IndexOutOfBoundsException on any invalid range.
+ */
+ public static void validateReadArgs(byte[] buffer, int off, int len) {
+ if (buffer == null) {
+ throw new NullPointerException("Null byte array in"+ READ);
+ }
+ if (off < 0 ) {
+ throw new IndexOutOfBoundsException("Negative buffer offset "
+ + off
+ + " in " + READ);
+ }
+ if (len < 0 ) {
+ throw new IndexOutOfBoundsException("Negative read length "
+ + len
+ + " in " + READ);
+ }
+ if (off > buffer.length) {
+ throw new IndexOutOfBoundsException("Buffer offset of "
+ + off
+ + "beyond buffer size of "
+ + buffer.length
+ + " in " + READ);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm b/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
new file mode 100644
index 00000000000..29fbd33f1f4
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
@@ -0,0 +1,686 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ Hadoop OpenStack Support: Swift Object Store
+ ---
+ ---
+ ${maven.build.timestamp}
+
+%{toc|section=1|fromDepth=0}
+
+Hadoop OpenStack Support: Swift Object Store
+
+* {Introduction}
+
+ {{{http://www.openstack.org/}OpenStack}} is an open source cloud infrastructure
+ which can be accessed
+ from multiple public IaaS providers, and deployed privately. It offers
+ infrastructure services such as VM hosting (Nova), authentication (Keystone)
+ and storage of binary objects (Swift).
+
+ This module enables Apache Hadoop applications -including MapReduce jobs,
+ read and write data to and from instances of the
+ {{{http://www.openstack.org/software/openstack-storage/}OpenStack Swift object store}}.
+
+* Features
+
+ * Read and write of data stored in a Swift object store
+
+ * Support of a pseudo-hierachical file system (directories, subdirectories and
+ files)
+
+ * Standard filesystem operations: <<>>, <<>>, <<>>,
+ <<>>, <<>>, <<>>.
+
+ * Can act as a source of data in a MapReduce job, or a sink.
+
+ * Support for multiple OpenStack services, and multiple containers from a
+ single service.
+
+ * Supports in-cluster and remote access to Swift data.
+
+ * Supports OpenStack Keystone authentication with password or token.
+
+ * Released under the Apache Software License
+
+ * Tested against the Hadoop 3.x and 1.x branches, against multiple public
+ OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
+
+ * Tested against private OpenStack clusters, including scalability tests of
+ large file uploads.
+
+* Using the Hadoop Swift Filesystem Client
+
+** Concepts: services and containers
+
+ OpenStack swift is an ; also known as a . It stores
+ arbitrary binary objects by name in a .
+
+ The Hadoop Swift filesystem library adds another concept, the , which
+ defines which Swift blobstore hosts a container -and how to connect to it.
+
+** Containers and Objects
+
+ * Containers are created by users with accounts on the Swift filestore, and hold
+ .
+
+ * Objects can be zero bytes long, or they can contain data.
+
+ * Objects in the container can be up to 5GB; there is a special support for
+ larger files than this, which merges multiple objects in to one.
+
+ * Each object is referenced by it's ; there is no notion of directories.
+
+ * You can use any characters in an object name that can be 'URL-encoded'; the
+ maximum length of a name is 1034 characters -after URL encoding.
+
+ * Names can have <<>> characters in them, which are used to create the illusion of
+ a directory structure. For example <<>>. Even though this looks
+ like a directory, . There is no requirement to have
+ any entries in the container called <<>> or <<>>
+
+ * That said. if the container has zero-byte objects that look like directory
+ names above other objects, they can pretend to be directories. Continuing the
+ example, a 0-byte object called <<>> would tell clients that it is a
+ directory while <<>> or <<>> were present. This creates an
+ illusion of containers holding a filesystem.
+
+ Client applications talk to Swift over HTTP or HTTPS, reading, writing and
+ deleting objects using standard HTTP operations (GET, PUT and DELETE,
+ respectively). There is also a COPY operation, that creates a new object in the
+ container, with a new name, containing the old data. There is no rename
+ operation itself, objects need to be copied -then the original entry deleted.
+
+** Eventual Consistency
+
+ The Swift Filesystem is *eventually consistent*: an operation on an object may
+ not be immediately visible to that client, or other clients. This is a
+ consequence of the goal of the filesystem: to span a set of machines, across
+ multiple datacenters, in such a way that the data can still be available when
+ many of them fail. (In contrast, the Hadoop HDFS filesystem is *immediately
+ consistent*, but it does not span datacenters.)
+
+ Eventual consistency can cause surprises for client applications that expect
+ immediate consistency: after an object is deleted or overwritten, the object
+ may still be visible -or the old data still retrievable. The Swift Filesystem
+ client for Apache Hadoop attempts to handle this, in conjunction with the
+ MapReduce engine, but there may be still be occasions when eventual consistency
+ causes surprises.
+
+** Non-atomic "directory" operations.
+
+ Hadoop expects some
+ operations to be atomic, especially <<>>, which is something
+ the MapReduce layer relies on to commit the output of a job, renaming data
+ from a temp directory to the final path. Because a rename
+ is implemented as a copy of every blob under the directory's path, followed
+ by a delete of the originals, the intermediate state of the operation
+ will be visible to other clients. If two Reducer tasks to rename their temp
+ directory to the final path, both operations may succeed, with the result that
+ output directory contains mixed data. This can happen if MapReduce jobs
+ are being run with enabled and Swift used as the direct output
+ of the MR job (it can also happen against Amazon S3).
+
+ Other consequences of the non-atomic operations are:
+
+ 1. If a program is looking for the presence of the directory before acting
+ on the data -it may start prematurely. This can be avoided by using
+ other mechanisms to co-ordinate the programs, such as the presence of a file
+ that is written any bulk directory operations.
+
+ 2. A <<>> or <<>> operation may include files added under
+ the source directory tree during the operation, may unintentionally delete
+ it, or delete the 0-byte swift entries that mimic directories and act
+ as parents for the files. Try to avoid doing this.
+
+ The best ways to avoid all these problems is not using Swift as
+ the filesystem between MapReduce jobs or other Hadoop workflows. It
+ can act as a source of data, and a final destination, but it doesn't meet
+ all of Hadoop's expectations of what a filesystem is -it's a .
+
+* Working with Swift Object Stores in Hadoop
+
+ Once installed, the Swift FileSystem client can be used by any Hadoop application
+ to read from or write to data stored in a Swift container.
+
+ Data stored in Swift can be used as the direct input to a MapReduce job
+ -simply use the <<>> URL (see below) to declare the source of the data.
+
+ This Swift Filesystem client is designed to work with multiple
+ Swift object stores, both public and private. This allows the client to work
+ with different clusters, reading and writing data to and from either of them.
+
+ It can also work with the same object stores using multiple login details.
+
+ These features are achieved by one basic concept: using a service name in
+ the URI referring to a swift filesystem, and looking up all the connection and
+ login details for that specific service. Different service names can be defined
+ in the Hadoop XML configuration file, so defining different clusters, or
+ providing different login details for the same object store(s).
+
+
+** Swift Filesystem URIs
+
+ Hadoop uses URIs to refer to files within a filesystem. Some common examples
+ are:
+
++--
+ local://etc/hosts
+ hdfs://cluster1/users/example/data/set1
+ hdfs://cluster2.example.org:8020/users/example/data/set1
++--
+
+ The Swift Filesystem Client adds a new URL type <<>>. In a Swift Filesystem
+ URL, the hostname part of a URL identifies the container and the service to
+ work with; the path the name of the object. Here are some examples
+
++--
+ swift://container.rackspace/my-object.csv
+ swift://data.hpcloud/data/set1
+ swift://dmitry.privatecloud/out/results
++--
+
+ In the last two examples, the paths look like directories: it is not, they are
+ simply the objects named <<>> and <<>> respectively.
+
+** Installing
+
+ The <<>> JAR must be on the classpath of the Hadoop program trying to
+ talk to the Swift service. If installed in the classpath of the Hadoop
+ MapReduce service, then all programs started by the MR engine will pick up the
+ JAR automatically. This is the easiest way to give all Hadoop jobs access to
+ Swift.
+
+ Alternatively, the JAR can be included as one of the JAR files that an
+ application uses. This lets the Hadoop jobs work with a Swift object store even
+ if the Hadoop cluster is not pre-configured for this.
+
+ The library also depends upon the Apache HttpComponents library, which
+ must also be on the classpath.
+
+** Configuring
+
+ To talk to a swift service, the user must must provide:
+
+ [[1]] The URL defining the container and the service.
+
+ [[1]] In the cluster/job configuration, the login details of that service.
+
+ Multiple service definitions can co-exist in the same configuration file: just
+ use different names for them.
+
+*** Example: Rackspace US, in-cluster access using API key
+
+ This service definition is for use in a Hadoop cluster deployed within Rackspace's
+ US infrastructure.
+
++--
+
+ fs.swift.service.rackspace.auth.url
+ https://auth.api.rackspacecloud.com/v2.0/tokens
+ Rackspace US (multiregion)
+
+
+
+ fs.swift.service.rackspace.username
+ user4
+
+
+
+ fs.swift.service.rackspace.region
+ DFW
+
+
+
+ fs.swift.service.rackspace.apikey
+ fe806aa86dfffe2f6ed8
+
++--
+
+ Here the API key visible in the account settings API keys page is used to log
+ in. No property for public/private access -the default is to use the private
+ endpoint for Swift operations.
+
+ This configuration also selects one of the regions, DFW, for its data.
+
+ A reference to this service would use the <<>> service name:
+
+---
+ swift://hadoop-container.rackspace/
+---
+
+*** Example: Rackspace UK: remote access with password authentication
+
+ This connects to Rackspace's UK ("LON") datacenter.
+
++--
+
+ fs.swift.service.rackspaceuk.auth.url
+ https://lon.identity.api.rackspacecloud.com/v2.0/tokens
+ Rackspace UK
+
+
+
+ fs.swift.service.rackspaceuk.username
+ user4
+
+
+
+ fs.swift.service.rackspaceuk.password
+ insert-password-here/value>
+
+
+
+ fs.swift.service.rackspace.public
+ true
+
++--
+
+ This is a public access point connection, using a password over an API key.
+
+ A reference to this service would use the <<>> service name:
+
++--
+ swift://hadoop-container.rackspaceuk/
++--
+
+ Because the public endpoint is used, if this service definition is used within
+ the London datacenter, all accesses will be billed at the public
+ upload/download rates, .
+
+*** Example: HP cloud service definition
+
+ Here is an example that connects to the HP Cloud object store.
+
++--
+
+ fs.swift.service.hpcloud.auth.url
+ https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
+
+ HP Cloud
+
+
+
+ fs.swift.service.hpcloud.tenant
+ FE806AA86
+
+
+
+ fs.swift.service.hpcloud.username
+ FE806AA86DFFFE2F6ED8
+
+
+
+ fs.swift.service.hpcloud.password
+ secret-password-goes-here
+
+
+
+ fs.swift.service.hpcloud.public
+ true
+
++--
+
+ A reference to this service would use the <<>> service name:
+
++--
+ swift://hadoop-container.hpcloud/
++--
+
+** General Swift Filesystem configuration options
+
+ Some configuration options apply to the Swift client, independent of
+ the specific Swift filesystem chosen.
+
+*** Blocksize fs.swift.blocksize
+
+ Swift does not break up files into blocks, except in the special case of files
+ over 5GB in length. Accordingly, there isn't a notion of a "block size"
+ to define where the data is kept.
+
+ Hadoop's MapReduce layer depends on files declaring their block size,
+ so that it knows how to partition work. Too small a blocksize means that
+ many mappers work on small pieces of data; too large a block size means
+ that only a few mappers get started.
+
+ The block size value reported by Swift, therefore, controls the basic workload
+ partioning of the MapReduce engine -and can be an important parameter to
+ tune for performance of the cluster.
+
+ The property has a unit of kilobytes; the default value is <<<32*1024>>>: 32 MB
+
++--
+
+ fs.swift.blocksize
+ 32768
+
++--
+
+ This blocksize has no influence on how files are stored in Swift; it only controls
+ what the reported size of blocks are - a value used in Hadoop MapReduce to
+ divide work.
+
+ Note that the MapReduce engine's split logic can be tuned independently by setting
+ the <<>> and <<>> properties,
+ which can be done in specific job configurations.
+
++--
+
+ mapred.min.split.size
+ 524288
+
+
+
+ mapred.max.split.size
+ 1048576
+
++--
+
+ In an Apache Pig script, these properties would be set as:
+
+---
+ mapred.min.split.size 524288
+ mapred.max.split.size 1048576
+---
+
+*** Partition size fs.swift.partsize
+
+ The Swift filesystem client breaks very large files into partitioned files,
+ uploading each as it progresses, and writing any remaning data and an XML
+ manifest when a partitioned file is closed.
+
+ The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that
+ Swift can support.
+
+ It is possible to set a smaller partition size, in the <<>>
+ option. This takes a value in KB.
+
++--
+
+ fs.swift.partsize
+ 1024
+ upload every MB
+
++--
+
+ When should this value be changed from its default?
+
+ While there is no need to ever change it for basic operation of
+ the Swift filesystem client, it can be tuned
+
+ * If a Swift filesystem is location aware, then breaking a file up into
+ smaller partitions scatters the data round the cluster. For best performance,
+ the property <<>> should be set to a smaller value than the
+ partition size of files.
+
+ * When writing to an unpartitioned file, the entire write is done in the
+ <<>> operation. When a file is partitioned, the outstanding data to
+ be written whenever the outstanding amount of data is greater than the
+ partition size. This means that data will be written more incrementally
+
+*** Request size fs.swift.requestsize
+
+ The Swift filesystem client reads files in HTTP GET operations, asking for
+ a block of data at a time.
+
+ The default value is 64KB. A larger value may be more efficient over faster
+ networks, as it reduces the overhead of setting up the HTTP operation.
+
+ However, if the file is read with many random accesses, requests for
+ data will be made from different parts of the file -discarding some of the
+ previously requested data. The benefits of larger request sizes may be wasted.
+
+ The property <<>> sets the request size in KB.
+
++--
+
+ fs.swift.requestsize
+ 128
+
++--
+
+*** Connection timeout fs.swift.connect.timeout
+
+ This sets the timeout in milliseconds to connect to a Swift service.
+
++--
+
+ fs.swift.connect.timeout
+ 15000
+
++--
+
+ A shorter timeout means that connection failures are raised faster -but
+ may trigger more false alarms. A longer timeout is more resilient to network
+ problems -and may be needed when talking to remote filesystems.
+
+*** Connection timeout fs.swift.socket.timeout
+
+ This sets the timeout in milliseconds to wait for data from a connected socket.
+
++--
+
+ fs.swift.socket.timeout
+ 60000
+
++--
+
+ A shorter timeout means that connection failures are raised faster -but
+ may trigger more false alarms. A longer timeout is more resilient to network
+ problems -and may be needed when talking to remote filesystems.
+
+*** Connection Retry Count fs.swift.connect.retry.count
+
+ This sets the number of times to try to connect to a service whenever
+ an HTTP request is made.
+
++--
+
+ fs.swift.connect.retry.count
+ 3
+
++--
+
+ The more retries, the more resilient it is to transient outages -and the
+ less rapid it is at detecting and reporting server connectivity problems.
+
+*** Connection Throttle Delay fs.swift.connect.throttle.delay
+
+ This property adds a delay between bulk file copy and delete operations,
+ to prevent requests being throttled or blocked by the remote service
+
++--
+
+ fs.swift.connect.throttle.delay
+ 0
+
++--
+
+ It is measured in milliseconds; "0" means do not add any delay.
+
+ Throttling is enabled on the public endpoints of some Swift services.
+ If <<>> or <<>> operations fail with
+ <<>>
+ exceptions, try setting this property.
+
+*** HTTP Proxy
+
+ If the client can only access the Swift filesystem via a web proxy
+ server, the client configuration must specify the proxy via
+ the <<>> and <<>>
+ properties.
+
++--
+
+ fs.swift.proxy.host
+ web-proxy
+
+
+
+ fs.swift.proxy.port
+ 8088
+
++--
+
+ If the host is declared, the proxy port must be set to a valid integer value.
+
+
+** Troubleshooting
+
+*** ClassNotFoundException
+
+ The <<>> JAR -or any dependencies- may not be on your classpath.
+
+ If it is a remote MapReduce job that is failing, make sure that the JAR is
+ installed on the servers in the cluster -or that the job submission process
+ uploads the JAR file to the distributed cache.
+
+*** Failure to Authenticate
+
+ A <<>> is thrown when the client
+ cannot authenticate with the OpenStack keystone server. This could be
+ because the URL in the service definition is wrong, or because
+ the supplied credentials are invalid.
+
+ [[1]] Check the authentication URL through <<>> or your browser
+
+ [[1]] Use a Swift client such as CyberDuck to validate your credentials
+
+ [[1]] If you have included a tenant ID, try leaving it out. Similarly,
+ try adding it if you had not included it.
+
+ [[1]] Try switching from API key authentication to password-based authentication,
+ by setting the password.
+
+ [[1]] Change your credentials. As with Amazon AWS clients, some credentials
+ don't seem to like going over the network.
+
+*** Timeout connecting to the Swift Service
+
+ This happens if the client application is running outside an OpenStack cluster,
+ where it does not have access to the private hostname/IP address for filesystem
+ operations. Set the <<>> flag to true -but remember to set it to false
+ for use in-cluster.
+
+** Warnings
+
+ [[1]] Do not share your login details with anyone, which means do not log the
+ details, or check the XML configuration files into any revision control system
+ to which you do not have exclusive access.
+
+ [[1]] Similarly, do not use your real account details in any documentation *or any
+ bug reports submitted online*
+
+ [[1]] Prefer the apikey authentication over passwords as it is easier
+ to revoke a key -and some service providers allow you to set
+ an automatic expiry date on a key when issued.
+
+ [[1]] Do not use the public service endpoint from within a public OpenStack
+ cluster, as it will run up large bills.
+
+ [[1]] Remember: it's not a real filesystem or hierarchical directory structure.
+ Some operations (directory rename and delete) take time and are not atomic or
+ isolated from other operations taking place.
+
+ [[1]] Append is not supported.
+
+ [[1]] Unix-style permissions are not supported. All accounts with write access to
+ a repository have unlimited access; the same goes for those with read access.
+
+ [[1]] In the public clouds, do not make the containers public unless you are happy
+ with anyone reading your data, and are prepared to pay the costs of their
+ downloads.
+
+** Limits
+
+ * Maximum length of an object path: 1024 characters
+
+ * Maximum size of a binary object: no absolute limit. Files > 5GB are
+ partitioned into separate files in the native filesystem, and merged during
+ retrieval. the partitioned/large file support is the
+ most complex part of the Hadoop/Swift FS integration, and, along with
+ authentication, the most troublesome to support.
+
+** Testing the hadoop-openstack module
+
+ The <<>> can be remotely tested against any public
+ or private cloud infrastructure which supports the OpenStack Keystone
+ authentication mechanism. It can also be tested against private
+ OpenStack clusters. OpenStack Development teams are strongly encouraged to test
+ the Hadoop swift filesystem client against any version of Swift that they
+ are developing or deploying, to stress their cluster and to identify
+ bugs early.
+
+ The module comes with a large suite of JUnit tests -tests that are
+ only executed if the source tree includes credentials to test against a
+ specific cluster.
+
+ After checking out the Hadoop source tree, create the file:
+
++--
+ hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
++--
+
+ Into this file, insert the credentials needed to bond to the test filesystem,
+ as decribed above.
+
+ Next set the property <<>> to the URL of a
+ swift container to test against. The tests expect exclusive access
+ to this container -do not keep any other data on it, or expect it
+ to be preserved.
+
++--
+
+ test.fs.swift.name
+ swift://test.myswift/
+
++--
+
+ In the base hadoop directory, run:
+
++--
+ mvn clean install -DskipTests
++--
+
+ This builds a set of Hadoop JARs consistent with the <<>>
+ module that is about to be tested.
+
+ In the <<>> directory run
+
++--
+ mvn test -Dtest=TestSwiftRestClient
++--
+
+ This runs some simple tests which include authenticating
+ against the remote swift service. If these tests fail, so will all
+ the rest. If it does fail: check your authentication.
+
+ Once this test succeeds, you can run the full test suite
+
++--
+ mvn test
++--
+
+ Be advised that these tests can take an hour or more, especially against a
+ remote Swift service -or one that throttles bulk operations.
+
+ Once the <<>> file is in place, the <<>> runs from
+ the Hadoop source base directory will automatically run these OpenStack tests
+ While this ensures that no regressions have occurred, it can also add significant
+ time to test runs, and may run up bills, depending on who is providing\
+ the Swift storage service. We recommend having a separate source tree
+ set up purely for the Swift tests, and running it manually or by the CI tooling
+ at a lower frequency than normal test runs.
+
+ Finally: Apache Hadoop is an open source project. Contributions of code
+ -including more tests- are very welcome.
diff --git a/hadoop-tools/hadoop-openstack/src/site/site.xml b/hadoop-tools/hadoop-openstack/src/site/site.xml
new file mode 100644
index 00000000000..97784bd4ad0
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/site/site.xml
@@ -0,0 +1,46 @@
+
+
+
+
+
+
+ org.apache.maven.skins
+ maven-stylus-skin
+ 1.2
+
+
+
+
+
+
+
+
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java
new file mode 100644
index 00000000000..16c9da25777
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+
+/**
+ * A path filter that accepts everything
+ */
+public class AcceptAllFilter implements PathFilter {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
new file mode 100644
index 00000000000..852796c8a82
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;
+
+/**
+ * This is the base class for most of the Swift tests
+ */
+public class SwiftFileSystemBaseTest extends Assert implements
+ SwiftTestConstants {
+
+ protected static final Log LOG =
+ LogFactory.getLog(SwiftFileSystemBaseTest.class);
+ protected SwiftNativeFileSystem fs;
+ protected static SwiftNativeFileSystem lastFs;
+ protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
+ private Configuration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ noteAction("setup");
+ final URI uri = getFilesystemURI();
+ conf = createConfiguration();
+
+ fs = createSwiftFS();
+ try {
+ fs.initialize(uri, conf);
+ } catch (IOException e) {
+ //FS init failed, set it to null so that teardown doesn't
+ //attempt to use it
+ fs = null;
+ throw e;
+ }
+ //remember the last FS
+ lastFs = fs;
+ noteAction("setup complete");
+ }
+
+ /**
+ * Configuration generator. May be overridden to inject
+ * some custom options
+ * @return a configuration with which to create FS instances
+ */
+ protected Configuration createConfiguration() {
+ return new Configuration();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ cleanupInTeardown(fs, "/test");
+ }
+
+ @AfterClass
+ public static void classTearDown() throws Exception {
+ if (lastFs != null) {
+ List statistics = lastFs.getOperationStatistics();
+ for (DurationStats stat : statistics) {
+ LOG.info(stat.toString());
+ }
+ }
+ }
+
+ /**
+ * Get the configuration used to set up the FS
+ * @return the configuration
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Describe the test, combining some logging with details
+ * for people reading the code
+ *
+ * @param description test description
+ */
+ protected void describe(String description) {
+ noteAction(description);
+ }
+
+ protected URI getFilesystemURI() throws URISyntaxException, IOException {
+ return getServiceURI(createConfiguration());
+ }
+
+ protected SwiftNativeFileSystem createSwiftFS() throws IOException {
+ SwiftNativeFileSystem swiftNativeFileSystem =
+ new SwiftNativeFileSystem();
+ return swiftNativeFileSystem;
+ }
+
+ protected int getBlockSize() {
+ return 1024;
+ }
+
+ /**
+ * Is rename supported?
+ * @return true
+ */
+ protected boolean renameSupported() {
+ return true;
+ }
+
+ /**
+ * assume in a test that rename is supported;
+ * skip it if not
+ */
+ protected void assumeRenameSupported() {
+ Assume.assumeTrue(renameSupported());
+ }
+
+ /**
+ * Take an unqualified path, and qualify it w.r.t the
+ * current filesystem
+ * @param pathString source path
+ * @return a qualified path instance
+ */
+ protected Path path(String pathString) {
+ return new Path(pathString).makeQualified(fs);
+ }
+
+ /**
+ * Get the filesystem
+ * @return the current FS
+ */
+ public SwiftNativeFileSystem getFs() {
+ return fs;
+ }
+
+ /**
+ * Create a file using the standard {@link #data} bytes.
+ *
+ * @param path path to write
+ * @throws IOException on any problem
+ */
+ protected void createFile(Path path) throws IOException {
+ createFile(path, data);
+ }
+
+ /**
+ * Create a file with the given data.
+ *
+ * @param path path to write
+ * @param sourceData source dataset
+ * @throws IOException on any problem
+ */
+ protected void createFile(Path path, byte[] sourceData) throws IOException {
+ FSDataOutputStream out = fs.create(path);
+ out.write(sourceData, 0, sourceData.length);
+ out.close();
+ }
+
+ /**
+ * Create and then close a file
+ * @param path path to create
+ * @throws IOException on a failure
+ */
+ protected void createEmptyFile(Path path) throws IOException {
+ FSDataOutputStream out = fs.create(path);
+ out.close();
+ }
+
+ /**
+ * Get the inner store -useful for lower level operations
+ *
+ * @return the store
+ */
+ protected SwiftNativeFileSystemStore getStore() {
+ return fs.getStore();
+ }
+
+ /**
+ * Rename a path
+ * @param src source
+ * @param dst dest
+ * @param renameMustSucceed flag to say "this rename must exist"
+ * @param srcExists add assert that the source exists afterwards
+ * @param dstExists add assert the dest exists afterwards
+ * @throws IOException IO trouble
+ */
+ protected void rename(Path src, Path dst, boolean renameMustSucceed,
+ boolean srcExists, boolean dstExists) throws IOException {
+ if (renameMustSucceed) {
+ renameToSuccess(src, dst, srcExists, dstExists);
+ } else {
+ renameToFailure(src, dst);
+ }
+ }
+
+ /**
+ * Get a string describing the outcome of a rename, by listing the dest
+ * path and its parent along with some covering text
+ * @param src source patj
+ * @param dst dest path
+ * @return a string for logs and exceptions
+ * @throws IOException IO problems
+ */
+ private String getRenameOutcome(Path src, Path dst) throws IOException {
+ String lsDst = ls(dst);
+ Path parent = dst.getParent();
+ String lsParent = parent != null ? ls(parent) : "";
+ return " result of " + src + " => " + dst
+ + " - " + lsDst
+ + " \n" + lsParent;
+ }
+
+ /**
+ * Rename, expecting an exception to be thrown
+ *
+ * @param src source
+ * @param dst dest
+ * @throws IOException a failure other than an
+ * expected SwiftRenameException or FileNotFoundException
+ */
+ protected void renameToFailure(Path src, Path dst) throws IOException {
+ try {
+ getStore().rename(src, dst);
+ fail("Expected failure renaming " + src + " to " + dst
+ + "- but got success");
+ } catch (SwiftOperationFailedException e) {
+ LOG.debug("Rename failed (expected):" + e);
+ } catch (FileNotFoundException e) {
+ LOG.debug("Rename failed (expected):" + e);
+ }
+ }
+
+ /**
+ * Rename to success
+ *
+ * @param src source
+ * @param dst dest
+ * @param srcExists add assert that the source exists afterwards
+ * @param dstExists add assert the dest exists afterwards
+ * @throws SwiftOperationFailedException operation failure
+ * @throws IOException IO problems
+ */
+ protected void renameToSuccess(Path src, Path dst,
+ boolean srcExists, boolean dstExists)
+ throws SwiftOperationFailedException, IOException {
+ getStore().rename(src, dst);
+ String outcome = getRenameOutcome(src, dst);
+ assertEquals("Source " + src + "exists: " + outcome,
+ srcExists, fs.exists(src));
+ assertEquals("Destination " + dstExists + " exists" + outcome,
+ dstExists, fs.exists(dst));
+ }
+
+ /**
+ * List a path in the test FS
+ * @param path path to list
+ * @return the contents of the path/dir
+ * @throws IOException IO problems
+ */
+ protected String ls(Path path) throws IOException {
+ return SwiftTestUtils.ls(fs, path);
+ }
+
+ /**
+ * assert that a path exists
+ * @param message message to use in an assertion
+ * @param path path to probe
+ * @throws IOException IO problems
+ */
+ public void assertExists(String message, Path path) throws IOException {
+ assertPathExists(fs, message, path);
+ }
+
+ /**
+ * assert that a path does not
+ * @param message message to use in an assertion
+ * @param path path to probe
+ * @throws IOException IO problems
+ */
+ public void assertPathDoesNotExist(String message, Path path) throws
+ IOException {
+ SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ *
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ protected void assertIsFile(Path filename) throws IOException {
+ SwiftTestUtils.assertIsFile(fs, filename);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ *
+ * @throws IOException IO problems during file operations
+ */
+ protected void mkdirs(Path path) throws IOException {
+ assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
+ }
+
+ /**
+ * Assert that a delete succeeded
+ * @param path path to delete
+ * @param recursive recursive flag
+ * @throws IOException IO problems
+ */
+ protected void assertDeleted(Path path, boolean recursive) throws IOException {
+ SwiftTestUtils.assertDeleted(fs, path, recursive);
+ }
+
+ /**
+ * Assert that a value is not equal to the expected value
+ * @param message message if the two values are equal
+ * @param expected expected value
+ * @param actual actual value
+ */
+ protected void assertNotEqual(String message, int expected, int actual) {
+ assertTrue(message,
+ actual != expected);
+ }
+
+ /**
+ * Get the number of partitions written from the Swift Native FS APIs
+ * @param out output stream
+ * @return the number of partitioned files written by the stream
+ */
+ protected int getPartitionsWritten(FSDataOutputStream out) {
+ return SwiftNativeFileSystem.getPartitionsWritten(out);
+ }
+
+ /**
+ * Assert that the no. of partitions written matches expectations
+ * @param action operation (for use in the assertions)
+ * @param out output stream
+ * @param expected expected no. of partitions
+ */
+ protected void assertPartitionsWritten(String action, FSDataOutputStream out,
+ long expected) {
+ OutputStream nativeStream = out.getWrappedStream();
+ int written = getPartitionsWritten(out);
+ if(written !=expected) {
+ Assert.fail(action + ": " +
+ TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT
+ + " + expected: " + expected + " actual: " + written
+ + " -- " + nativeStream);
+ }
+ }
+
+ /**
+ * Assert that the result value == -1; which implies
+ * that a read was successful
+ * @param text text to include in a message (usually the operation)
+ * @param result read result to validate
+ */
+ protected void assertMinusOne(String text, int result) {
+ assertEquals(text + " wrong read result " + result, -1, result);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java
new file mode 100644
index 00000000000..6948cf92fa6
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+/**
+ * Hard coded constants for the test timeouts
+ */
+public interface SwiftTestConstants {
+ /**
+ * Timeout for swift tests: {@value}
+ */
+ int SWIFT_TEST_TIMEOUT = 5 * 60 * 1000;
+
+ /**
+ * Timeout for tests performing bulk operations: {@value}
+ */
+ int SWIFT_BULK_IO_TEST_TIMEOUT = 12 * 60 * 1000;
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java
new file mode 100644
index 00000000000..74299df9042
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSMainOperationsBaseTest;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import static org.apache.hadoop.fs.swift.SwiftTestConstants.SWIFT_TEST_TIMEOUT;
+import java.io.IOException;
+import java.net.URI;
+
+public class TestFSMainOperationsSwift extends FSMainOperationsBaseTest {
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ //small blocksize for faster remote tests
+ conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, 2);
+ URI serviceURI = SwiftTestUtils.getServiceURI(conf);
+ fSys = FileSystem.get(serviceURI, conf);
+ super.setUp();
+ }
+
+ private Path wd = null;
+
+ @Override
+ protected FileSystem createFileSystem() throws Exception {
+ return fSys;
+ }
+
+ @Override
+ protected Path getDefaultWorkingDirectory() throws IOException {
+ if (wd == null) {
+ wd = fSys.getWorkingDirectory();
+ }
+ return wd;
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWDAbsolute() throws IOException {
+ Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
+ fSys.mkdirs(absoluteDir);
+ fSys.setWorkingDirectory(absoluteDir);
+ Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testListStatusThrowsExceptionForUnreadableDir() {
+ SwiftTestUtils.skip("unsupported");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testFsStatus() throws Exception {
+ super.testFsStatus();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWorkingDirectory() throws Exception {
+ super.testWorkingDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testMkdirs() throws Exception {
+ super.testMkdirs();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ super.testMkdirsFailsForSubdirectoryOfExistingFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGetFileStatusThrowsExceptionForNonExistentFile() throws
+ Exception {
+ super.testGetFileStatusThrowsExceptionForNonExistentFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testListStatusThrowsExceptionForNonExistentFile() throws
+ Exception {
+ super.testListStatusThrowsExceptionForNonExistentFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testListStatus() throws Exception {
+ super.testListStatus();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testListStatusFilterWithNoMatches() throws Exception {
+ super.testListStatusFilterWithNoMatches();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testListStatusFilterWithSomeMatches() throws Exception {
+ super.testListStatusFilterWithSomeMatches();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusNonExistentFile() throws Exception {
+ super.testGlobStatusNonExistentFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusWithNoMatchesInPath() throws Exception {
+ super.testGlobStatusWithNoMatchesInPath();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusSomeMatchesInDirectories() throws Exception {
+ super.testGlobStatusSomeMatchesInDirectories();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusWithMultipleWildCardMatches() throws Exception {
+ super.testGlobStatusWithMultipleWildCardMatches();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception {
+ super.testGlobStatusWithMultipleMatchesOfSingleChar();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithEmptyPathResults() throws Exception {
+ super.testGlobStatusFilterWithEmptyPathResults();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithSomePathMatchesAndTrivialFilter() throws
+ Exception {
+ super.testGlobStatusFilterWithSomePathMatchesAndTrivialFilter();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter() throws
+ Exception {
+ super.testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter() throws
+ Exception {
+ super.testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter() throws
+ Exception {
+ super.testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter() throws
+ Exception {
+ super.testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteReadAndDeleteEmptyFile() throws Exception {
+ super.testWriteReadAndDeleteEmptyFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteReadAndDeleteHalfABlock() throws Exception {
+ super.testWriteReadAndDeleteHalfABlock();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteReadAndDeleteOneBlock() throws Exception {
+ super.testWriteReadAndDeleteOneBlock();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
+ super.testWriteReadAndDeleteOneAndAHalfBlocks();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteReadAndDeleteTwoBlocks() throws Exception {
+ super.testWriteReadAndDeleteTwoBlocks();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testOverwrite() throws IOException {
+ super.testOverwrite();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testWriteInNonExistentDirectory() throws IOException {
+ super.testWriteInNonExistentDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testDeleteNonExistentFile() throws IOException {
+ super.testDeleteNonExistentFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testDeleteRecursively() throws IOException {
+ super.testDeleteRecursively();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testDeleteEmptyDirectory() throws IOException {
+ super.testDeleteEmptyDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameNonExistentPath() throws Exception {
+ super.testRenameNonExistentPath();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileToNonExistentDirectory() throws Exception {
+ super.testRenameFileToNonExistentDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileToDestinationWithParentFile() throws Exception {
+ super.testRenameFileToDestinationWithParentFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileToExistingParent() throws Exception {
+ super.testRenameFileToExistingParent();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileToItself() throws Exception {
+ super.testRenameFileToItself();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ super.testRenameFileAsExistingFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameFileAsExistingDirectory() throws Exception {
+ super.testRenameFileAsExistingDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryToItself() throws Exception {
+ super.testRenameDirectoryToItself();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryToNonExistentParent() throws Exception {
+ super.testRenameDirectoryToNonExistentParent();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryAsNonExistentDirectory() throws Exception {
+ super.testRenameDirectoryAsNonExistentDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryAsEmptyDirectory() throws Exception {
+ super.testRenameDirectoryAsEmptyDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryAsNonEmptyDirectory() throws Exception {
+ super.testRenameDirectoryAsNonEmptyDirectory();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testRenameDirectoryAsFile() throws Exception {
+ super.testRenameDirectoryAsFile();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testInputStreamClosedTwice() throws IOException {
+ super.testInputStreamClosedTwice();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testOutputStreamClosedTwice() throws IOException {
+ super.testOutputStreamClosedTwice();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testGetWrappedInputStream() throws IOException {
+ super.testGetWrappedInputStream();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ @Override
+ public void testCopyToLocalWithUseRawLocalFileSystemOption() throws
+ Exception {
+ super.testCopyToLocalWithUseRawLocalFileSystemOption();
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java
new file mode 100644
index 00000000000..6b1b79fa5e2
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Test;
+
+import java.net.URL;
+
+/**
+ * This test just debugs which log resources are being picked up
+ */
+public class TestLogResources implements SwiftTestConstants {
+ protected static final Log LOG =
+ LogFactory.getLog(TestLogResources.class);
+
+ private void printf(String format, Object... args) {
+ String msg = String.format(format, args);
+ System.out.printf(msg + "\n");
+ LOG.info(msg);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testWhichLog4JPropsFile() throws Throwable {
+ locateResource("log4j.properties");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testWhichLog4JXMLFile() throws Throwable {
+ locateResource("log4j.XML");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCommonsLoggingProps() throws Throwable {
+ locateResource("commons-logging.properties");
+ }
+
+ private void locateResource(String resource) {
+ URL url = this.getClass().getClassLoader().getResource(resource);
+ if (url != null) {
+ printf("resource %s is at %s", resource, url);
+ } else {
+ printf("resource %s is not on the classpath", resource);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java
new file mode 100644
index 00000000000..cdcd7626c87
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Seek tests verify that
+ *
+ * When you seek on a 0 byte file to byte (0), it's not an error.
+ * When you seek past the end of a file, it's an error that should
+ * raise -what- EOFException?
+ * when you seek forwards, you get new data
+ * when you seek backwards, you get the previous data
+ * That this works for big multi-MB files as well as small ones.
+ *
+ * These may seem "obvious", but the more the input streams try to be clever
+ * about offsets and buffering, the more likely it is that seek() will start
+ * to get confused.
+ */
+public class TestReadPastBuffer extends SwiftFileSystemBaseTest {
+ protected static final Log LOG =
+ LogFactory.getLog(TestReadPastBuffer.class);
+ public static final int SWIFT_READ_BLOCKSIZE = 4096;
+ public static final int SEEK_FILE_LEN = SWIFT_READ_BLOCKSIZE * 2;
+
+ private Path testPath;
+ private Path readFile;
+ private Path zeroByteFile;
+ private FSDataInputStream instream;
+
+
+ /**
+ * Get a configuration which a small blocksize reported to callers
+ * @return a configuration for this test
+ */
+ @Override
+ public Configuration getConf() {
+ Configuration conf = super.getConf();
+ /*
+ * set to 4KB
+ */
+ conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, SWIFT_READ_BLOCKSIZE);
+ return conf;
+ }
+
+ /**
+ * Setup creates dirs under test/hadoop
+ *
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);
+
+ //delete the test directory
+ testPath = path("/test");
+ readFile = new Path(testPath, "TestReadPastBuffer.txt");
+ createFile(readFile, block);
+ }
+
+ @After
+ public void cleanFile() {
+ IOUtils.closeStream(instream);
+ instream = null;
+ }
+
+ /**
+ * Create a config with a 1KB request size
+ * @return a config
+ */
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
+ return conf;
+ }
+
+ /**
+ * Seek past the buffer then read
+ * @throws Throwable problems
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekAndReadPastEndOfFile() throws Throwable {
+ instream = fs.open(readFile);
+ assertEquals(0, instream.getPos());
+ //expect that seek to 0 works
+ //go just before the end
+ instream.seek(SEEK_FILE_LEN - 2);
+ assertTrue("Premature EOF", instream.read() != -1);
+ assertTrue("Premature EOF", instream.read() != -1);
+ assertMinusOne("read past end of file", instream.read());
+ }
+
+ /**
+ * Seek past the buffer and attempt a read(buffer)
+ * @throws Throwable failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekBulkReadPastEndOfFile() throws Throwable {
+ instream = fs.open(readFile);
+ assertEquals(0, instream.getPos());
+ //go just before the end
+ instream.seek(SEEK_FILE_LEN - 1);
+ byte[] buffer = new byte[1];
+ int result = instream.read(buffer, 0, 1);
+ //next byte is expected to fail
+ result = instream.read(buffer, 0, 1);
+ assertMinusOne("read past end of file", result);
+ //and this one
+ result = instream.read(buffer, 0, 1);
+ assertMinusOne("read past end of file", result);
+
+ //now do an 0-byte read and expect it to
+ //to be checked first
+ result = instream.read(buffer, 0, 0);
+ assertEquals("EOF checks coming before read range check", 0, result);
+
+ }
+
+
+
+ /**
+ * Read past the buffer size byte by byte and verify that it refreshed
+ * @throws Throwable
+ */
+ @Test
+ public void testReadPastBufferSize() throws Throwable {
+ instream = fs.open(readFile);
+
+ while (instream.read() != -1);
+ //here we have gone past the end of a file and its buffer. Now try again
+ assertMinusOne("reading after the (large) file was read: "+ instream,
+ instream.read());
+ }
+}
+
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java
new file mode 100644
index 00000000000..9a28886473c
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * Seek tests verify that
+ *
+ * When you seek on a 0 byte file to byte (0), it's not an error.
+ * When you seek past the end of a file, it's an error that should
+ * raise -what- EOFException?
+ * when you seek forwards, you get new data
+ * when you seek backwards, you get the previous data
+ * That this works for big multi-MB files as well as small ones.
+ *
+ * These may seem "obvious", but the more the input streams try to be clever
+ * about offsets and buffering, the more likely it is that seek() will start
+ * to get confused.
+ */
+public class TestSeek extends SwiftFileSystemBaseTest {
+ protected static final Log LOG =
+ LogFactory.getLog(TestSeek.class);
+ public static final int SMALL_SEEK_FILE_LEN = 256;
+
+ private Path testPath;
+ private Path smallSeekFile;
+ private Path zeroByteFile;
+ private FSDataInputStream instream;
+
+ /**
+ * Setup creates dirs under test/hadoop
+ *
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ //delete the test directory
+ testPath = path("/test");
+ smallSeekFile = new Path(testPath, "seekfile.txt");
+ zeroByteFile = new Path(testPath, "zero.txt");
+ byte[] block = SwiftTestUtils.dataset(SMALL_SEEK_FILE_LEN, 0, 255);
+ //this file now has a simple rule: offset => value
+ createFile(smallSeekFile, block);
+ createEmptyFile(zeroByteFile);
+ }
+
+ @After
+ public void cleanFile() {
+ IOUtils.closeStream(instream);
+ instream = null;
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekZeroByteFile() throws Throwable {
+ instream = fs.open(zeroByteFile);
+ assertEquals(0, instream.getPos());
+ //expect initial read to fai;
+ int result = instream.read();
+ assertMinusOne("initial byte read", result);
+ byte[] buffer = new byte[1];
+ //expect that seek to 0 works
+ instream.seek(0);
+ //reread, expect same exception
+ result = instream.read();
+ assertMinusOne("post-seek byte read", result);
+ result = instream.read(buffer, 0, 1);
+ assertMinusOne("post-seek buffer read", result);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testBlockReadZeroByteFile() throws Throwable {
+ instream = fs.open(zeroByteFile);
+ assertEquals(0, instream.getPos());
+ //expect that seek to 0 works
+ byte[] buffer = new byte[1];
+ int result = instream.read(buffer, 0, 1);
+ assertMinusOne("block read zero byte file", result);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekReadClosedFile() throws Throwable {
+ instream = fs.open(smallSeekFile);
+ instream.close();
+ try {
+ instream.seek(0);
+ } catch (SwiftConnectionClosedException e) {
+ //expected a closed file
+ }
+ try {
+ instream.read();
+ } catch (IOException e) {
+ //expected a closed file
+ }
+ try {
+ byte[] buffer = new byte[1];
+ int result = instream.read(buffer, 0, 1);
+ } catch (IOException e) {
+ //expected a closed file
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testNegativeSeek() throws Throwable {
+ instream = fs.open(smallSeekFile);
+ assertEquals(0, instream.getPos());
+ try {
+ instream.seek(-1);
+ long p = instream.getPos();
+ LOG.warn("Seek to -1 returned a position of " + p);
+ int result = instream.read();
+ fail(
+ "expected an exception, got data " + result + " at a position of " + p);
+ } catch (IOException e) {
+ //bad seek -expected
+ }
+ assertEquals(0, instream.getPos());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekFile() throws Throwable {
+ instream = fs.open(smallSeekFile);
+ assertEquals(0, instream.getPos());
+ //expect that seek to 0 works
+ instream.seek(0);
+ int result = instream.read();
+ assertEquals(0, result);
+ assertEquals(1, instream.read());
+ assertEquals(2, instream.getPos());
+ assertEquals(2, instream.read());
+ assertEquals(3, instream.getPos());
+ instream.seek(128);
+ assertEquals(128, instream.getPos());
+ assertEquals(128, instream.read());
+ instream.seek(63);
+ assertEquals(63, instream.read());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekAndReadPastEndOfFile() throws Throwable {
+ instream = fs.open(smallSeekFile);
+ assertEquals(0, instream.getPos());
+ //expect that seek to 0 works
+ //go just before the end
+ instream.seek(SMALL_SEEK_FILE_LEN - 2);
+ assertTrue("Premature EOF", instream.read() != -1);
+ assertTrue("Premature EOF", instream.read() != -1);
+ assertMinusOne("read past end of file", instream.read());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekAndPastEndOfFileThenReseekAndRead() throws Throwable {
+ instream = fs.open(smallSeekFile);
+ //go just before the end. This may or may not fail; it may be delayed until the
+ //read
+ try {
+ instream.seek(SMALL_SEEK_FILE_LEN);
+ //if this doesn't trigger, then read() is expected to fail
+ assertMinusOne("read after seeking past EOF", instream.read());
+ } catch (EOFException expected) {
+ //here an exception was raised in seek
+ }
+ instream.seek(1);
+ assertTrue("Premature EOF", instream.read() != -1);
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
+ return conf;
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSeekBigFile() throws Throwable {
+ Path testSeekFile = new Path(testPath, "bigseekfile.txt");
+ byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
+ createFile(testSeekFile, block);
+ instream = fs.open(testSeekFile);
+ assertEquals(0, instream.getPos());
+ //expect that seek to 0 works
+ instream.seek(0);
+ int result = instream.read();
+ assertEquals(0, result);
+ assertEquals(1, instream.read());
+ assertEquals(2, instream.read());
+
+ //do seek 32KB ahead
+ instream.seek(32768);
+ assertEquals("@32768", block[32768], (byte) instream.read());
+ instream.seek(40000);
+ assertEquals("@40000", block[40000], (byte) instream.read());
+ instream.seek(8191);
+ assertEquals("@8191", block[8191], (byte) instream.read());
+ instream.seek(0);
+ assertEquals("@0", 0, (byte) instream.read());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
+ Path testSeekFile = new Path(testPath, "bigseekfile.txt");
+ byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
+ createFile(testSeekFile, block);
+ instream = fs.open(testSeekFile);
+ instream.seek(39999);
+ assertTrue(-1 != instream.read());
+ assertEquals (40000, instream.getPos());
+
+ byte[] readBuffer = new byte[256];
+ instream.read(128, readBuffer, 0, readBuffer.length);
+ //have gone back
+ assertEquals(40000, instream.getPos());
+ //content is the same too
+ assertEquals("@40000", block[40000], (byte) instream.read());
+ //now verify the picked up data
+ for (int i = 0; i < 256; i++) {
+ assertEquals("@" + i, block[i + 128], readBuffer[i]);
+ }
+ }
+
+ /**
+ * work out the expected byte from a specific offset
+ * @param offset offset in the file
+ * @return the value
+ */
+ int expectedByte(int offset) {
+ return offset & 0xff;
+ }
+}
+
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java
new file mode 100644
index 00000000000..0212b4d9c65
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_LOCATION_AWARE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_TENANT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_BLOCKSIZE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONNECTION_TIMEOUT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PARTITION_SIZE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_HOST_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_RETRY_COUNT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PREFIX;
+
+/**
+ * Test the swift service-specific configuration binding features
+ */
+public class TestSwiftConfig extends Assert {
+
+
+ public static final String SERVICE = "openstack";
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testEmptyUrl() throws Exception {
+ final Configuration configuration = new Configuration();
+
+ set(configuration, DOT_TENANT, "tenant");
+ set(configuration, DOT_USERNAME, "username");
+ set(configuration, DOT_PASSWORD, "password");
+ mkInstance(configuration);
+ }
+
+@Test
+ public void testEmptyTenant() throws Exception {
+ final Configuration configuration = new Configuration();
+ set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+ set(configuration, DOT_USERNAME, "username");
+ set(configuration, DOT_PASSWORD, "password");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testEmptyUsername() throws Exception {
+ final Configuration configuration = new Configuration();
+ set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+ set(configuration, DOT_TENANT, "tenant");
+ set(configuration, DOT_PASSWORD, "password");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testEmptyPassword() throws Exception {
+ final Configuration configuration = new Configuration();
+ set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+ set(configuration, DOT_TENANT, "tenant");
+ set(configuration, DOT_USERNAME, "username");
+ mkInstance(configuration);
+ }
+
+ @Test
+ public void testGoodRetryCount() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_RETRY_COUNT, "3");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testBadRetryCount() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_RETRY_COUNT, "three");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testBadConnectTimeout() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_CONNECTION_TIMEOUT, "three");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testZeroBlocksize() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_BLOCKSIZE, "0");
+ mkInstance(configuration);
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testNegativeBlocksize() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_BLOCKSIZE, "-1");
+ mkInstance(configuration);
+ }
+
+ @Test
+ public void testPositiveBlocksize() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ int size = 127;
+ configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
+ SwiftRestClient restClient = mkInstance(configuration);
+ assertEquals(size, restClient.getBlocksizeKB());
+ }
+
+ @Test
+ public void testLocationAwareTruePropagates() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ set(configuration, DOT_LOCATION_AWARE, "true");
+ SwiftRestClient restClient = mkInstance(configuration);
+ assertTrue(restClient.isLocationAware());
+ }
+
+ @Test
+ public void testLocationAwareFalsePropagates() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ set(configuration, DOT_LOCATION_AWARE, "false");
+ SwiftRestClient restClient = mkInstance(configuration);
+ assertFalse(restClient.isLocationAware());
+ }
+
+ @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+ public void testNegativePartsize() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ configuration.set(SWIFT_PARTITION_SIZE, "-1");
+ SwiftRestClient restClient = mkInstance(configuration);
+ }
+
+ @Test
+ public void testPositivePartsize() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ int size = 127;
+ configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
+ SwiftRestClient restClient = mkInstance(configuration);
+ assertEquals(size, restClient.getPartSizeKB());
+ }
+
+ @Test
+ public void testProxyData() throws Exception {
+ final Configuration configuration = createCoreConfig();
+ String proxy="web-proxy";
+ int port = 8088;
+ configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
+ configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
+ SwiftRestClient restClient = mkInstance(configuration);
+ assertEquals(proxy, restClient.getProxyHost());
+ assertEquals(port, restClient.getProxyPort());
+ }
+
+ private Configuration createCoreConfig() {
+ final Configuration configuration = new Configuration();
+ set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+ set(configuration, DOT_TENANT, "tenant");
+ set(configuration, DOT_USERNAME, "username");
+ set(configuration, DOT_PASSWORD, "password");
+ return configuration;
+ }
+
+ private void set(Configuration configuration, String field, String value) {
+ configuration.set(SWIFT_SERVICE_PREFIX + SERVICE + field, value);
+ }
+
+ private SwiftRestClient mkInstance(Configuration configuration) throws
+ IOException,
+ URISyntaxException {
+ URI uri = new URI("swift://container.openstack/");
+ return SwiftRestClient.getInstance(uri, configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
new file mode 100644
index 00000000000..8ad0ca49a35
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertFileHasLength;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertIsDirectory;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
+
+
+/**
+ * Test basic filesystem operations.
+ * Many of these are similar to those in {@link TestSwiftFileSystemContract}
+ * -this is a JUnit4 test suite used to initially test the Swift
+ * component. Once written, there's no reason not to retain these tests.
+ */
+public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
+
+ private static final Log LOG =
+ LogFactory.getLog(TestSwiftFileSystemBasicOps.class);
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLsRoot() throws Throwable {
+ Path path = new Path("/");
+ FileStatus[] statuses = fs.listStatus(path);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMkDir() throws Throwable {
+ Path path = new Path("/test/MkDir");
+ fs.mkdirs(path);
+ //success then -so try a recursive operation
+ fs.delete(path, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteNonexistentFile() throws Throwable {
+ Path path = new Path("/test/DeleteNonexistentFile");
+ assertFalse("delete returned true", fs.delete(path, false));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPutFile() throws Throwable {
+ Path path = new Path("/test/PutFile");
+ Exception caught = null;
+ writeTextFile(fs, path, "Testing a put to a file", false);
+ assertDeleted(path, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPutGetFile() throws Throwable {
+ Path path = new Path("/test/PutGetFile");
+ try {
+ String text = "Testing a put and get to a file "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text, false);
+
+ String result = readBytesToString(fs, path, text.length());
+ assertEquals(text, result);
+ } finally {
+ delete(fs, path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPutDeleteFileInSubdir() throws Throwable {
+ Path path =
+ new Path("/test/PutDeleteFileInSubdir/testPutDeleteFileInSubdir");
+ String text = "Testing a put and get to a file in a subdir "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text, false);
+ assertDeleted(path, false);
+ //now delete the parent that should have no children
+ assertDeleted(new Path("/test/PutDeleteFileInSubdir"), false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRecursiveDelete() throws Throwable {
+ Path childpath =
+ new Path("/test/testRecursiveDelete");
+ String text = "Testing a put and get to a file in a subdir "
+ + System.currentTimeMillis();
+ writeTextFile(fs, childpath, text, false);
+ //now delete the parent that should have no children
+ assertDeleted(new Path("/test"), true);
+ assertFalse("child entry still present " + childpath, fs.exists(childpath));
+ }
+
+ private void delete(SwiftNativeFileSystem fs, Path path) {
+ try {
+ if (!fs.delete(path, false)) {
+ LOG.warn("Failed to delete " + path);
+ }
+ } catch (IOException e) {
+ LOG.warn("deleting " + path, e);
+ }
+ }
+
+ private void deleteR(SwiftNativeFileSystem fs, Path path) {
+ try {
+ if (!fs.delete(path, true)) {
+ LOG.warn("Failed to delete " + path);
+ }
+ } catch (IOException e) {
+ LOG.warn("deleting " + path, e);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOverwrite() throws Throwable {
+ Path path = new Path("/test/Overwrite");
+ try {
+ String text = "Testing a put to a file "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text, false);
+ assertFileHasLength(fs, path, text.length());
+ String text2 = "Overwriting a file "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text2, true);
+ assertFileHasLength(fs, path, text2.length());
+ String result = readBytesToString(fs, path, text2.length());
+ assertEquals(text2, result);
+ } finally {
+ delete(fs, path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOverwriteDirectory() throws Throwable {
+ Path path = new Path("/test/testOverwriteDirectory");
+ try {
+ fs.mkdirs(path.getParent());
+ String text = "Testing a put to a file "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text, false);
+ assertFileHasLength(fs, path, text.length());
+ } finally {
+ delete(fs, path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testFileStatus() throws Throwable {
+ Path path = new Path("/test/FileStatus");
+ try {
+ String text = "Testing File Status "
+ + System.currentTimeMillis();
+ writeTextFile(fs, path, text, false);
+ SwiftTestUtils.assertIsFile(fs, path);
+ } finally {
+ delete(fs, path);
+ }
+ }
+
+ /**
+ * Assert that a newly created directory is a directory
+ *
+ * @throws Throwable if not, or if something else failed
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirStatus() throws Throwable {
+ Path path = new Path("/test/DirStatus");
+ try {
+ fs.mkdirs(path);
+ assertIsDirectory(fs, path);
+ } finally {
+ delete(fs, path);
+ }
+ }
+
+ /**
+ * Assert that if a directory that has children is deleted, it is still
+ * a directory
+ *
+ * @throws Throwable if not, or if something else failed
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirStaysADir() throws Throwable {
+ Path path = new Path("/test/dirStaysADir");
+ Path child = new Path(path, "child");
+ try {
+ //create the dir
+ fs.mkdirs(path);
+ //assert the parent has the directory nature
+ assertIsDirectory(fs, path);
+ //create the child dir
+ writeTextFile(fs, child, "child file", true);
+ //assert the parent has the directory nature
+ assertIsDirectory(fs, path);
+ //now rm the child
+ delete(fs, child);
+ } finally {
+ deleteR(fs, path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreateMultilevelDir() throws Throwable {
+ Path base = new Path("/test/CreateMultilevelDir");
+ Path path = new Path(base, "1/2/3");
+ fs.mkdirs(path);
+ assertExists("deep multilevel dir not created", path);
+ fs.delete(base, true);
+ assertPathDoesNotExist("Multilevel delete failed", path);
+ assertPathDoesNotExist("Multilevel delete failed", base);
+
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreateDirWithFileParent() throws Throwable {
+ Path path = new Path("/test/CreateDirWithFileParent");
+ Path child = new Path(path, "subdir/child");
+ fs.mkdirs(path.getParent());
+ try {
+ //create the child dir
+ writeTextFile(fs, path, "parent", true);
+ try {
+ fs.mkdirs(child);
+ } catch (SwiftNotDirectoryException expected) {
+ LOG.debug("Expected Exception", expected);
+ }
+ } finally {
+ fs.delete(path, true);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLongObjectNamesForbidden() throws Throwable {
+ StringBuilder buffer = new StringBuilder(1200);
+ buffer.append("/");
+ for (int i = 0; i < (1200 / 4); i++) {
+ buffer.append(String.format("%04x", i));
+ }
+ String pathString = buffer.toString();
+ Path path = new Path(pathString);
+ try {
+ writeTextFile(fs, path, pathString, true);
+ //if we get here, problems.
+ fs.delete(path, false);
+ fail("Managed to create an object with a name of length "
+ + pathString.length());
+ } catch (SwiftBadRequestException e) {
+ //expected
+ //LOG.debug("Caught exception " + e, e);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLsNonExistentFile() throws Exception {
+ try {
+ Path path = new Path("/test/hadoop/file");
+ FileStatus[] statuses = fs.listStatus(path);
+ fail("Should throw FileNotFoundException on " + path
+ + " but got list of length " + statuses.length);
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java
new file mode 100644
index 00000000000..20f574996f9
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Test block location logic.
+ * The endpoint may or may not be location-aware
+ */
+public class TestSwiftFileSystemBlockLocation extends SwiftFileSystemBaseTest {
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateSingleFileBlocks() throws Throwable {
+ describe("verify that a file returns 1+ blocks");
+ FileStatus fileStatus = createFileAndGetStatus();
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations(fileStatus, 0, 1);
+ assertNotEqual("No block locations supplied for " + fileStatus, 0,
+ locations.length);
+ for (BlockLocation location : locations) {
+ assertLocationValid(location);
+ }
+ }
+
+ private void assertLocationValid(BlockLocation location) throws
+ IOException {
+ LOG.info(location);
+ String[] hosts = location.getHosts();
+ String[] names = location.getNames();
+ assertNotEqual("No hosts supplied for " + location, 0, hosts.length);
+ //for every host, there's a name.
+ assertEquals("Unequal names and hosts in " + location,
+ hosts.length, names.length);
+ assertEquals(SwiftProtocolConstants.BLOCK_LOCATION,
+ location.getNames()[0]);
+ assertEquals(SwiftProtocolConstants.TOPOLOGY_PATH,
+ location.getTopologyPaths()[0]);
+ }
+
+ private FileStatus createFileAndGetStatus() throws IOException {
+ Path path = path("/test/locatedFile");
+ createFile(path);
+ return fs.getFileStatus(path);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateNullStatus() throws Throwable {
+ describe("verify that a null filestatus maps to a null location array");
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations((FileStatus) null, 0, 1);
+ assertNull(locations);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateNegativeSeek() throws Throwable {
+ describe("verify that a negative offset is illegal");
+ try {
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations(createFileAndGetStatus(),
+ -1,
+ 1);
+ fail("Expected an exception, got " + locations.length + " locations");
+ } catch (IllegalArgumentException e) {
+ //expected
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateNegativeLen() throws Throwable {
+ describe("verify that a negative length is illegal");
+ try {
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations(createFileAndGetStatus(),
+ 0,
+ -1);
+ fail("Expected an exception, got " + locations.length + " locations");
+ } catch (IllegalArgumentException e) {
+ //expected
+ }
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateOutOfRangeLen() throws Throwable {
+ describe("overshooting the length is legal, as long as the" +
+ " origin location is valid");
+
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations(createFileAndGetStatus(),
+ 0,
+ data.length + 100);
+ assertNotNull(locations);
+ assertTrue(locations.length > 0);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateOutOfRangeSrc() throws Throwable {
+ describe("Seeking out of the file length returns an empty array");
+
+ BlockLocation[] locations =
+ getFs().getFileBlockLocations(createFileAndGetStatus(),
+ data.length + 100,
+ 1);
+ assertEmptyBlockLocations(locations);
+ }
+
+ private void assertEmptyBlockLocations(BlockLocation[] locations) {
+ assertNotNull(locations);
+ if (locations.length!=0) {
+ fail("non empty locations[] with first entry of " + locations[0]);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateDirectory() throws Throwable {
+ describe("verify that locating a directory is an error");
+ createFile(path("/test/filename"));
+ FileStatus status = fs.getFileStatus(path("/test"));
+ LOG.info("Filesystem is " + fs + "; target is " + status);
+ SwiftTestUtils.assertIsDirectory(status);
+ BlockLocation[] locations;
+ locations = getFs().getFileBlockLocations(status,
+ 0,
+ 1);
+ assertEmptyBlockLocations(locations);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLocateRootDirectory() throws Throwable {
+ describe("verify that locating the root directory is an error");
+ FileStatus status = fs.getFileStatus(path("/"));
+ SwiftTestUtils.assertIsDirectory(status);
+ BlockLocation[] locations;
+ locations = getFs().getFileBlockLocations(status,
+ 0,
+ 1);
+ assertEmptyBlockLocations(locations);
+ }
+
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java
new file mode 100644
index 00000000000..02111632486
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+/**
+ * Tests that blocksize is never zero for a file, either in the FS default
+ * or the FileStatus value of a queried file
+ */
+public class TestSwiftFileSystemBlocksize extends SwiftFileSystemBaseTest {
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDefaultBlocksizeNonZero() throws Throwable {
+ assertTrue("Zero default blocksize", 0L != getFs().getDefaultBlockSize());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDefaultBlocksizeRootPathNonZero() throws Throwable {
+ assertTrue("Zero default blocksize",
+ 0L != getFs().getDefaultBlockSize(new Path("/")));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDefaultBlocksizeOtherPathNonZero() throws Throwable {
+ assertTrue("Zero default blocksize",
+ 0L != getFs().getDefaultBlockSize(new Path("/test")));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testBlocksizeNonZeroForFile() throws Throwable {
+ Path smallfile = new Path("/test/smallfile");
+ SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
+ createFile(smallfile);
+ FileStatus status = getFs().getFileStatus(smallfile);
+ assertTrue("Zero blocksize in " + status,
+ status.getBlockSize() != 0L);
+ assertTrue("Zero replication in " + status,
+ status.getReplication() != 0L);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java
new file mode 100644
index 00000000000..9b557e7850b
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test Swift FS concurrency logic. This isn't a very accurate test,
+ * because it is hard to consistently generate race conditions.
+ * Consider it "best effort"
+ */
+public class TestSwiftFileSystemConcurrency extends SwiftFileSystemBaseTest {
+ protected static final Log LOG =
+ LogFactory.getLog(TestSwiftFileSystemConcurrency.class);
+ private Exception thread1Ex, thread2Ex;
+ public static final String TEST_RACE_CONDITION_ON_DELETE_DIR =
+ "/test/testraceconditionondirdeletetest";
+
+ /**
+ * test on concurrent file system changes
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRaceConditionOnDirDeleteTest() throws Exception {
+ SwiftTestUtils.skip("Skipping unreliable test");
+
+ final String message = "message";
+ final Path fileToRead = new Path(
+ TEST_RACE_CONDITION_ON_DELETE_DIR +"/files/many-files/file");
+ final ExecutorService executorService = Executors.newFixedThreadPool(2);
+ fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR +"/file/test/file1"));
+ fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR + "/documents/doc1"));
+ fs.create(new Path(
+ TEST_RACE_CONDITION_ON_DELETE_DIR + "/pictures/picture"));
+
+
+ executorService.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ assertDeleted(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR), true);
+ } catch (IOException e) {
+ LOG.warn("deletion thread:" + e, e);
+ thread1Ex = e;
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ executorService.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ final FSDataOutputStream outputStream = fs.create(fileToRead);
+ outputStream.write(message.getBytes());
+ outputStream.close();
+ } catch (IOException e) {
+ LOG.warn("writer thread:" + e, e);
+ thread2Ex = e;
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ executorService.awaitTermination(1, TimeUnit.MINUTES);
+ if (thread1Ex != null) {
+ throw thread1Ex;
+ }
+ if (thread2Ex != null) {
+ throw thread2Ex;
+ }
+ try {
+ fs.open(fileToRead);
+ LOG.info("concurrency test failed to trigger a failure");
+ } catch (FileNotFoundException expected) {
+
+ }
+
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
new file mode 100644
index 00000000000..42ca39a46e4
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import junit.framework.AssertionFailedError;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * This is the full filesystem contract test -which requires the
+ * Default config set up to point to a filesystem.
+ *
+ * Some of the tests override the base class tests -these
+ * are where SwiftFS does not implement those features, or
+ * when the behavior of SwiftFS does not match the normal
+ * contract -which normally means that directories and equal files
+ * are being treated as equal.
+ */
+public class TestSwiftFileSystemContract
+ extends FileSystemContractBaseTest {
+ private static final Log LOG =
+ LogFactory.getLog(TestSwiftFileSystemContract.class);
+
+ @Override
+ protected void setUp() throws Exception {
+ final URI uri = getFilesystemURI();
+ final Configuration conf = new Configuration();
+ fs = createSwiftFS();
+ try {
+ fs.initialize(uri, conf);
+ } catch (IOException e) {
+ //FS init failed, set it to null so that teardown doesn't
+ //attempt to use it
+ fs = null;
+ throw e;
+ }
+ super.setUp();
+ }
+
+ protected URI getFilesystemURI() throws URISyntaxException, IOException {
+ return SwiftTestUtils.getServiceURI(new Configuration());
+ }
+
+ protected SwiftNativeFileSystem createSwiftFS() throws IOException {
+ SwiftNativeFileSystem swiftNativeFileSystem =
+ new SwiftNativeFileSystem();
+ return swiftNativeFileSystem;
+ }
+
+ @Override
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ Path testDir = path("/test/hadoop");
+ assertFalse(fs.exists(testDir));
+ assertTrue(fs.mkdirs(testDir));
+ assertTrue(fs.exists(testDir));
+
+ Path filepath = path("/test/hadoop/file");
+ SwiftTestUtils.writeTextFile(fs, filepath, "hello, world", false);
+
+ Path testSubDir = new Path(filepath, "subdir");
+ SwiftTestUtils.assertPathDoesNotExist(fs, "subdir before mkdir", testSubDir);
+
+ try {
+ fs.mkdirs(testSubDir);
+ fail("Should throw IOException.");
+ } catch (SwiftNotDirectoryException e) {
+ // expected
+ assertEquals(filepath,e.getPath());
+ }
+ //now verify that the subdir path does not exist
+ SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir);
+
+ Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir");
+ try {
+ fs.mkdirs(testDeepSubDir);
+ fail("Should throw IOException.");
+ } catch (SwiftNotDirectoryException e) {
+ // expected
+ }
+ SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir after mkdir",
+ testDeepSubDir);
+
+ }
+
+ @Override
+ public void testWriteReadAndDeleteEmptyFile() throws Exception {
+ try {
+ super.testWriteReadAndDeleteEmptyFile();
+ } catch (AssertionFailedError e) {
+ SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
+ }
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ //unsupported
+ }
+
+ public void testZeroByteFilesAreFiles() throws Exception {
+// SwiftTestUtils.unsupported("testZeroByteFilesAreFiles");
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java
new file mode 100644
index 00000000000..81af49c2a34
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+/**
+ * Test deletion operations
+ */
+public class TestSwiftFileSystemDelete extends SwiftFileSystemBaseTest {
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteEmptyFile() throws IOException {
+ final Path file = new Path("/test/testDeleteEmptyFile");
+ createEmptyFile(file);
+ SwiftTestUtils.noteAction("about to delete");
+ assertDeleted(file, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteEmptyFileTwice() throws IOException {
+ final Path file = new Path("/test/testDeleteEmptyFileTwice");
+ createEmptyFile(file);
+ assertDeleted(file, true);
+ SwiftTestUtils.noteAction("multiple creates, and deletes");
+ assertFalse("Delete returned true", fs.delete(file, false));
+ createEmptyFile(file);
+ assertDeleted(file, true);
+ assertFalse("Delete returned true", fs.delete(file, false));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteNonEmptyFile() throws IOException {
+ final Path file = new Path("/test/testDeleteNonEmptyFile");
+ createFile(file);
+ assertDeleted(file, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteNonEmptyFileTwice() throws IOException {
+ final Path file = new Path("/test/testDeleteNonEmptyFileTwice");
+ createFile(file);
+ assertDeleted(file, true);
+ assertFalse("Delete returned true", fs.delete(file, false));
+ createFile(file);
+ assertDeleted(file, true);
+ assertFalse("Delete returned true", fs.delete(file, false));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDeleteTestDir() throws IOException {
+ final Path file = new Path("/test/");
+ fs.delete(file, true);
+ assertPathDoesNotExist("Test dir found", file);
+ }
+
+ /**
+ * Test recursive root directory deletion fails if there is an entry underneath
+ * @throws Throwable
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRmRootDirRecursiveIsForbidden() throws Throwable {
+ Path root = path("/");
+ Path testFile = path("/test");
+ createFile(testFile);
+ assertTrue("rm(/) returned false", fs.delete(root, true));
+ assertExists("Root dir is missing", root);
+ assertPathDoesNotExist("test file not deleted", testFile);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
new file mode 100644
index 00000000000..ae598201c3d
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+
+/**
+ * Test swift-specific directory logic.
+ * This class is HDFS-1 compatible; its designed to be subclases by something
+ * with HDFS2 extensions
+ */
+public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
+
+ /**
+ * Asserts that a zero byte file has a status of file and not
+ * directory or symlink
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testZeroByteFilesAreDirectories() throws Exception {
+ Path src = path("/test/testZeroByteFilesAreFiles");
+ //create a zero byte file
+ SwiftTestUtils.touch(fs, src);
+ SwiftTestUtils.assertIsDirectory(fs, src);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testNoStatusForMissingDirectories() throws Throwable {
+ Path missing = path("/test/testNoStatusForMissingDirectories");
+ assertPathDoesNotExist("leftover?", missing);
+ try {
+ FileStatus[] statuses = fs.listStatus(missing);
+ //not expected
+ fail("Expected a FileNotFoundException, got the status " + statuses);
+ } catch (FileNotFoundException expected) {
+ //expected
+ }
+ }
+
+ /**
+ * test that a dir off root has a listStatus() call that
+ * works as expected. and that when a child is added. it changes
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ assertExists("created test directory", test);
+ FileStatus[] statuses = fs.listStatus(test);
+ String statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString, 0,
+ statuses.length);
+
+ Path src = path("/test/file");
+
+ //create a zero byte file
+ SwiftTestUtils.touch(fs, src);
+ //stat it
+ statuses = fs.listStatus(test);
+ statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString, 1,
+ statuses.length);
+ SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
+ assertTrue("isDir(): Not a directory: " + stat, stat.isDir());
+ extraStatusAssertions(stat);
+ }
+
+ /**
+ * test that a dir two levels down has a listStatus() call that
+ * works as expected.
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
+ Path test = path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
+ fs.delete(test, true);
+ mkdirs(test);
+ assertExists("created test sub directory", test);
+ FileStatus[] statuses = fs.listStatus(test);
+ String statusString = statusToString(test.toString(), statuses);
+ assertEquals("Wrong number of elements in file status " + statusString,0,
+ statuses.length);
+ }
+
+ private String statusToString(String pathname,
+ FileStatus[] statuses) {
+ assertNotNull(statuses);
+ return SwiftTestUtils.dumpStats(pathname,statuses);
+ }
+
+ /**
+ * method for subclasses to add extra assertions
+ * @param stat status to look at
+ */
+ protected void extraStatusAssertions(SwiftFileStatus stat) {
+
+ }
+
+ /**
+ * Asserts that a zero byte file has a status of file and not
+ * directory or symlink
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMultiByteFilesAreFiles() throws Exception {
+ Path src = path("/test/testMultiByteFilesAreFiles");
+ SwiftTestUtils.writeTextFile(fs, src, "testMultiByteFilesAreFiles", false);
+ assertIsFile(src);
+ FileStatus status = fs.getFileStatus(src);
+ assertFalse(status.isDir());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java
new file mode 100644
index 00000000000..7a35b46b3e4
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Locale;
+
+public class TestSwiftFileSystemExtendedContract extends SwiftFileSystemBaseTest {
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOpenNonExistingFile() throws IOException {
+ final Path p = new Path("/test/testOpenNonExistingFile");
+ //open it as a file, should get FileNotFoundException
+ try {
+ final FSDataInputStream in = fs.open(p);
+ in.close();
+ fail("didn't expect to get here");
+ } catch (FileNotFoundException fnfe) {
+ LOG.debug("Expected: " + fnfe, fnfe);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testFilesystemHasURI() throws Throwable {
+ assertNotNull(fs.getUri());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreateFile() throws Exception {
+ final Path f = new Path("/test/testCreateFile");
+ final FSDataOutputStream fsDataOutputStream = fs.create(f);
+ fsDataOutputStream.close();
+ assertExists("created file", f);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testWriteReadFile() throws Exception {
+ final Path f = new Path("/test/test");
+ final FSDataOutputStream fsDataOutputStream = fs.create(f);
+ final String message = "Test string";
+ fsDataOutputStream.write(message.getBytes());
+ fsDataOutputStream.close();
+ assertExists("created file", f);
+ FSDataInputStream open = null;
+ try {
+ open = fs.open(f);
+ final byte[] bytes = new byte[512];
+ final int read = open.read(bytes);
+ final byte[] buffer = new byte[read];
+ System.arraycopy(bytes, 0, buffer, 0, read);
+ assertEquals(message, new String(buffer));
+ } finally {
+ fs.delete(f, false);
+ IOUtils.closeStream(open);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConfDefinesFilesystem() throws Throwable {
+ Configuration conf = new Configuration();
+ SwiftTestUtils.getServiceURI(conf);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConfIsValid() throws Throwable {
+ Configuration conf = new Configuration();
+ URI fsURI = SwiftTestUtils.getServiceURI(conf);
+ RestClientBindings.bind(fsURI, conf);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testGetSchemeImplemented() throws Throwable {
+ String scheme = fs.getScheme();
+ assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
+ }
+
+ /**
+ * Assert that a filesystem is case sensitive.
+ * This is done by creating a mixed-case filename and asserting that
+ * its lower case version is not there.
+ *
+ * @throws Exception failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testFilesystemIsCaseSensitive() throws Exception {
+ String mixedCaseFilename = "/test/UPPER.TXT";
+ Path upper = path(mixedCaseFilename);
+ Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
+ assertFalse("File exists" + upper, fs.exists(upper));
+ assertFalse("File exists" + lower, fs.exists(lower));
+ FSDataOutputStream out = fs.create(upper);
+ out.writeUTF("UPPER");
+ out.close();
+ FileStatus upperStatus = fs.getFileStatus(upper);
+ assertExists("Original upper case file" + upper, upper);
+ //verify the lower-case version of the filename doesn't exist
+ assertPathDoesNotExist("lower case file", lower);
+ //now overwrite the lower case version of the filename with a
+ //new version.
+ out = fs.create(lower);
+ out.writeUTF("l");
+ out.close();
+ assertExists("lower case file", lower);
+ //verifEy the length of the upper file hasn't changed
+ assertExists("Original upper case file " + upper, upper);
+ FileStatus newStatus = fs.getFileStatus(upper);
+ assertEquals("Expected status:" + upperStatus
+ + " actual status " + newStatus,
+ upperStatus.getLen(),
+ newStatus.getLen());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java
new file mode 100644
index 00000000000..5e2b1b72319
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertListStatusFinds;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dumpStats;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.touch;
+
+/**
+ * Test the FileSystem#listStatus() operations
+ */
+public class TestSwiftFileSystemLsOperations extends SwiftFileSystemBaseTest {
+
+ private Path[] testDirs;
+
+ /**
+ * Setup creates dirs under test/hadoop
+ *
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ //delete the test directory
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ }
+
+ /**
+ * Create subdirectories and files under test/ for those tests
+ * that want them. Doing so adds overhead to setup and teardown,
+ * so should only be done for those tests that need them.
+ * @throws IOException on an IO problem
+ */
+ private void createTestSubdirs() throws IOException {
+ testDirs = new Path[]{
+ path("/test/hadoop/a"),
+ path("/test/hadoop/b"),
+ path("/test/hadoop/c/1"),
+ };
+
+ assertPathDoesNotExist("test directory setup", testDirs[0]);
+ for (Path path : testDirs) {
+ mkdirs(path);
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListLevelTest() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths = fs.listStatus(path("/test"));
+ assertEquals(dumpStats("/test", paths), 1, paths.length);
+ assertEquals(path("/test/hadoop"), paths[0].getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListLevelTestHadoop() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths;
+ paths = fs.listStatus(path("/test/hadoop"));
+ String stats = dumpStats("/test/hadoop", paths);
+ assertEquals("Paths.length wrong in " + stats, 3, paths.length);
+ assertEquals("Path element[0] wrong: " + stats, path("/test/hadoop/a"),
+ paths[0].getPath());
+ assertEquals("Path element[1] wrong: " + stats, path("/test/hadoop/b"),
+ paths[1].getPath());
+ assertEquals("Path element[2] wrong: " + stats, path("/test/hadoop/c"),
+ paths[2].getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusEmptyDirectory() throws Exception {
+ createTestSubdirs();
+ FileStatus[] paths;
+ paths = fs.listStatus(path("/test/hadoop/a"));
+ assertEquals(dumpStats("/test/hadoop/a", paths), 0,
+ paths.length);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusFile() throws Exception {
+ describe("Create a single file under /test;" +
+ " assert that listStatus(/test) finds it");
+ Path file = path("/test/filename");
+ createFile(file);
+ FileStatus[] pathStats = fs.listStatus(file);
+ assertEquals(dumpStats("/test/", pathStats),
+ 1,
+ pathStats.length);
+ //and assert that the len of that ls'd path is the same as the original
+ FileStatus lsStat = pathStats[0];
+ assertEquals("Wrong file len in listing of " + lsStat,
+ data.length, lsStat.getLen());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListEmptyRoot() throws Throwable {
+ describe("Empty the root dir and verify that an LS / returns {}");
+ cleanup("testListEmptyRoot", fs, "/test");
+ cleanup("testListEmptyRoot", fs, "/user");
+ FileStatus[] fileStatuses = fs.listStatus(path("/"));
+ assertEquals("Non-empty root" + dumpStats("/", fileStatuses),
+ 0,
+ fileStatuses.length);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListNonEmptyRoot() throws Throwable {
+ Path test = path("/test");
+ touch(fs, test);
+ FileStatus[] fileStatuses = fs.listStatus(path("/"));
+ String stats = dumpStats("/", fileStatuses);
+ assertEquals("Wrong #of root children" + stats, 1, fileStatuses.length);
+ FileStatus status = fileStatuses[0];
+ assertEquals("Wrong path value" + stats,test, status.getPath());
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusRootDir() throws Throwable {
+ Path dir = path("/");
+ Path child = path("/test");
+ touch(fs, child);
+ assertListStatusFinds(fs, dir, child);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListStatusFiltered() throws Throwable {
+ Path dir = path("/");
+ Path child = path("/test");
+ touch(fs, child);
+ FileStatus[] stats = fs.listStatus(dir, new AcceptAllFilter());
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(child)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + child
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
new file mode 100644
index 00000000000..9b223f878fa
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
@@ -0,0 +1,442 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+
+/**
+ * Test partitioned uploads.
+ * This is done by forcing a very small partition size and verifying that it
+ * is picked up.
+ */
+public class TestSwiftFileSystemPartitionedUploads extends
+ SwiftFileSystemBaseTest {
+
+ public static final String WRONG_PARTITION_COUNT =
+ "wrong number of partitions written into ";
+ public static final int PART_SIZE = 1;
+ public static final int PART_SIZE_BYTES = PART_SIZE * 1024;
+ public static final int BLOCK_SIZE = 1024;
+ private URI uri;
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ //set the partition size to 1 KB
+ conf.setInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE, PART_SIZE);
+ return conf;
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testPartitionPropertyPropagatesToConf() throws Throwable {
+ assertEquals(1,
+ getConf().getInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE,
+ 0));
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testPartionPropertyPropagatesToStore() throws Throwable {
+ assertEquals(1, fs.getStore().getPartsizeKB());
+ }
+
+ /**
+ * tests functionality for big files ( > 5Gb) upload
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testFilePartUpload() throws Throwable {
+
+ final Path path = new Path("/test/testFilePartUpload");
+
+ int len = 8192;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ try {
+ int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+ assertPartitionsWritten("Startup", out, 0);
+ //write 2048
+ int firstWriteLen = 2048;
+ out.write(src, 0, firstWriteLen);
+ //assert
+ long expected = getExpectedPartitionsWritten(firstWriteLen,
+ PART_SIZE_BYTES,
+ false);
+ SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+ expected);
+ assertPartitionsWritten("First write completed", out, expected);
+ //write the rest
+ int remainder = len - firstWriteLen;
+ SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+ out.write(src, firstWriteLen, remainder);
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+ assertPartitionsWritten("Remaining data", out, expected);
+ out.close();
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ assertPartitionsWritten("Stream closed", out, expected);
+
+ Header[] headers = fs.getStore().getObjectHeaders(path, true);
+ for (Header header : headers) {
+ LOG.info(header.toString());
+ }
+
+ byte[] dest = readDataset(fs, path, len);
+ LOG.info("Read dataset from " + path + ": data length =" + len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ FileStatus status;
+
+ final Path qualifiedPath = path.makeQualified(fs);
+ status = fs.getFileStatus(qualifiedPath);
+ //now see what block location info comes back.
+ //This will vary depending on the Swift version, so the results
+ //aren't checked -merely that the test actually worked
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+ assertNotNull("Null getFileBlockLocations()", locations);
+ assertTrue("empty array returned for getFileBlockLocations()",
+ locations.length > 0);
+
+ //last bit of test -which seems to play up on partitions, which we download
+ //to a skip
+ try {
+ validatePathLen(path, len);
+ } catch (AssertionError e) {
+ //downgrade to a skip
+ throw new AssumptionViolatedException(e, null);
+ }
+
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ }
+ /**
+ * tests functionality for big files ( > 5Gb) upload
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
+
+ final Path path = new Path("/test/testFilePartUploadLengthCheck");
+
+ int len = 8192;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ try {
+ int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+ assertPartitionsWritten("Startup", out, 0);
+ //write 2048
+ int firstWriteLen = 2048;
+ out.write(src, 0, firstWriteLen);
+ //assert
+ long expected = getExpectedPartitionsWritten(firstWriteLen,
+ PART_SIZE_BYTES,
+ false);
+ SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+ expected);
+ assertPartitionsWritten("First write completed", out, expected);
+ //write the rest
+ int remainder = len - firstWriteLen;
+ SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+ out.write(src, firstWriteLen, remainder);
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+ assertPartitionsWritten("Remaining data", out, expected);
+ out.close();
+ expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ assertPartitionsWritten("Stream closed", out, expected);
+
+ Header[] headers = fs.getStore().getObjectHeaders(path, true);
+ for (Header header : headers) {
+ LOG.info(header.toString());
+ }
+
+ byte[] dest = readDataset(fs, path, len);
+ LOG.info("Read dataset from " + path + ": data length =" + len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ FileStatus status = fs.getFileStatus(path);
+
+ //now see what block location info comes back.
+ //This will vary depending on the Swift version, so the results
+ //aren't checked -merely that the test actually worked
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+ assertNotNull("Null getFileBlockLocations()", locations);
+ assertTrue("empty array returned for getFileBlockLocations()",
+ locations.length > 0);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ }
+
+ private FileStatus validatePathLen(Path path, int len) throws IOException {
+ //verify that the length is what was written in a direct status check
+ final Path qualifiedPath = path.makeQualified(fs);
+ FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
+ StringBuilder listing = lsToString(parentDirListing);
+ String parentDirLS = listing.toString();
+ FileStatus status = fs.getFileStatus(qualifiedPath);
+ assertEquals("Length of written file " + qualifiedPath
+ + " from status check " + status
+ + " in dir " + listing,
+ len,
+ status.getLen());
+ String fileInfo = qualifiedPath + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDir());
+
+ FileStatus listedFileStat = resolveChild(parentDirListing, qualifiedPath);
+ assertNotNull("Did not find " + path + " in " + parentDirLS,
+ listedFileStat);
+ //file is in the parent dir. Now validate it's stats
+ assertEquals("Wrong len for " + path + " in listing " + parentDirLS,
+ len,
+ listedFileStat.getLen());
+ listedFileStat.toString();
+ return status;
+ }
+
+ private FileStatus resolveChild(FileStatus[] parentDirListing,
+ Path childPath) {
+ FileStatus listedFileStat = null;
+ for (FileStatus stat : parentDirListing) {
+ if (stat.getPath().equals(childPath)) {
+ listedFileStat = stat;
+ }
+ }
+ return listedFileStat;
+ }
+
+ private StringBuilder lsToString(FileStatus[] parentDirListing) {
+ StringBuilder listing = new StringBuilder();
+ for (FileStatus stat : parentDirListing) {
+ listing.append(stat).append("\n");
+ }
+ return listing;
+ }
+
+ /**
+ * Calculate the #of partitions expected from the upload
+ * @param uploaded number of bytes uploaded
+ * @param partSizeBytes the partition size
+ * @param closed whether or not the stream has closed
+ * @return the expected number of partitions, for use in assertions.
+ */
+ private int getExpectedPartitionsWritten(long uploaded,
+ int partSizeBytes,
+ boolean closed) {
+ //#of partitions in total
+ int partitions = (int) (uploaded / partSizeBytes);
+ //#of bytes past the last partition
+ int remainder = (int) (uploaded % partSizeBytes);
+ if (closed) {
+ //all data is written, so if there was any remainder, it went up
+ //too
+ return partitions + ((remainder > 0) ? 1 : 0);
+ } else {
+ //not closed. All the remainder is buffered,
+ return partitions;
+ }
+ }
+
+ private int getBufferSize() {
+ return fs.getConf().getInt("io.file.buffer.size", 4096);
+ }
+
+ /**
+ * Test sticks up a very large partitioned file and verifies that
+ * it comes back unchanged.
+ * @throws Throwable
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testManyPartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testManyPartitionedFile");
+
+ int len = PART_SIZE_BYTES * 15;
+ final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ BLOCK_SIZE);
+
+ out.write(src, 0, src.length);
+ int expected =
+ getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+ out.close();
+ assertPartitionsWritten("write completed", out, expected);
+ assertEquals("too few bytes written", len,
+ SwiftNativeFileSystem.getBytesWritten(out));
+ assertEquals("too few bytes uploaded", len,
+ SwiftNativeFileSystem.getBytesUploaded(out));
+ //now we verify that the data comes back. If it
+ //doesn't, it means that the ordering of the partitions
+ //isn't right
+ byte[] dest = readDataset(fs, path, len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src, dest, len);
+ //finally, check the data
+ FileStatus[] stats = fs.listStatus(path);
+ assertEquals("wrong entry count in "
+ + SwiftTestUtils.dumpStats(path.toString(), stats),
+ expected, stats.length);
+ }
+
+ /**
+ * Test that when a partitioned file is overwritten by a smaller one,
+ * all the old partitioned files go away
+ * @throws Throwable
+ */
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testOverwritePartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testOverwritePartitionedFile");
+
+ final int len1 = 8192;
+ final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+ FSDataOutputStream out = fs.create(path,
+ false,
+ getBufferSize(),
+ (short) 1,
+ 1024);
+ out.write(src1, 0, len1);
+ out.close();
+ long expected = getExpectedPartitionsWritten(len1,
+ PART_SIZE_BYTES,
+ false);
+ assertPartitionsWritten("initial upload", out, expected);
+ assertExists("Exists", path);
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals("Length", len1, status.getLen());
+ //now write a shorter file with a different dataset
+ final int len2 = 4095;
+ final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
+ out = fs.create(path,
+ true,
+ getBufferSize(),
+ (short) 1,
+ 1024);
+ out.write(src2, 0, len2);
+ out.close();
+ status = fs.getFileStatus(path);
+ assertEquals("Length", len2, status.getLen());
+ byte[] dest = readDataset(fs, path, len2);
+ //compare data
+ SwiftTestUtils.compareByteArrays(src2, dest, len2);
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testDeleteSmallPartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testDeleteSmallPartitionedFile");
+
+ final int len1 = 1024;
+ final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+ SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
+ assertExists("Exists", path);
+
+ Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+ Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+ String ls = SwiftTestUtils.ls(fs, path);
+ assertExists("Partition 0001 Exists in " + ls, part_0001);
+ assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
+ assertExists("Partition 0002 Exists in " + ls, part_0001);
+ fs.delete(path, false);
+ assertPathDoesNotExist("deleted file still there", path);
+ ls = SwiftTestUtils.ls(fs, path);
+ assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+ }
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testDeletePartitionedFile() throws Throwable {
+ final Path path = new Path("/test/testDeletePartitionedFile");
+
+ SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
+ assertExists("Exists", path);
+
+ Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+ Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+ String ls = SwiftTestUtils.ls(fs, path);
+ assertExists("Partition 0001 Exists in " + ls, part_0001);
+ assertExists("Partition 0002 Exists in " + ls, part_0001);
+ fs.delete(path, false);
+ assertPathDoesNotExist("deleted file still there", path);
+ ls = SwiftTestUtils.ls(fs, path);
+ assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+ assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
+ }
+
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testRenamePartitionedFile() throws Throwable {
+ Path src = new Path("/test/testRenamePartitionedFileSrc");
+
+ int len = data.length;
+ SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
+ assertExists("Exists", src);
+
+ String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
+ Path srcPart = new Path(src, partOneName);
+ Path dest = new Path("/test/testRenamePartitionedFileDest");
+ Path destPart = new Path(src, partOneName);
+ assertExists("Partition Exists", srcPart);
+ fs.rename(src, dest);
+ assertPathExists(fs, "dest file missing", dest);
+ FileStatus status = fs.getFileStatus(dest);
+ assertEquals("Length of renamed file is wrong", len, status.getLen());
+ byte[] destData = readDataset(fs, dest, len);
+ //compare data
+ SwiftTestUtils.compareByteArrays(data, destData, len);
+ String srcLs = SwiftTestUtils.ls(fs, src);
+ String destLs = SwiftTestUtils.ls(fs, dest);
+
+ assertPathDoesNotExist("deleted file still found in " + srcLs, src);
+
+ assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
+ }
+
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java
new file mode 100644
index 00000000000..84794cb7250
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
+
+/**
+ * Test filesystem read operations
+ */
+public class TestSwiftFileSystemRead extends SwiftFileSystemBaseTest {
+
+
+ /**
+ * Read past the end of a file: expect the operation to fail
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testOverRead() throws IOException {
+ final String message = "message";
+ final Path filePath = new Path("/test/file.txt");
+
+ writeTextFile(fs, filePath, message, false);
+
+ try {
+ readBytesToString(fs, filePath, 20);
+ fail("expected an exception");
+ } catch (EOFException e) {
+ //expected
+ }
+ }
+
+ /**
+ * Read and write some JSON
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRWJson() throws IOException {
+ final String message = "{" +
+ " 'json': { 'i':43, 'b':true}," +
+ " 's':'string'" +
+ "}";
+ final Path filePath = new Path("/test/file.json");
+
+ writeTextFile(fs, filePath, message, false);
+ String readJson = readBytesToString(fs, filePath, message.length());
+ assertEquals(message,readJson);
+ //now find out where it is
+ FileStatus status = fs.getFileStatus(filePath);
+ BlockLocation[] locations = fs.getFileBlockLocations(status, 0, 10);
+ }
+
+ /**
+ * Read and write some XML
+ * @throws IOException
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRWXML() throws IOException {
+ final String message = "" +
+ " " +
+ " string" +
+ " ";
+ final Path filePath = new Path("/test/file.xml");
+
+ writeTextFile(fs, filePath, message, false);
+ String read = readBytesToString(fs, filePath, message.length());
+ assertEquals(message,read);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
new file mode 100644
index 00000000000..99011d834c5
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.compareByteArrays;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeDataset;
+
+public class TestSwiftFileSystemRename extends SwiftFileSystemBaseTest {
+
+ /**
+ * Rename a file into a directory
+ *
+ * @throws Exception
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileIntoExistingDirectory() throws Exception {
+ assumeRenameSupported();
+
+ Path src = path("/test/olddir/file");
+ createFile(src);
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ Path newFile = path("/test/new/newdir/file");
+ if (!fs.exists(newFile)) {
+ String ls = ls(dst);
+ LOG.info(ls(path("/test/new")));
+ LOG.info(ls(path("/test/hadoop")));
+ fail("did not find " + newFile + " - directory: " + ls);
+ }
+ assertTrue("Destination changed",
+ fs.exists(path("/test/new/newdir/file")));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFile() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/alice/file");
+ final Path newPath = new Path("/test/bob/file");
+ fs.mkdirs(newPath.getParent());
+ final FSDataOutputStream fsDataOutputStream = fs.create(old);
+ final byte[] message = "Some data".getBytes();
+ fsDataOutputStream.write(message);
+ fsDataOutputStream.close();
+
+ assertTrue(fs.exists(old));
+ rename(old, newPath, true, false, true);
+
+ final FSDataInputStream bobStream = fs.open(newPath);
+ final byte[] bytes = new byte[512];
+ final int read = bobStream.read(bytes);
+ bobStream.close();
+ final byte[] buffer = new byte[read];
+ System.arraycopy(bytes, 0, buffer, 0, read);
+ assertEquals(new String(message), new String(buffer));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirectory() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/data/logs");
+ final Path newPath = new Path("/test/var/logs");
+ fs.mkdirs(old);
+ fs.mkdirs(newPath.getParent());
+ assertTrue(fs.exists(old));
+ rename(old, newPath, true, false, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameTheSameDirectory() throws Exception {
+ assumeRenameSupported();
+
+ final Path old = new Path("/test/usr/data");
+ fs.mkdirs(old);
+ rename(old, old, false, true, true);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirectoryIntoExistingDirectory() throws Exception {
+ assumeRenameSupported();
+
+ Path src = path("/test/olddir/dir");
+ fs.mkdirs(src);
+ createFile(path("/test/olddir/dir/file1"));
+ createFile(path("/test/olddir/dir/subdir/file2"));
+
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ //this renames into a child
+ rename(src, dst, true, false, true);
+ assertExists("new dir", path("/test/new/newdir/dir"));
+ assertExists("Renamed nested file1", path("/test/new/newdir/dir/file1"));
+ assertPathDoesNotExist("Nested file1 should have been deleted",
+ path("/test/olddir/dir/file1"));
+ assertExists("Renamed nested subdir",
+ path("/test/new/newdir/dir/subdir/"));
+ assertExists("file under subdir",
+ path("/test/new/newdir/dir/subdir/file2"));
+
+ assertPathDoesNotExist("Nested /test/hadoop/dir/subdir/file2 still exists",
+ path("/test/olddir/dir/subdir/file2"));
+ }
+
+ /**
+ * trying to rename a directory onto itself should fail,
+ * preserving everything underneath.
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameDirToSelf() throws Throwable {
+ assumeRenameSupported();
+ Path parentdir = path("/test/parentdir");
+ fs.mkdirs(parentdir);
+ Path child = new Path(parentdir, "child");
+ createFile(child);
+
+ rename(parentdir, parentdir, false, true, true);
+ //verify the child is still there
+ assertIsFile(child);
+ }
+
+ /**
+ * Assert that root directory renames are not allowed
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameRootDirForbidden() throws Exception {
+ assumeRenameSupported();
+ rename(path("/"),
+ path("/test/newRootDir"),
+ false, true, false);
+ }
+
+ /**
+ * Assert that renaming a parent directory to be a child
+ * of itself is forbidden
+ *
+ * @throws Exception on failures
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameChildDirForbidden() throws Exception {
+ assumeRenameSupported();
+
+ Path parentdir = path("/test/parentdir");
+ fs.mkdirs(parentdir);
+ Path childFile = new Path(parentdir, "childfile");
+ createFile(childFile);
+ //verify one level down
+ Path childdir = new Path(parentdir, "childdir");
+ rename(parentdir, childdir, false, true, false);
+ //now another level
+ fs.mkdirs(childdir);
+ Path childchilddir = new Path(childdir, "childdir");
+ rename(parentdir, childchilddir, false, true, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileAndVerifyContents() throws IOException {
+ assumeRenameSupported();
+
+ final Path filePath = new Path("/test/home/user/documents/file.txt");
+ final Path newFilePath = new Path("/test/home/user/files/file.txt");
+ mkdirs(newFilePath.getParent());
+ int len = 1024;
+ byte[] dataset = dataset(len, 'A', 26);
+ writeDataset(fs, filePath, dataset, len, len, false);
+ rename(filePath, newFilePath, true, false, true);
+ byte[] dest = readDataset(fs, newFilePath, len);
+ compareByteArrays(dataset, dest, len);
+ String reread = readBytesToString(fs, newFilePath, 20);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMoveFileUnderParent() throws Throwable {
+ if (!renameSupported()) return;
+ Path filepath = path("test/file");
+ createFile(filepath);
+ //HDFS expects rename src, src -> true
+ rename(filepath, filepath, true, true, true);
+ //verify the file is still there
+ assertIsFile(filepath);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testMoveDirUnderParent() throws Throwable {
+ if (!renameSupported()) {
+ return;
+ }
+ Path testdir = path("test/dir");
+ fs.mkdirs(testdir);
+ Path parent = testdir.getParent();
+ //the outcome here is ambiguous, so is not checked
+ fs.rename(testdir, parent);
+ assertExists("Source directory has been deleted ", testdir);
+ }
+
+ /**
+ * trying to rename a file onto itself should succeed (it's a no-op)
+ */
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameFileToSelf() throws Throwable {
+ if (!renameSupported()) return;
+ Path filepath = path("test/file");
+ createFile(filepath);
+ //HDFS expects rename src, src -> true
+ rename(filepath, filepath, true, true, true);
+ //verify the file is still there
+ assertIsFile(filepath);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenamedConsistence() throws IOException {
+ assumeRenameSupported();
+ describe("verify that overwriting a file with new data doesn't impact" +
+ " the existing content");
+
+ final Path filePath = new Path("/test/home/user/documents/file.txt");
+ final Path newFilePath = new Path("/test/home/user/files/file.txt");
+ mkdirs(newFilePath.getParent());
+ int len = 1024;
+ byte[] dataset = dataset(len, 'A', 26);
+ byte[] dataset2 = dataset(len, 'a', 26);
+ writeDataset(fs, filePath, dataset, len, len, false);
+ rename(filePath, newFilePath, true, false, true);
+ SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
+ byte[] dest = readDataset(fs, newFilePath, len);
+ compareByteArrays(dataset, dest, len);
+ String reread = readBytesToString(fs, newFilePath, 20);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRenameMissingFile() throws Throwable {
+ assumeRenameSupported();
+ Path path = path("/test/RenameMissingFile");
+ Path path2 = path("/test/RenameMissingFileDest");
+ mkdirs(path("test"));
+ rename(path, path2, false, false, false);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java
new file mode 100644
index 00000000000..cb6ba09ef44
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for SwiftObjectPath class.
+ */
+public class TestSwiftObjectPath implements SwiftTestConstants {
+ private static final Log LOG = LogFactory.getLog(TestSwiftObjectPath.class);
+
+ /**
+ * What an endpoint looks like. This is derived from a (valid)
+ * rackspace endpoint address
+ */
+ private static final String ENDPOINT =
+ "https://storage101.region1.example.org/v1/MossoCloudFS_9fb40cc0-1234-5678-9abc-def000c9a66";
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParsePath() throws Exception {
+ final String pathString = "/home/user/files/file1";
+ final Path path = new Path(pathString);
+ final URI uri = new URI("http://container.localhost");
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ pathString);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParseUrlPath() throws Exception {
+ final String pathString = "swift://container.service1/home/user/files/file1";
+ final URI uri = new URI(pathString);
+ final Path path = new Path(pathString);
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ "/home/user/files/file1");
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testParseAuthenticatedUrl() throws Exception {
+ final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
+ final URI uri = new URI(pathString);
+ final Path path = new Path(pathString);
+ final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+ final SwiftObjectPath actual = new SwiftObjectPath(
+ RestClientBindings.extractContainerName(uri),
+ "/home/tom/documents/finance.docx");
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testConvertToPath() throws Throwable {
+ String initialpath = "/dir/file1";
+ Path ipath = new Path(initialpath);
+ SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
+ ipath);
+ URI endpoint = new URI(ENDPOINT);
+ URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
+ LOG.info("Inital Hadoop Path =" + initialpath);
+ LOG.info("Merged URI=" + uri);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRootDirProbeEmptyPath() throws Throwable {
+ SwiftObjectPath object=new SwiftObjectPath("container","");
+ assertTrue(SwiftUtils.isRootDir(object));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testRootDirProbeRootPath() throws Throwable {
+ SwiftObjectPath object=new SwiftObjectPath("container","/");
+ assertTrue(SwiftUtils.isRootDir(object));
+ }
+
+ private void assertParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+ assertTrue(p1.toString() + " is not a parent of " + p2 ,p1.isEqualToOrParentOf(
+ p2));
+ }
+
+ private void assertNotParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+ assertFalse(p1.toString() + " is a parent of " + p2, p1.isEqualToOrParentOf(
+ p2));
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testChildOfProbe() throws Throwable {
+ SwiftObjectPath parent = new SwiftObjectPath("container",
+ "/parent");
+ SwiftObjectPath parent2 = new SwiftObjectPath("container",
+ "/parent2");
+ SwiftObjectPath child = new SwiftObjectPath("container",
+ "/parent/child");
+ SwiftObjectPath sibling = new SwiftObjectPath("container",
+ "/parent/sibling");
+ SwiftObjectPath grandchild = new SwiftObjectPath("container",
+ "/parent/child/grandchild");
+ assertParentOf(parent, child);
+ assertParentOf(parent, grandchild);
+ assertParentOf(child, grandchild);
+ assertParentOf(parent, parent);
+ assertNotParentOf(child, parent);
+ assertParentOf(child, child);
+ assertNotParentOf(parent, parent2);
+ assertNotParentOf(grandchild, parent);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testChildOfRoot() throws Throwable {
+ SwiftObjectPath root = new SwiftObjectPath("container", "/");
+ SwiftObjectPath child = new SwiftObjectPath("container", "child");
+ SwiftObjectPath grandchild = new SwiftObjectPath("container",
+ "/child/grandchild");
+ assertParentOf(root, child);
+ assertParentOf(root, grandchild);
+ assertParentOf(child, grandchild);
+ assertParentOf(root, root);
+ assertNotParentOf(child, root);
+ assertParentOf(child, child);
+ assertNotParentOf(grandchild, root);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java
new file mode 100644
index 00000000000..cb64bef6c5c
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+
+/**
+ * Add some HDFS-2 only assertions to {@link TestSwiftFileSystemDirectories}
+ */
+public class TestSwiftFileSystemDirectoriesHdfs2 extends
+ TestSwiftFileSystemDirectories {
+
+
+ /**
+ * make assertions about fields that only appear in
+ * FileStatus in HDFS2
+ * @param stat status to look at
+ */
+ protected void extraStatusAssertions(SwiftFileStatus stat) {
+ //HDFS2
+ assertTrue("isDirectory(): Not a directory: " + stat, stat.isDirectory());
+ assertFalse("isFile(): declares itself a file: " + stat, stat.isFile());
+ assertFalse("isFile(): declares itself a file: " + stat, stat.isSymlink());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
new file mode 100644
index 00000000000..2adb8f4264b
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestV2LsOperations extends SwiftFileSystemBaseTest {
+
+ private Path[] testDirs;
+
+ /**
+ * Setup creates dirs under test/hadoop
+ * @throws Exception
+ */
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ //delete the test directory
+ Path test = path("/test");
+ fs.delete(test, true);
+ mkdirs(test);
+ }
+
+ /**
+ * Create subdirectories and files under test/ for those tests
+ * that want them. Doing so adds overhead to setup and teardown,
+ * so should only be done for those tests that need them.
+ * @throws IOException on an IO problem
+ */
+ private void createTestSubdirs() throws IOException {
+ testDirs = new Path[]{
+ path("/test/hadoop/a"),
+ path("/test/hadoop/b"),
+ path("/test/hadoop/c/1"),
+ };
+ assertPathDoesNotExist("test directory setup", testDirs[0]);
+ for (Path path : testDirs) {
+ mkdirs(path);
+ }
+ }
+
+ /**
+ * To get this project to compile under Hadoop 1, this code needs to be
+ * commented out
+ *
+ *
+ * @param fs filesystem
+ * @param dir dir
+ * @param subdir subdir
+ * @param recursive recurse?
+ * @throws IOException IO problems
+ */
+ public static void assertListFilesFinds(FileSystem fs,
+ Path dir,
+ Path subdir,
+ boolean recursive) throws IOException {
+ RemoteIterator iterator =
+ fs.listFiles(dir, recursive);
+ boolean found = false;
+ int entries = 0;
+ StringBuilder builder = new StringBuilder();
+ while (iterator.hasNext()) {
+ LocatedFileStatus next = iterator.next();
+ entries++;
+ builder.append(next.toString()).append('\n');
+ if (next.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + " : "
+ + " entries=" + entries
+ + " content"
+ + builder.toString(),
+ found);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesRootDir() throws Throwable {
+ Path dir = path("/");
+ Path child = new Path(dir, "test");
+ fs.delete(child, true);
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesSubDir() throws Throwable {
+ createTestSubdirs();
+ Path dir = path("/test");
+ Path child = new Path(dir, "text.txt");
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, false);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testListFilesRecursive() throws Throwable {
+ createTestSubdirs();
+ Path dir = path("/test");
+ Path child = new Path(dir, "hadoop/a/a.txt");
+ SwiftTestUtils.writeTextFile(fs, child, "text", false);
+ assertListFilesFinds(fs, dir, child, true);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java
new file mode 100644
index 00000000000..8075e08404a
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_AUTH_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONTAINER_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTPS_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTP_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PASSWORD_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_REGION_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_TENANT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_USERNAME_PROPERTY;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPropertyEquals;
+
+public class TestRestClientBindings extends Assert
+ implements SwiftTestConstants {
+
+ private static final String SERVICE = "sname";
+ private static final String CONTAINER = "cname";
+ private static final String FS_URI = "swift://"
+ + CONTAINER + "." + SERVICE + "/";
+ private static final String AUTH_URL = "http://localhost:8080/auth";
+ private static final String USER = "user";
+ private static final String PASS = "pass";
+ private static final String TENANT = "tenant";
+ private URI filesysURI;
+ private Configuration conf;
+
+ @Before
+ public void setup() throws URISyntaxException {
+ filesysURI = new URI(FS_URI);
+ conf = new Configuration(true);
+ setInstanceVal(conf, SERVICE, DOT_AUTH_URL, AUTH_URL);
+ setInstanceVal(conf, SERVICE, DOT_USERNAME, USER);
+ setInstanceVal(conf, SERVICE, DOT_PASSWORD, PASS);
+ }
+
+ private void setInstanceVal(Configuration conf,
+ String host,
+ String key,
+ String val) {
+ String instance = RestClientBindings.buildSwiftInstancePrefix(host);
+ String confkey = instance
+ + key;
+ conf.set(confkey, val);
+ }
+
+ public void testPrefixBuilder() throws Throwable {
+ String built = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+ assertEquals("fs.swift.service." + SERVICE, built);
+ }
+
+ public void testBindAgainstConf() throws Exception {
+ Properties props = RestClientBindings.bind(filesysURI, conf);
+ assertPropertyEquals(props, SWIFT_CONTAINER_PROPERTY, CONTAINER);
+ assertPropertyEquals(props, SWIFT_SERVICE_PROPERTY, SERVICE);
+ assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+ assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+ assertPropertyEquals(props, SWIFT_USERNAME_PROPERTY, USER);
+ assertPropertyEquals(props, SWIFT_PASSWORD_PROPERTY, PASS);
+
+ assertPropertyEquals(props, SWIFT_TENANT_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_REGION_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_HTTP_PORT_PROPERTY, null);
+ assertPropertyEquals(props, SWIFT_HTTPS_PORT_PROPERTY, null);
+ }
+
+ public void expectBindingFailure(URI fsURI, Configuration config) {
+ try {
+ Properties binding = RestClientBindings.bind(fsURI, config);
+ //if we get here, binding didn't fail- there is something else.
+ //list the properties but not the values.
+ StringBuilder details = new StringBuilder() ;
+ for (Object key: binding.keySet()) {
+ details.append(key.toString()).append(" ");
+ }
+ fail("Expected a failure, got the binding [ "+ details+"]");
+ } catch (SwiftConfigurationException expected) {
+
+ }
+ }
+
+ public void testBindAgainstConfMissingInstance() throws Exception {
+ Configuration badConf = new Configuration();
+ expectBindingFailure(filesysURI, badConf);
+ }
+
+
+/* Hadoop 2.x+ only, as conf.unset() isn't a v1 feature
+ public void testBindAgainstConfIncompleteInstance() throws Exception {
+ String instance = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+ conf.unset(instance + DOT_PASSWORD);
+ expectBindingFailure(filesysURI, conf);
+ }
+*/
+
+ @Test(expected = SwiftConfigurationException.class)
+ public void testDottedServiceURL() throws Exception {
+ RestClientBindings.bind(new URI("swift://hadoop.apache.org/"), conf);
+ }
+
+ @Test(expected = SwiftConfigurationException.class)
+ public void testMissingServiceURL() throws Exception {
+ RestClientBindings.bind(new URI("swift:///"), conf);
+ }
+
+ /**
+ * inner test method that expects container extraction to fail
+ * -if not prints a meaningful error message.
+ *
+ * @param hostname hostname to parse
+ */
+ private static void expectExtractContainerFail(String hostname) {
+ try {
+ String container = RestClientBindings.extractContainerName(hostname);
+ fail("Expected an error -got a container of '" + container
+ + "' from " + hostname);
+ } catch (SwiftConfigurationException expected) {
+ //expected
+ }
+ }
+
+ /**
+ * inner test method that expects service extraction to fail
+ * -if not prints a meaningful error message.
+ *
+ * @param hostname hostname to parse
+ */
+ public static void expectExtractServiceFail(String hostname) {
+ try {
+ String service = RestClientBindings.extractServiceName(hostname);
+ fail("Expected an error -got a service of '" + service
+ + "' from " + hostname);
+ } catch (SwiftConfigurationException expected) {
+ //expected
+ }
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testEmptyHostname() throws Throwable {
+ expectExtractContainerFail("");
+ expectExtractServiceFail("");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testDot() throws Throwable {
+ expectExtractContainerFail(".");
+ expectExtractServiceFail(".");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testSimple() throws Throwable {
+ expectExtractContainerFail("simple");
+ expectExtractServiceFail("simple");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testTrailingDot() throws Throwable {
+ expectExtractServiceFail("simple.");
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testLeadingDot() throws Throwable {
+ expectExtractServiceFail(".leading");
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java
new file mode 100644
index 00000000000..f24a194cb1c
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+
+public class TestSwiftRestClient implements SwiftTestConstants {
+ private static final Log LOG =
+ LogFactory.getLog(TestSwiftRestClient.class);
+
+ private Configuration conf;
+ private boolean runTests;
+ private URI serviceURI;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+ runTests = SwiftTestUtils.hasServiceURI(conf);
+ if (runTests) {
+ serviceURI = SwiftTestUtils.getServiceURI(conf);
+ }
+ }
+
+ protected void assumeEnabled() {
+ Assume.assumeTrue(runTests);
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testCreate() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ }
+
+ private SwiftRestClient createClient() throws IOException {
+ return SwiftRestClient.getInstance(serviceURI, conf);
+ }
+
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testAuthenticate() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ client.authenticate();
+ }
+
+ @Test(timeout = SWIFT_TEST_TIMEOUT)
+ public void testPutAndDelete() throws Throwable {
+ assumeEnabled();
+ SwiftRestClient client = createClient();
+ client.authenticate();
+ Path path = new Path("restTestPutAndDelete");
+ SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path);
+ byte[] stuff = new byte[1];
+ stuff[0] = 'a';
+ client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length);
+ //check file exists
+ Duration head = new Duration();
+ Header[] responseHeaders = client.headRequest("expect success",
+ sobject,
+ SwiftRestClient.NEWEST);
+ head.finished();
+ LOG.info("head request duration " + head);
+ for (Header header: responseHeaders) {
+ LOG.info(header.toString());
+ }
+ //delete the file
+ client.delete(sobject);
+ //check file is gone
+ try {
+ Header[] headers = client.headRequest("expect fail",
+ sobject,
+ SwiftRestClient.NEWEST);
+ Assert.fail("Expected deleted file, but object is still present: "
+ + sobject);
+ } catch (FileNotFoundException e) {
+ //expected
+ }
+ for (DurationStats stats: client.getOperationStatistics()) {
+ LOG.info(stats);
+ }
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java
new file mode 100644
index 00000000000..314e7a1dfb8
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+
+/**
+ * Base class for scale tests; here is where the common scale configuration
+ * keys are defined
+ */
+
+public class SwiftScaleTestBase extends SwiftFileSystemBaseTest {
+
+ public static final String SCALE_TEST = "scale.test.";
+ public static final String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
+ public static final long DEFAULT_OPERATION_COUNT = 10;
+
+ protected long getOperationCount() {
+ return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT);
+ }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java
new file mode 100644
index 00000000000..8694f6ca482
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+public class TestWriteManySmallFiles extends SwiftScaleTestBase {
+
+ public static final Log LOG = LogFactory.getLog(TestWriteManySmallFiles.class);
+
+ @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+ public void testScaledWriteThenRead() throws Throwable {
+ Path dir = new Path("/test/manysmallfiles");
+ Duration rm1 = new Duration();
+ fs.delete(dir, true);
+ rm1.finished();
+ fs.mkdirs(dir);
+ Duration ls1 = new Duration();
+ fs.listStatus(dir);
+ ls1.finished();
+ long count = getOperationCount();
+ SwiftTestUtils.noteAction("Beginning Write of "+ count + " files ");
+ DurationStats writeStats = new DurationStats("write");
+ DurationStats readStats = new DurationStats("read");
+ String format = "%08d";
+ for (long l = 0; l < count; l++) {
+ String name = String.format(format, l);
+ Path p = new Path(dir, "part-" + name);
+ Duration d = new Duration();
+ SwiftTestUtils.writeTextFile(fs, p, name, false);
+ d.finished();
+ writeStats.add(d);
+ Thread.sleep(1000);
+ }
+ //at this point, the directory is full.
+ SwiftTestUtils.noteAction("Beginning ls");
+
+ Duration ls2 = new Duration();
+ FileStatus[] status2 = (FileStatus[]) fs.listStatus(dir);
+ ls2.finished();
+ assertEquals("Not enough entries in the directory", count, status2.length);
+
+ SwiftTestUtils.noteAction("Beginning read");
+
+ for (long l = 0; l < count; l++) {
+ String name = String.format(format, l);
+ Path p = new Path(dir, "part-" + name);
+ Duration d = new Duration();
+ String result = SwiftTestUtils.readBytesToString(fs, p, name.length());
+ assertEquals(name, result);
+ d.finished();
+ readStats.add(d);
+ }
+ //do a recursive delete
+ SwiftTestUtils.noteAction("Beginning delete");
+ Duration rm2 = new Duration();
+ fs.delete(dir, true);
+ rm2.finished();
+ //print the stats
+ LOG.info(String.format("'filesystem','%s'",fs.getUri()));
+ LOG.info(writeStats.toString());
+ LOG.info(readStats.toString());
+ LOG.info(String.format(
+ "'rm1',%d,'ls1',%d",
+ rm1.value(),
+ ls1.value()));
+ LOG.info(String.format(
+ "'rm2',%d,'ls2',%d",
+ rm2.value(),
+ ls2.value()));
+ }
+
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml b/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml
new file mode 100644
index 00000000000..9252e885871
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml
@@ -0,0 +1,51 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ hadoop.tmp.dir
+ target/build/test
+ A base for other temporary directories.
+ true
+
+
+
+
+ hadoop.security.authentication
+ simple
+
+
+
+
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties b/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties
new file mode 100644
index 00000000000..6aeb41dcdd0
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n"
+#log4j.logger.org.apache.hadoop.fs.swift=DEBUG
+
+#crank back on warnings about -1 content length GETs
+log4j.logger.org.apache.commons.httpclient.HttpMethodBase=ERROR
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index cbd0bb545a2..061602bf910 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -77,6 +77,12 @@
pom
${project.version}