HDFS-8979. Clean up checkstyle warnings in hadoop-hdfs-client module. Contributed by Mingliang Liu.
This commit is contained in:
parent
21b4ba48ce
commit
1257483ebf
|
@ -34,7 +34,7 @@ public enum CacheFlag {
|
||||||
FORCE((short) 0x01);
|
FORCE((short) 0x01);
|
||||||
private final short mode;
|
private final short mode;
|
||||||
|
|
||||||
private CacheFlag(short mode) {
|
CacheFlag(short mode) {
|
||||||
this.mode = mode;
|
this.mode = mode;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,8 +17,6 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.fs;
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
|
@ -34,8 +32,7 @@ public class HdfsBlockLocation extends BlockLocation {
|
||||||
|
|
||||||
private final LocatedBlock block;
|
private final LocatedBlock block;
|
||||||
|
|
||||||
public HdfsBlockLocation(BlockLocation loc, LocatedBlock block)
|
public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) {
|
||||||
throws IOException {
|
|
||||||
// Initialize with data from passed in BlockLocation
|
// Initialize with data from passed in BlockLocation
|
||||||
super(loc);
|
super(loc);
|
||||||
this.block = block;
|
this.block = block;
|
||||||
|
|
|
@ -57,12 +57,12 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class XAttr {
|
public class XAttr {
|
||||||
|
|
||||||
public static enum NameSpace {
|
public enum NameSpace {
|
||||||
USER,
|
USER,
|
||||||
TRUSTED,
|
TRUSTED,
|
||||||
SECURITY,
|
SECURITY,
|
||||||
SYSTEM,
|
SYSTEM,
|
||||||
RAW;
|
RAW
|
||||||
}
|
}
|
||||||
|
|
||||||
private final NameSpace ns;
|
private final NameSpace ns;
|
||||||
|
|
|
@ -24,8 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This exception is thrown when a read encounters a block that has no locations
|
* This exception is thrown when a read encounters a block that has no
|
||||||
* associated with it.
|
* locations associated with it.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
|
@ -41,7 +41,8 @@ public class BlockMissingException extends IOException {
|
||||||
* @param filename name of corrupted file
|
* @param filename name of corrupted file
|
||||||
* @param description a description of the corruption details
|
* @param description a description of the corruption details
|
||||||
*/
|
*/
|
||||||
public BlockMissingException(String filename, String description, long offset) {
|
public BlockMissingException(String filename, String description,
|
||||||
|
long offset) {
|
||||||
super(description);
|
super(description);
|
||||||
this.filename = filename;
|
this.filename = filename;
|
||||||
this.offset = offset;
|
this.offset = offset;
|
||||||
|
|
|
@ -32,7 +32,6 @@ import org.apache.hadoop.hdfs.shortcircuit.ClientMmap;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public interface BlockReader extends ByteBufferReadable {
|
public interface BlockReader extends ByteBufferReadable {
|
||||||
|
|
||||||
|
|
||||||
/* same interface as inputStream java.io.InputStream#read()
|
/* same interface as inputStream java.io.InputStream#read()
|
||||||
* used by DFSInputStream#read()
|
* used by DFSInputStream#read()
|
||||||
* This violates one rule when there is a checksum error:
|
* This violates one rule when there is a checksum error:
|
||||||
|
@ -55,7 +54,7 @@ public interface BlockReader extends ByteBufferReadable {
|
||||||
* network I/O.
|
* network I/O.
|
||||||
* This may return more than what is actually present in the block.
|
* This may return more than what is actually present in the block.
|
||||||
*/
|
*/
|
||||||
int available() throws IOException;
|
int available();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Close the block reader.
|
* Close the block reader.
|
||||||
|
|
|
@ -170,7 +170,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
private RemotePeerFactory remotePeerFactory;
|
private RemotePeerFactory remotePeerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* UserGroupInformation to use for legacy block reader local objects, if needed.
|
* UserGroupInformation to use for legacy block reader local objects,
|
||||||
|
* if needed.
|
||||||
*/
|
*/
|
||||||
private UserGroupInformation userGroupInformation;
|
private UserGroupInformation userGroupInformation;
|
||||||
|
|
||||||
|
@ -426,9 +427,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (clientContext.getDisableLegacyBlockReaderLocal()) {
|
if (clientContext.getDisableLegacyBlockReaderLocal()) {
|
||||||
PerformanceAdvisory.LOG.debug("{}: can't construct " +
|
PerformanceAdvisory.LOG.debug("{}: can't construct " +
|
||||||
"BlockReaderLocalLegacy because " +
|
"BlockReaderLocalLegacy because " +
|
||||||
"disableLegacyBlockReaderLocal is set.", this);
|
"disableLegacyBlockReaderLocal is set.", this);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
IOException ioe;
|
IOException ioe;
|
||||||
|
@ -470,7 +471,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
ShortCircuitCache cache = clientContext.getShortCircuitCache();
|
ShortCircuitCache cache = clientContext.getShortCircuitCache();
|
||||||
ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
|
ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(),
|
||||||
|
block.getBlockPoolId());
|
||||||
ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
|
ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
|
||||||
InvalidToken exc = info.getInvalidTokenException();
|
InvalidToken exc = info.getInvalidTokenException();
|
||||||
if (exc != null) {
|
if (exc != null) {
|
||||||
|
@ -501,14 +503,15 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
*
|
*
|
||||||
* @return Null if we could not communicate with the datanode,
|
* @return Null if we could not communicate with the datanode,
|
||||||
* a new ShortCircuitReplicaInfo object otherwise.
|
* a new ShortCircuitReplicaInfo object otherwise.
|
||||||
* ShortCircuitReplicaInfo objects may contain either an InvalidToken
|
* ShortCircuitReplicaInfo objects may contain either an
|
||||||
* exception, or a ShortCircuitReplica object ready to use.
|
* InvalidToken exception, or a ShortCircuitReplica object ready to
|
||||||
|
* use.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
|
public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
|
||||||
if (createShortCircuitReplicaInfoCallback != null) {
|
if (createShortCircuitReplicaInfoCallback != null) {
|
||||||
ShortCircuitReplicaInfo info =
|
ShortCircuitReplicaInfo info =
|
||||||
createShortCircuitReplicaInfoCallback.createShortCircuitReplicaInfo();
|
createShortCircuitReplicaInfoCallback.createShortCircuitReplicaInfo();
|
||||||
if (info != null) return info;
|
if (info != null) return info;
|
||||||
}
|
}
|
||||||
LOG.trace("{}: trying to create ShortCircuitReplicaInfo.", this);
|
LOG.trace("{}: trying to create ShortCircuitReplicaInfo.", this);
|
||||||
|
@ -682,7 +685,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
}
|
}
|
||||||
if (curPeer.fromCache) {
|
if (curPeer.fromCache) {
|
||||||
// Handle an I/O error we got when using a cached peer. These are
|
// Handle an I/O error we got when using a cached peer. These are
|
||||||
// considered less serious, because the underlying socket may be stale.
|
// considered less serious because the underlying socket may be stale.
|
||||||
LOG.debug("Closed potentially stale domain peer {}", peer, ioe);
|
LOG.debug("Closed potentially stale domain peer {}", peer, ioe);
|
||||||
} else {
|
} else {
|
||||||
// Handle an I/O error we got when using a newly created domain peer.
|
// Handle an I/O error we got when using a newly created domain peer.
|
||||||
|
@ -800,7 +803,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
Peer peer = remotePeerFactory.newConnectedPeer(inetSocketAddress, token,
|
Peer peer = remotePeerFactory.newConnectedPeer(inetSocketAddress, token,
|
||||||
datanode);
|
datanode);
|
||||||
LOG.trace("nextTcpPeer: created newConnectedPeer {}", peer);
|
LOG.trace("nextTcpPeer: created newConnectedPeer {}", peer);
|
||||||
return new BlockReaderPeer(peer, false);
|
return new BlockReaderPeer(peer, false);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
|
|
@ -335,9 +335,8 @@ class BlockReaderLocal implements BlockReader {
|
||||||
*/
|
*/
|
||||||
private synchronized int fillBuffer(ByteBuffer buf, boolean canSkipChecksum)
|
private synchronized int fillBuffer(ByteBuffer buf, boolean canSkipChecksum)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = tracer.newScope(
|
try (TraceScope ignored = tracer.newScope(
|
||||||
"BlockReaderLocal#fillBuffer(" + block.getBlockId() + ")");
|
"BlockReaderLocal#fillBuffer(" + block.getBlockId() + ")")) {
|
||||||
try {
|
|
||||||
int total = 0;
|
int total = 0;
|
||||||
long startDataPos = dataPos;
|
long startDataPos = dataPos;
|
||||||
int startBufPos = buf.position();
|
int startBufPos = buf.position();
|
||||||
|
@ -358,7 +357,8 @@ class BlockReaderLocal implements BlockReader {
|
||||||
buf.limit(buf.position());
|
buf.limit(buf.position());
|
||||||
buf.position(startBufPos);
|
buf.position(startBufPos);
|
||||||
createChecksumBufIfNeeded();
|
createChecksumBufIfNeeded();
|
||||||
int checksumsNeeded = (total + bytesPerChecksum - 1) / bytesPerChecksum;
|
int checksumsNeeded = (total + bytesPerChecksum - 1) /
|
||||||
|
bytesPerChecksum;
|
||||||
checksumBuf.clear();
|
checksumBuf.clear();
|
||||||
checksumBuf.limit(checksumsNeeded * checksumSize);
|
checksumBuf.limit(checksumsNeeded * checksumSize);
|
||||||
long checksumPos = BlockMetadataHeader.getHeaderSize()
|
long checksumPos = BlockMetadataHeader.getHeaderSize()
|
||||||
|
@ -367,8 +367,8 @@ class BlockReaderLocal implements BlockReader {
|
||||||
int nRead = checksumIn.read(checksumBuf, checksumPos);
|
int nRead = checksumIn.read(checksumBuf, checksumPos);
|
||||||
if (nRead < 0) {
|
if (nRead < 0) {
|
||||||
throw new IOException("Got unexpected checksum file EOF at " +
|
throw new IOException("Got unexpected checksum file EOF at " +
|
||||||
checksumPos + ", block file position " + startDataPos + " for " +
|
checksumPos + ", block file position " + startDataPos +
|
||||||
"block " + block + " of file " + filename);
|
" for block " + block + " of file " + filename);
|
||||||
}
|
}
|
||||||
checksumPos += nRead;
|
checksumPos += nRead;
|
||||||
}
|
}
|
||||||
|
@ -380,24 +380,16 @@ class BlockReaderLocal implements BlockReader {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return total;
|
return total;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean createNoChecksumContext() {
|
private boolean createNoChecksumContext() {
|
||||||
if (verifyChecksum) {
|
return !verifyChecksum ||
|
||||||
if (storageType != null && storageType.isTransient()) {
|
// Checksums are not stored for replicas on transient storage. We do
|
||||||
// Checksums are not stored for replicas on transient storage. We do not
|
// not anchor, because we do not intend for client activity to block
|
||||||
// anchor, because we do not intend for client activity to block eviction
|
// eviction from transient storage on the DataNode side.
|
||||||
// from transient storage on the DataNode side.
|
(storageType != null && storageType.isTransient()) ||
|
||||||
return true;
|
replica.addNoChecksumAnchor();
|
||||||
} else {
|
|
||||||
return replica.addNoChecksumAnchor();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void releaseNoChecksumContext() {
|
private void releaseNoChecksumContext() {
|
||||||
|
@ -473,11 +465,11 @@ class BlockReaderLocal implements BlockReader {
|
||||||
dataBuf.limit(maxReadaheadLength);
|
dataBuf.limit(maxReadaheadLength);
|
||||||
if (canSkipChecksum) {
|
if (canSkipChecksum) {
|
||||||
dataBuf.position(slop);
|
dataBuf.position(slop);
|
||||||
fillBuffer(dataBuf, canSkipChecksum);
|
fillBuffer(dataBuf, true);
|
||||||
} else {
|
} else {
|
||||||
dataPos -= slop;
|
dataPos -= slop;
|
||||||
dataBuf.position(0);
|
dataBuf.position(0);
|
||||||
fillBuffer(dataBuf, canSkipChecksum);
|
fillBuffer(dataBuf, false);
|
||||||
}
|
}
|
||||||
dataBuf.limit(dataBuf.position());
|
dataBuf.limit(dataBuf.position());
|
||||||
dataBuf.position(Math.min(dataBuf.position(), slop));
|
dataBuf.position(Math.min(dataBuf.position(), slop));
|
||||||
|
@ -621,7 +613,7 @@ class BlockReaderLocal implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int available() throws IOException {
|
public int available() {
|
||||||
// We never do network I/O in BlockReaderLocal.
|
// We never do network I/O in BlockReaderLocal.
|
||||||
return Integer.MAX_VALUE;
|
return Integer.MAX_VALUE;
|
||||||
}
|
}
|
||||||
|
@ -677,7 +669,7 @@ class BlockReaderLocal implements BlockReader {
|
||||||
@Override
|
@Override
|
||||||
public ClientMmap getClientMmap(EnumSet<ReadOption> opts) {
|
public ClientMmap getClientMmap(EnumSet<ReadOption> opts) {
|
||||||
boolean anchor = verifyChecksum &&
|
boolean anchor = verifyChecksum &&
|
||||||
(opts.contains(ReadOption.SKIP_CHECKSUMS) == false);
|
!opts.contains(ReadOption.SKIP_CHECKSUMS);
|
||||||
if (anchor) {
|
if (anchor) {
|
||||||
if (!createNoChecksumContext()) {
|
if (!createNoChecksumContext()) {
|
||||||
LOG.trace("can't get an mmap for {} of {} since SKIP_CHECKSUMS was not "
|
LOG.trace("can't get an mmap for {} of {} since SKIP_CHECKSUMS was not "
|
||||||
|
|
|
@ -56,10 +56,10 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* BlockReaderLocalLegacy enables local short circuited reads. If the DFS client is on
|
* BlockReaderLocalLegacy enables local short circuited reads. If the DFS client
|
||||||
* the same machine as the datanode, then the client can read files directly
|
* is on the same machine as the datanode, then the client can read files
|
||||||
* from the local file system rather than going through the datanode for better
|
* directly from the local file system rather than going through the datanode
|
||||||
* performance. <br>
|
* for better performance. <br>
|
||||||
*
|
*
|
||||||
* This is the legacy implementation based on HDFS-2246, which requires
|
* This is the legacy implementation based on HDFS-2246, which requires
|
||||||
* permissions on the datanode to be set so that clients can directly access the
|
* permissions on the datanode to be set so that clients can directly access the
|
||||||
|
@ -90,7 +90,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
LocalDatanodeInfo() {
|
LocalDatanodeInfo() {
|
||||||
final int cacheSize = 10000;
|
final int cacheSize = 10000;
|
||||||
final float hashTableLoadFactor = 0.75f;
|
final float hashTableLoadFactor = 0.75f;
|
||||||
int hashTableCapacity = (int) Math.ceil(cacheSize / hashTableLoadFactor) + 1;
|
int hashTableCapacity = (int) Math.ceil(cacheSize / hashTableLoadFactor)
|
||||||
|
+ 1;
|
||||||
cache = Collections
|
cache = Collections
|
||||||
.synchronizedMap(new LinkedHashMap<ExtendedBlock, BlockLocalPathInfo>(
|
.synchronizedMap(new LinkedHashMap<ExtendedBlock, BlockLocalPathInfo>(
|
||||||
hashTableCapacity, hashTableLoadFactor, true) {
|
hashTableCapacity, hashTableLoadFactor, true) {
|
||||||
|
@ -135,7 +136,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
return cache.get(b);
|
return cache.get(b);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setBlockLocalPathInfo(ExtendedBlock b, BlockLocalPathInfo info) {
|
private void setBlockLocalPathInfo(ExtendedBlock b,
|
||||||
|
BlockLocalPathInfo info) {
|
||||||
cache.put(b, info);
|
cache.put(b, info);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,7 +148,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
|
|
||||||
// Multiple datanodes could be running on the local machine. Store proxies in
|
// Multiple datanodes could be running on the local machine. Store proxies in
|
||||||
// a map keyed by the ipc port of the datanode.
|
// a map keyed by the ipc port of the datanode.
|
||||||
private static final Map<Integer, LocalDatanodeInfo> localDatanodeInfoMap = new HashMap<Integer, LocalDatanodeInfo>();
|
private static final Map<Integer, LocalDatanodeInfo> localDatanodeInfoMap =
|
||||||
|
new HashMap<>();
|
||||||
|
|
||||||
private final FileInputStream dataIn; // reader for the data file
|
private final FileInputStream dataIn; // reader for the data file
|
||||||
private final FileInputStream checksumIn; // reader for the checksum file
|
private final FileInputStream checksumIn; // reader for the checksum file
|
||||||
|
@ -234,12 +237,12 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
new DataInputStream(checksumIn), blk);
|
new DataInputStream(checksumIn), blk);
|
||||||
long firstChunkOffset = startOffset
|
long firstChunkOffset = startOffset
|
||||||
- (startOffset % checksum.getBytesPerChecksum());
|
- (startOffset % checksum.getBytesPerChecksum());
|
||||||
localBlockReader = new BlockReaderLocalLegacy(scConf, file, blk, token,
|
localBlockReader = new BlockReaderLocalLegacy(scConf, file, blk,
|
||||||
startOffset, length, pathinfo, checksum, true, dataIn,
|
startOffset, checksum, true, dataIn, firstChunkOffset, checksumIn,
|
||||||
firstChunkOffset, checksumIn, tracer);
|
tracer);
|
||||||
} else {
|
} else {
|
||||||
localBlockReader = new BlockReaderLocalLegacy(scConf, file, blk, token,
|
localBlockReader = new BlockReaderLocalLegacy(scConf, file, blk,
|
||||||
startOffset, length, pathinfo, dataIn, tracer);
|
startOffset, dataIn, tracer);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
// remove from cache
|
// remove from cache
|
||||||
|
@ -274,14 +277,15 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout,
|
ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout,
|
||||||
Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname,
|
Token<BlockTokenIdentifier> token, boolean connectToDnViaHostname,
|
||||||
StorageType storageType) throws IOException {
|
StorageType storageType) throws IOException {
|
||||||
LocalDatanodeInfo localDatanodeInfo = getLocalDatanodeInfo(node.getIpcPort());
|
LocalDatanodeInfo localDatanodeInfo =
|
||||||
BlockLocalPathInfo pathinfo = null;
|
getLocalDatanodeInfo(node.getIpcPort());
|
||||||
|
BlockLocalPathInfo pathinfo;
|
||||||
ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(ugi, node,
|
ClientDatanodeProtocol proxy = localDatanodeInfo.getDatanodeProxy(ugi, node,
|
||||||
conf, timeout, connectToDnViaHostname);
|
conf, timeout, connectToDnViaHostname);
|
||||||
try {
|
try {
|
||||||
// make RPC to local datanode to find local pathnames of blocks
|
// make RPC to local datanode to find local pathnames of blocks
|
||||||
pathinfo = proxy.getBlockLocalPathInfo(blk, token);
|
pathinfo = proxy.getBlockLocalPathInfo(blk, token);
|
||||||
// We cannot cache the path information for a replica on transient storage.
|
// We can't cache the path information for a replica on transient storage.
|
||||||
// If the replica gets evicted, then it moves to a different path. Then,
|
// If the replica gets evicted, then it moves to a different path. Then,
|
||||||
// our next attempt to read from the cached path would fail to find the
|
// our next attempt to read from the cached path would fail to find the
|
||||||
// file. Additionally, the failure would cause us to disable legacy
|
// file. Additionally, the failure would cause us to disable legacy
|
||||||
|
@ -315,17 +319,15 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
private BlockReaderLocalLegacy(ShortCircuitConf conf, String hdfsfile,
|
private BlockReaderLocalLegacy(ShortCircuitConf conf, String hdfsfile,
|
||||||
ExtendedBlock block, Token<BlockTokenIdentifier> token, long startOffset,
|
ExtendedBlock block, long startOffset, FileInputStream dataIn,
|
||||||
long length, BlockLocalPathInfo pathinfo, FileInputStream dataIn,
|
|
||||||
Tracer tracer) throws IOException {
|
Tracer tracer) throws IOException {
|
||||||
this(conf, hdfsfile, block, token, startOffset, length, pathinfo,
|
this(conf, hdfsfile, block, startOffset,
|
||||||
DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 4), false,
|
DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 4), false,
|
||||||
dataIn, startOffset, null, tracer);
|
dataIn, startOffset, null, tracer);
|
||||||
}
|
}
|
||||||
|
|
||||||
private BlockReaderLocalLegacy(ShortCircuitConf conf, String hdfsfile,
|
private BlockReaderLocalLegacy(ShortCircuitConf conf, String hdfsfile,
|
||||||
ExtendedBlock block, Token<BlockTokenIdentifier> token, long startOffset,
|
ExtendedBlock block, long startOffset, DataChecksum checksum,
|
||||||
long length, BlockLocalPathInfo pathinfo, DataChecksum checksum,
|
|
||||||
boolean verifyChecksum, FileInputStream dataIn, long firstChunkOffset,
|
boolean verifyChecksum, FileInputStream dataIn, long firstChunkOffset,
|
||||||
FileInputStream checksumIn, Tracer tracer) throws IOException {
|
FileInputStream checksumIn, Tracer tracer) throws IOException {
|
||||||
this.filename = hdfsfile;
|
this.filename = hdfsfile;
|
||||||
|
@ -343,17 +345,20 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
|
|
||||||
final int chunksPerChecksumRead = getSlowReadBufferNumChunks(
|
final int chunksPerChecksumRead = getSlowReadBufferNumChunks(
|
||||||
conf.getShortCircuitBufferSize(), bytesPerChecksum);
|
conf.getShortCircuitBufferSize(), bytesPerChecksum);
|
||||||
slowReadBuff = bufferPool.getBuffer(bytesPerChecksum * chunksPerChecksumRead);
|
slowReadBuff = bufferPool.getBuffer(
|
||||||
|
bytesPerChecksum * chunksPerChecksumRead);
|
||||||
checksumBuff = bufferPool.getBuffer(checksumSize * chunksPerChecksumRead);
|
checksumBuff = bufferPool.getBuffer(checksumSize * chunksPerChecksumRead);
|
||||||
// Initially the buffers have nothing to read.
|
// Initially the buffers have nothing to read.
|
||||||
slowReadBuff.flip();
|
slowReadBuff.flip();
|
||||||
checksumBuff.flip();
|
checksumBuff.flip();
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
try {
|
try {
|
||||||
// Skip both input streams to beginning of the chunk containing startOffset
|
// Skip both input streams to beginning of the chunk containing
|
||||||
|
// startOffset
|
||||||
IOUtils.skipFully(dataIn, firstChunkOffset);
|
IOUtils.skipFully(dataIn, firstChunkOffset);
|
||||||
if (checksumIn != null) {
|
if (checksumIn != null) {
|
||||||
long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize;
|
long checkSumOffset = (firstChunkOffset / bytesPerChecksum) *
|
||||||
|
checksumSize;
|
||||||
IOUtils.skipFully(checksumIn, checkSumOffset);
|
IOUtils.skipFully(checksumIn, checkSumOffset);
|
||||||
}
|
}
|
||||||
success = true;
|
success = true;
|
||||||
|
@ -371,9 +376,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
*/
|
*/
|
||||||
private int fillBuffer(FileInputStream stream, ByteBuffer buf)
|
private int fillBuffer(FileInputStream stream, ByteBuffer buf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = tracer.
|
try (TraceScope ignored = tracer.
|
||||||
newScope("BlockReaderLocalLegacy#fillBuffer(" + blockId + ")");
|
newScope("BlockReaderLocalLegacy#fillBuffer(" + blockId + ")")) {
|
||||||
try {
|
|
||||||
int bytesRead = stream.getChannel().read(buf);
|
int bytesRead = stream.getChannel().read(buf);
|
||||||
if (bytesRead < 0) {
|
if (bytesRead < 0) {
|
||||||
//EOF
|
//EOF
|
||||||
|
@ -388,8 +392,6 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
bytesRead += n;
|
bytesRead += n;
|
||||||
}
|
}
|
||||||
return bytesRead;
|
return bytesRead;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -426,7 +428,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
if (slowReadBuff.hasRemaining()) {
|
if (slowReadBuff.hasRemaining()) {
|
||||||
// There are remaining bytes from a small read available. This usually
|
// There are remaining bytes from a small read available. This usually
|
||||||
// means this read is unaligned, which falls back to the slow path.
|
// means this read is unaligned, which falls back to the slow path.
|
||||||
int fromSlowReadBuff = Math.min(buf.remaining(), slowReadBuff.remaining());
|
int fromSlowReadBuff = Math.min(buf.remaining(),
|
||||||
|
slowReadBuff.remaining());
|
||||||
writeSlice(slowReadBuff, buf, fromSlowReadBuff);
|
writeSlice(slowReadBuff, buf, fromSlowReadBuff);
|
||||||
nRead += fromSlowReadBuff;
|
nRead += fromSlowReadBuff;
|
||||||
}
|
}
|
||||||
|
@ -458,8 +461,10 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
|
|
||||||
// offsetFromChunkBoundary > 0 => unaligned read, use slow path to read
|
// offsetFromChunkBoundary > 0 => unaligned read, use slow path to read
|
||||||
// until chunk boundary
|
// until chunk boundary
|
||||||
if ((buf.remaining() > 0 && buf.remaining() < bytesPerChecksum) || offsetFromChunkBoundary > 0) {
|
if ((buf.remaining() > 0 && buf.remaining() < bytesPerChecksum) ||
|
||||||
int toRead = Math.min(buf.remaining(), bytesPerChecksum - offsetFromChunkBoundary);
|
offsetFromChunkBoundary > 0) {
|
||||||
|
int toRead = Math.min(buf.remaining(),
|
||||||
|
bytesPerChecksum - offsetFromChunkBoundary);
|
||||||
int readResult = fillSlowReadBuffer(toRead);
|
int readResult = fillSlowReadBuffer(toRead);
|
||||||
if (readResult == -1) {
|
if (readResult == -1) {
|
||||||
return nRead;
|
return nRead;
|
||||||
|
@ -470,7 +475,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Non-checksummed reads are much easier; we can just fill the buffer directly.
|
// Non-checksummed reads are much easier; we can just fill the buffer
|
||||||
|
// directly.
|
||||||
nRead = doByteBufferRead(buf);
|
nRead = doByteBufferRead(buf);
|
||||||
if (nRead > 0) {
|
if (nRead > 0) {
|
||||||
buf.position(buf.position() + nRead);
|
buf.position(buf.position() + nRead);
|
||||||
|
@ -512,7 +518,7 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
if (verifyChecksum) {
|
if (verifyChecksum) {
|
||||||
assert buf.remaining() % bytesPerChecksum == 0;
|
assert buf.remaining() % bytesPerChecksum == 0;
|
||||||
}
|
}
|
||||||
int dataRead = -1;
|
int dataRead;
|
||||||
|
|
||||||
int oldpos = buf.position();
|
int oldpos = buf.position();
|
||||||
// Read as much as we can into the buffer.
|
// Read as much as we can into the buffer.
|
||||||
|
@ -528,9 +534,10 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
toChecksum.limit(oldpos + dataRead);
|
toChecksum.limit(oldpos + dataRead);
|
||||||
|
|
||||||
checksumBuff.clear();
|
checksumBuff.clear();
|
||||||
// Equivalent to (int)Math.ceil(toChecksum.remaining() * 1.0 / bytesPerChecksum );
|
// Equivalent to
|
||||||
|
// (int)Math.ceil(toChecksum.remaining() * 1.0 / bytesPerChecksum );
|
||||||
int numChunks =
|
int numChunks =
|
||||||
(toChecksum.remaining() + bytesPerChecksum - 1) / bytesPerChecksum;
|
(toChecksum.remaining() + bytesPerChecksum - 1) / bytesPerChecksum;
|
||||||
checksumBuff.limit(checksumSize * numChunks);
|
checksumBuff.limit(checksumSize * numChunks);
|
||||||
|
|
||||||
fillBuffer(checksumIn, checksumBuff);
|
fillBuffer(checksumIn, checksumBuff);
|
||||||
|
@ -571,7 +578,7 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
* @return the number of bytes available to read, or -1 if EOF.
|
* @return the number of bytes available to read, or -1 if EOF.
|
||||||
*/
|
*/
|
||||||
private synchronized int fillSlowReadBuffer(int len) throws IOException {
|
private synchronized int fillSlowReadBuffer(int len) throws IOException {
|
||||||
int nRead = -1;
|
int nRead;
|
||||||
if (slowReadBuff.hasRemaining()) {
|
if (slowReadBuff.hasRemaining()) {
|
||||||
// Already got data, good to go.
|
// Already got data, good to go.
|
||||||
nRead = Math.min(len, slowReadBuff.remaining());
|
nRead = Math.min(len, slowReadBuff.remaining());
|
||||||
|
@ -579,7 +586,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
// Round a complete read of len bytes (plus any implicit offset) to the
|
// Round a complete read of len bytes (plus any implicit offset) to the
|
||||||
// next chunk boundary, since we try and read in multiples of a chunk
|
// next chunk boundary, since we try and read in multiples of a chunk
|
||||||
int nextChunk = len + offsetFromChunkBoundary +
|
int nextChunk = len + offsetFromChunkBoundary +
|
||||||
(bytesPerChecksum - ((len + offsetFromChunkBoundary) % bytesPerChecksum));
|
(bytesPerChecksum -
|
||||||
|
((len + offsetFromChunkBoundary) % bytesPerChecksum));
|
||||||
int limit = Math.min(nextChunk, slowReadBuff.capacity());
|
int limit = Math.min(nextChunk, slowReadBuff.capacity());
|
||||||
assert limit % bytesPerChecksum == 0;
|
assert limit % bytesPerChecksum == 0;
|
||||||
|
|
||||||
|
@ -598,7 +606,8 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(byte[] buf, int off, int len) throws IOException {
|
public synchronized int read(byte[] buf, int off, int len)
|
||||||
|
throws IOException {
|
||||||
LOG.trace("read off {} len {}", off, len);
|
LOG.trace("read off {} len {}", off, len);
|
||||||
if (!verifyChecksum) {
|
if (!verifyChecksum) {
|
||||||
return dataIn.read(buf, off, len);
|
return dataIn.read(buf, off, len);
|
||||||
|
@ -708,7 +717,7 @@ class BlockReaderLocalLegacy implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int available() throws IOException {
|
public int available() {
|
||||||
// We never do network I/O in BlockReaderLocalLegacy.
|
// We never do network I/O in BlockReaderLocalLegacy.
|
||||||
return Integer.MAX_VALUE;
|
return Integer.MAX_VALUE;
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,13 +41,13 @@ import org.slf4j.LoggerFactory;
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class ClientContext {
|
public class ClientContext {
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(ClientContext.class);
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
|
ClientContext.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Global map of context names to caches contexts.
|
* Global map of context names to caches contexts.
|
||||||
*/
|
*/
|
||||||
private final static HashMap<String, ClientContext> CACHES =
|
private final static HashMap<String, ClientContext> CACHES = new HashMap<>();
|
||||||
new HashMap<String, ClientContext>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Name of context.
|
* Name of context.
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -69,8 +69,8 @@ public class DFSInotifyEventInputStream {
|
||||||
this(namenode, tracer, namenode.getCurrentEditLogTxid());
|
this(namenode, tracer, namenode.getCurrentEditLogTxid());
|
||||||
}
|
}
|
||||||
|
|
||||||
DFSInotifyEventInputStream(ClientProtocol namenode,
|
DFSInotifyEventInputStream(ClientProtocol namenode, Tracer tracer,
|
||||||
Tracer tracer, long lastReadTxid) throws IOException {
|
long lastReadTxid) {
|
||||||
this.namenode = namenode;
|
this.namenode = namenode;
|
||||||
this.it = Iterators.emptyIterator();
|
this.it = Iterators.emptyIterator();
|
||||||
this.lastReadTxid = lastReadTxid;
|
this.lastReadTxid = lastReadTxid;
|
||||||
|
@ -94,8 +94,7 @@ public class DFSInotifyEventInputStream {
|
||||||
* The next available batch of events will be returned.
|
* The next available batch of events will be returned.
|
||||||
*/
|
*/
|
||||||
public EventBatch poll() throws IOException, MissingEventsException {
|
public EventBatch poll() throws IOException, MissingEventsException {
|
||||||
TraceScope scope = tracer.newScope("inotifyPoll");
|
try (TraceScope ignored = tracer.newScope("inotifyPoll")) {
|
||||||
try {
|
|
||||||
// need to keep retrying until the NN sends us the latest committed txid
|
// need to keep retrying until the NN sends us the latest committed txid
|
||||||
if (lastReadTxid == -1) {
|
if (lastReadTxid == -1) {
|
||||||
LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
|
LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
|
||||||
|
@ -119,7 +118,7 @@ public class DFSInotifyEventInputStream {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG.debug("poll(): read no edits from the NN when requesting edits " +
|
LOG.debug("poll(): read no edits from the NN when requesting edits " +
|
||||||
"after txid {}", lastReadTxid);
|
"after txid {}", lastReadTxid);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -130,8 +129,6 @@ public class DFSInotifyEventInputStream {
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -175,9 +172,8 @@ public class DFSInotifyEventInputStream {
|
||||||
*/
|
*/
|
||||||
public EventBatch poll(long time, TimeUnit tu) throws IOException,
|
public EventBatch poll(long time, TimeUnit tu) throws IOException,
|
||||||
InterruptedException, MissingEventsException {
|
InterruptedException, MissingEventsException {
|
||||||
TraceScope scope = tracer.newScope("inotifyPollWithTimeout");
|
EventBatch next;
|
||||||
EventBatch next = null;
|
try (TraceScope ignored = tracer.newScope("inotifyPollWithTimeout")) {
|
||||||
try {
|
|
||||||
long initialTime = Time.monotonicNow();
|
long initialTime = Time.monotonicNow();
|
||||||
long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
|
long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
|
||||||
long nextWait = INITIAL_WAIT_MS;
|
long nextWait = INITIAL_WAIT_MS;
|
||||||
|
@ -195,8 +191,6 @@ public class DFSInotifyEventInputStream {
|
||||||
nextWait);
|
nextWait);
|
||||||
Thread.sleep(nextWait);
|
Thread.sleep(nextWait);
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
return next;
|
return next;
|
||||||
}
|
}
|
||||||
|
@ -212,9 +206,8 @@ public class DFSInotifyEventInputStream {
|
||||||
*/
|
*/
|
||||||
public EventBatch take() throws IOException, InterruptedException,
|
public EventBatch take() throws IOException, InterruptedException,
|
||||||
MissingEventsException {
|
MissingEventsException {
|
||||||
TraceScope scope = tracer.newScope("inotifyTake");
|
EventBatch next;
|
||||||
EventBatch next = null;
|
try (TraceScope ignored = tracer.newScope("inotifyTake")) {
|
||||||
try {
|
|
||||||
int nextWaitMin = INITIAL_WAIT_MS;
|
int nextWaitMin = INITIAL_WAIT_MS;
|
||||||
while ((next = poll()) == null) {
|
while ((next = poll()) == null) {
|
||||||
// sleep for a random period between nextWaitMin and nextWaitMin * 2
|
// sleep for a random period between nextWaitMin and nextWaitMin * 2
|
||||||
|
@ -225,8 +218,6 @@ public class DFSInotifyEventInputStream {
|
||||||
// the maximum sleep is 2 minutes
|
// the maximum sleep is 2 minutes
|
||||||
nextWaitMin = Math.min(60000, nextWaitMin * 2);
|
nextWaitMin = Math.min(60000, nextWaitMin * 2);
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return next;
|
return next;
|
||||||
|
|
|
@ -55,11 +55,9 @@ import org.apache.hadoop.fs.CanUnbuffer;
|
||||||
import org.apache.hadoop.fs.ChecksumException;
|
import org.apache.hadoop.fs.ChecksumException;
|
||||||
import org.apache.hadoop.fs.FSInputStream;
|
import org.apache.hadoop.fs.FSInputStream;
|
||||||
import org.apache.hadoop.fs.FileEncryptionInfo;
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
|
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
|
||||||
import org.apache.hadoop.fs.ReadOption;
|
import org.apache.hadoop.fs.ReadOption;
|
||||||
import org.apache.hadoop.fs.StorageType;
|
import org.apache.hadoop.fs.StorageType;
|
||||||
import org.apache.hadoop.fs.UnresolvedLinkException;
|
|
||||||
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
|
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
|
@ -85,13 +83,15 @@ import org.apache.htrace.core.Tracer;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/****************************************************************
|
/****************************************************************
|
||||||
* DFSInputStream provides bytes from a named file. It handles
|
* DFSInputStream provides bytes from a named file. It handles
|
||||||
* negotiation of the namenode and various datanodes as necessary.
|
* negotiation of the namenode and various datanodes as necessary.
|
||||||
****************************************************************/
|
****************************************************************/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class DFSInputStream extends FSInputStream
|
public class DFSInputStream extends FSInputStream
|
||||||
implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
HasEnhancedByteBufferAccess, CanUnbuffer {
|
HasEnhancedByteBufferAccess, CanUnbuffer {
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public static boolean tcpReadsDisabledForTesting = false;
|
public static boolean tcpReadsDisabledForTesting = false;
|
||||||
|
@ -137,7 +137,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
private synchronized IdentityHashStore<ByteBuffer, Object>
|
private synchronized IdentityHashStore<ByteBuffer, Object>
|
||||||
getExtendedReadBuffers() {
|
getExtendedReadBuffers() {
|
||||||
if (extendedReadBuffers == null) {
|
if (extendedReadBuffers == null) {
|
||||||
extendedReadBuffers = new IdentityHashStore<ByteBuffer, Object>(0);
|
extendedReadBuffers = new IdentityHashStore<>(0);
|
||||||
}
|
}
|
||||||
return extendedReadBuffers;
|
return extendedReadBuffers;
|
||||||
}
|
}
|
||||||
|
@ -246,7 +246,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
/* XXX Use of CocurrentHashMap is temp fix. Need to fix
|
/* XXX Use of CocurrentHashMap is temp fix. Need to fix
|
||||||
* parallel accesses to DFSInputStream (through ptreads) properly */
|
* parallel accesses to DFSInputStream (through ptreads) properly */
|
||||||
private final ConcurrentHashMap<DatanodeInfo, DatanodeInfo> deadNodes =
|
private final ConcurrentHashMap<DatanodeInfo, DatanodeInfo> deadNodes =
|
||||||
new ConcurrentHashMap<DatanodeInfo, DatanodeInfo>();
|
new ConcurrentHashMap<>();
|
||||||
|
|
||||||
private byte[] oneByteBuf; // used for 'int read()'
|
private byte[] oneByteBuf; // used for 'int read()'
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
}
|
}
|
||||||
|
|
||||||
DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum,
|
DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum,
|
||||||
LocatedBlocks locatedBlocks) throws IOException, UnresolvedLinkException {
|
LocatedBlocks locatedBlocks) throws IOException {
|
||||||
this.dfsClient = dfsClient;
|
this.dfsClient = dfsClient;
|
||||||
this.verifyChecksum = verifyChecksum;
|
this.verifyChecksum = verifyChecksum;
|
||||||
this.src = src;
|
this.src = src;
|
||||||
|
@ -270,8 +270,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
* Grab the open-file info from namenode
|
* Grab the open-file info from namenode
|
||||||
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
|
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
|
||||||
*/
|
*/
|
||||||
void openInfo(boolean refreshLocatedBlocks) throws IOException,
|
void openInfo(boolean refreshLocatedBlocks) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
final DfsClientConf conf = dfsClient.getConf();
|
final DfsClientConf conf = dfsClient.getConf();
|
||||||
synchronized(infoLock) {
|
synchronized(infoLock) {
|
||||||
lastBlockBeingWrittenLength =
|
lastBlockBeingWrittenLength =
|
||||||
|
@ -375,8 +374,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
}
|
}
|
||||||
catch(IOException ioe) {
|
catch(IOException ioe) {
|
||||||
if (ioe instanceof RemoteException &&
|
if (ioe instanceof RemoteException &&
|
||||||
(((RemoteException) ioe).unwrapRemoteException() instanceof
|
(((RemoteException) ioe).unwrapRemoteException() instanceof
|
||||||
ReplicaNotFoundException)) {
|
ReplicaNotFoundException)) {
|
||||||
// special case : replica might not be on the DN, treat as 0 length
|
// special case : replica might not be on the DN, treat as 0 length
|
||||||
replicaNotFoundCount--;
|
replicaNotFoundCount--;
|
||||||
}
|
}
|
||||||
|
@ -526,12 +525,12 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
blocks = getFinalizedBlockRange(offset,
|
blocks = getFinalizedBlockRange(offset,
|
||||||
Math.min(length, lengthOfCompleteBlk - offset));
|
Math.min(length, lengthOfCompleteBlk - offset));
|
||||||
} else {
|
} else {
|
||||||
blocks = new ArrayList<LocatedBlock>(1);
|
blocks = new ArrayList<>(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// get the blocks from incomplete block range
|
// get the blocks from incomplete block range
|
||||||
if (readLengthPastCompleteBlk) {
|
if (readLengthPastCompleteBlk) {
|
||||||
blocks.add(locatedBlocks.getLastLocatedBlock());
|
blocks.add(locatedBlocks.getLastLocatedBlock());
|
||||||
}
|
}
|
||||||
|
|
||||||
return blocks;
|
return blocks;
|
||||||
|
@ -547,7 +546,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
long offset, long length) throws IOException {
|
long offset, long length) throws IOException {
|
||||||
synchronized(infoLock) {
|
synchronized(infoLock) {
|
||||||
assert (locatedBlocks != null) : "locatedBlocks is null";
|
assert (locatedBlocks != null) : "locatedBlocks is null";
|
||||||
List<LocatedBlock> blockRange = new ArrayList<LocatedBlock>();
|
List<LocatedBlock> blockRange = new ArrayList<>();
|
||||||
// search cached blocks first
|
// search cached blocks first
|
||||||
int blockIdx = locatedBlocks.findBlock(offset);
|
int blockIdx = locatedBlocks.findBlock(offset);
|
||||||
if (blockIdx < 0) { // block is not cached
|
if (blockIdx < 0) { // block is not cached
|
||||||
|
@ -591,7 +590,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
//
|
//
|
||||||
// Connect to best DataNode for desired Block, with potential offset
|
// Connect to best DataNode for desired Block, with potential offset
|
||||||
//
|
//
|
||||||
DatanodeInfo chosenNode = null;
|
DatanodeInfo chosenNode;
|
||||||
int refetchToken = 1; // only need to get a new access token once
|
int refetchToken = 1; // only need to get a new access token once
|
||||||
int refetchEncryptionKey = 1; // only need to get a new encryption key once
|
int refetchEncryptionKey = 1; // only need to get a new encryption key once
|
||||||
|
|
||||||
|
@ -639,7 +638,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
} else {
|
} else {
|
||||||
connectFailedOnce = true;
|
connectFailedOnce = true;
|
||||||
DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
|
DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
|
||||||
+ ", add to deadNodes and continue. " + ex, ex);
|
+ ", add to deadNodes and continue. " + ex, ex);
|
||||||
// Put chosen node into dead list, continue
|
// Put chosen node into dead list, continue
|
||||||
addToDeadNodes(chosenNode);
|
addToDeadNodes(chosenNode);
|
||||||
}
|
}
|
||||||
|
@ -722,8 +721,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
* strategy-agnostic.
|
* strategy-agnostic.
|
||||||
*/
|
*/
|
||||||
interface ReaderStrategy {
|
interface ReaderStrategy {
|
||||||
public int doRead(BlockReader blockReader, int off, int len)
|
int doRead(BlockReader blockReader, int off, int len)
|
||||||
throws ChecksumException, IOException;
|
throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Copy data from the src ByteBuffer into the read buffer.
|
* Copy data from the src ByteBuffer into the read buffer.
|
||||||
|
@ -733,7 +732,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
* @param length Useful only when the ReadStrategy is based on a byte array.
|
* @param length Useful only when the ReadStrategy is based on a byte array.
|
||||||
* Indicate the length of the data to copy.
|
* Indicate the length of the data to copy.
|
||||||
*/
|
*/
|
||||||
public int copyFrom(ByteBuffer src, int offset, int length);
|
int copyFrom(ByteBuffer src, int offset, int length);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void updateReadStatistics(ReadStatistics readStatistics,
|
protected void updateReadStatistics(ReadStatistics readStatistics,
|
||||||
|
@ -762,7 +761,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int doRead(BlockReader blockReader, int off, int len)
|
public int doRead(BlockReader blockReader, int off, int len)
|
||||||
throws ChecksumException, IOException {
|
throws IOException {
|
||||||
int nRead = blockReader.read(buf, off, len);
|
int nRead = blockReader.read(buf, off, len);
|
||||||
updateReadStatistics(readStatistics, nRead, blockReader);
|
updateReadStatistics(readStatistics, nRead, blockReader);
|
||||||
return nRead;
|
return nRead;
|
||||||
|
@ -787,7 +786,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int doRead(BlockReader blockReader, int off, int len)
|
public int doRead(BlockReader blockReader, int off, int len)
|
||||||
throws ChecksumException, IOException {
|
throws IOException {
|
||||||
int oldpos = buf.position();
|
int oldpos = buf.position();
|
||||||
int oldlimit = buf.limit();
|
int oldlimit = buf.limit();
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
|
@ -856,7 +855,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
}
|
}
|
||||||
ioe = e;
|
ioe = e;
|
||||||
}
|
}
|
||||||
boolean sourceFound = false;
|
boolean sourceFound;
|
||||||
if (retryCurrentNode) {
|
if (retryCurrentNode) {
|
||||||
/* possibly retry the same node so that transient errors don't
|
/* possibly retry the same node so that transient errors don't
|
||||||
* result in application level failures (e.g. Datanode could have
|
* result in application level failures (e.g. Datanode could have
|
||||||
|
@ -879,8 +878,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
if (closed.get()) {
|
if (closed.get()) {
|
||||||
throw new IOException("Stream closed");
|
throw new IOException("Stream closed");
|
||||||
}
|
}
|
||||||
Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap
|
Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap = new HashMap<>();
|
||||||
= new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
|
|
||||||
failures = 0;
|
failures = 0;
|
||||||
if (pos < getFileLength()) {
|
if (pos < getFileLength()) {
|
||||||
int retries = 2;
|
int retries = 2;
|
||||||
|
@ -936,26 +934,21 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
* Read the entire buffer.
|
* Read the entire buffer.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(final byte buf[], int off, int len) throws IOException {
|
public synchronized int read(@Nonnull final byte buf[], int off, int len)
|
||||||
|
throws IOException {
|
||||||
ReaderStrategy byteArrayReader = new ByteArrayStrategy(buf);
|
ReaderStrategy byteArrayReader = new ByteArrayStrategy(buf);
|
||||||
TraceScope scope =
|
try (TraceScope ignored =
|
||||||
dfsClient.newPathTraceScope("DFSInputStream#byteArrayRead", src);
|
dfsClient.newPathTraceScope("DFSInputStream#byteArrayRead", src)) {
|
||||||
try {
|
|
||||||
return readWithStrategy(byteArrayReader, off, len);
|
return readWithStrategy(byteArrayReader, off, len);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(final ByteBuffer buf) throws IOException {
|
public synchronized int read(final ByteBuffer buf) throws IOException {
|
||||||
ReaderStrategy byteBufferReader = new ByteBufferStrategy(buf);
|
ReaderStrategy byteBufferReader = new ByteBufferStrategy(buf);
|
||||||
TraceScope scope =
|
try (TraceScope ignored =
|
||||||
dfsClient.newPathTraceScope("DFSInputStream#byteBufferRead", src);
|
dfsClient.newPathTraceScope("DFSInputStream#byteBufferRead", src)){
|
||||||
try {
|
|
||||||
return readWithStrategy(byteBufferReader, 0, buf.remaining());
|
return readWithStrategy(byteBufferReader, 0, buf.remaining());
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -965,11 +958,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
*/
|
*/
|
||||||
protected void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node,
|
protected void addIntoCorruptedBlockMap(ExtendedBlock blk, DatanodeInfo node,
|
||||||
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
|
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
|
||||||
Set<DatanodeInfo> dnSet = null;
|
Set<DatanodeInfo> dnSet;
|
||||||
if((corruptedBlockMap.containsKey(blk))) {
|
if((corruptedBlockMap.containsKey(blk))) {
|
||||||
dnSet = corruptedBlockMap.get(blk);
|
dnSet = corruptedBlockMap.get(blk);
|
||||||
}else {
|
}else {
|
||||||
dnSet = new HashSet<DatanodeInfo>();
|
dnSet = new HashSet<>();
|
||||||
}
|
}
|
||||||
if (!dnSet.contains(node)) {
|
if (!dnSet.contains(node)) {
|
||||||
dnSet.add(node);
|
dnSet.add(node);
|
||||||
|
@ -985,7 +978,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
return result;
|
return result;
|
||||||
} else {
|
} else {
|
||||||
String errMsg = getBestNodeDNAddrPairErrorString(block.getLocations(),
|
String errMsg = getBestNodeDNAddrPairErrorString(block.getLocations(),
|
||||||
deadNodes, ignoredNodes);
|
deadNodes, ignoredNodes);
|
||||||
String blockInfo = block.getBlock() + " file=" + src;
|
String blockInfo = block.getBlock() + " file=" + src;
|
||||||
if (failures >= dfsClient.getConf().getMaxBlockAcquireFailures()) {
|
if (failures >= dfsClient.getConf().getMaxBlockAcquireFailures()) {
|
||||||
String description = "Could not obtain block: " + blockInfo;
|
String description = "Could not obtain block: " + blockInfo;
|
||||||
|
@ -1019,7 +1012,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
ThreadLocalRandom.current().nextDouble();
|
ThreadLocalRandom.current().nextDouble();
|
||||||
DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1) + " IOException, will wait for " + waitTime + " msec.");
|
DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1) + " IOException, will wait for " + waitTime + " msec.");
|
||||||
Thread.sleep((long)waitTime);
|
Thread.sleep((long)waitTime);
|
||||||
} catch (InterruptedException iex) {
|
} catch (InterruptedException ignored) {
|
||||||
}
|
}
|
||||||
deadNodes.clear(); //2nd option is to remove only nodes[blockId]
|
deadNodes.clear(); //2nd option is to remove only nodes[blockId]
|
||||||
openInfo(true);
|
openInfo(true);
|
||||||
|
@ -1122,14 +1115,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
public ByteBuffer call() throws Exception {
|
public ByteBuffer call() throws Exception {
|
||||||
byte[] buf = bb.array();
|
byte[] buf = bb.array();
|
||||||
int offset = bb.position();
|
int offset = bb.position();
|
||||||
TraceScope scope = dfsClient.getTracer().
|
try (TraceScope ignored = dfsClient.getTracer().
|
||||||
newScope("hedgedRead" + hedgedReadId, parentSpanId);
|
newScope("hedgedRead" + hedgedReadId, parentSpanId)) {
|
||||||
try {
|
|
||||||
actualGetFromOneDataNode(datanode, block, start, end, buf,
|
actualGetFromOneDataNode(datanode, block, start, end, buf,
|
||||||
offset, corruptedBlockMap);
|
offset, corruptedBlockMap);
|
||||||
return bb;
|
return bb;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -1271,12 +1261,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
|
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final DfsClientConf conf = dfsClient.getConf();
|
final DfsClientConf conf = dfsClient.getConf();
|
||||||
ArrayList<Future<ByteBuffer>> futures = new ArrayList<Future<ByteBuffer>>();
|
ArrayList<Future<ByteBuffer>> futures = new ArrayList<>();
|
||||||
CompletionService<ByteBuffer> hedgedService =
|
CompletionService<ByteBuffer> hedgedService =
|
||||||
new ExecutorCompletionService<ByteBuffer>(
|
new ExecutorCompletionService<>(dfsClient.getHedgedReadsThreadPool());
|
||||||
dfsClient.getHedgedReadsThreadPool());
|
ArrayList<DatanodeInfo> ignored = new ArrayList<>();
|
||||||
ArrayList<DatanodeInfo> ignored = new ArrayList<DatanodeInfo>();
|
ByteBuffer bb;
|
||||||
ByteBuffer bb = null;
|
|
||||||
int len = (int) (end - start + 1);
|
int len = (int) (end - start + 1);
|
||||||
int hedgedReadId = 0;
|
int hedgedReadId = 0;
|
||||||
block = refreshLocatedBlock(block);
|
block = refreshLocatedBlock(block);
|
||||||
|
@ -1308,11 +1297,9 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
// Ignore this node on next go around.
|
// Ignore this node on next go around.
|
||||||
ignored.add(chosenNode.info);
|
ignored.add(chosenNode.info);
|
||||||
dfsClient.getHedgedReadMetrics().incHedgedReadOps();
|
dfsClient.getHedgedReadMetrics().incHedgedReadOps();
|
||||||
continue; // no need to refresh block locations
|
// continue; no need to refresh block locations
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException | ExecutionException e) {
|
||||||
// Ignore
|
// Ignore
|
||||||
} catch (ExecutionException e) {
|
|
||||||
// Ignore already logged in the call.
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// We are starting up a 'hedged' read. We have a read already
|
// We are starting up a 'hedged' read. We have a read already
|
||||||
|
@ -1377,10 +1364,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
ByteBuffer bb = future.get();
|
ByteBuffer bb = future.get();
|
||||||
futures.remove(future);
|
futures.remove(future);
|
||||||
return bb;
|
return bb;
|
||||||
} catch (ExecutionException e) {
|
} catch (ExecutionException | CancellationException e) {
|
||||||
// already logged in the Callable
|
|
||||||
futures.remove(future);
|
|
||||||
} catch (CancellationException ce) {
|
|
||||||
// already logged in the Callable
|
// already logged in the Callable
|
||||||
futures.remove(future);
|
futures.remove(future);
|
||||||
}
|
}
|
||||||
|
@ -1440,12 +1424,9 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
@Override
|
@Override
|
||||||
public int read(long position, byte[] buffer, int offset, int length)
|
public int read(long position, byte[] buffer, int offset, int length)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = dfsClient.
|
try (TraceScope ignored = dfsClient.
|
||||||
newPathTraceScope("DFSInputStream#byteArrayPread", src);
|
newPathTraceScope("DFSInputStream#byteArrayPread", src)) {
|
||||||
try {
|
|
||||||
return pread(position, buffer, offset, length);
|
return pread(position, buffer, offset, length);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1470,8 +1451,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
// corresponding to position and realLen
|
// corresponding to position and realLen
|
||||||
List<LocatedBlock> blockRange = getBlockRange(position, realLen);
|
List<LocatedBlock> blockRange = getBlockRange(position, realLen);
|
||||||
int remaining = realLen;
|
int remaining = realLen;
|
||||||
Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap
|
Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap = new HashMap<>();
|
||||||
= new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
|
|
||||||
for (LocatedBlock blk : blockRange) {
|
for (LocatedBlock blk : blockRange) {
|
||||||
long targetStart = position - blk.getStartOffset();
|
long targetStart = position - blk.getStartOffset();
|
||||||
long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
|
long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
|
||||||
|
@ -1748,7 +1728,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
|
||||||
* zero-copy read.
|
* zero-copy read.
|
||||||
*/
|
*/
|
||||||
private static final ByteBuffer EMPTY_BUFFER =
|
private static final ByteBuffer EMPTY_BUFFER =
|
||||||
ByteBuffer.allocateDirect(0).asReadOnlyBuffer();
|
ByteBuffer.allocateDirect(0).asReadOnlyBuffer();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized ByteBuffer read(ByteBufferPool bufferPool,
|
public synchronized ByteBuffer read(ByteBufferPool bufferPool,
|
||||||
|
|
|
@ -32,7 +32,6 @@ import org.apache.hadoop.fs.CreateFlag;
|
||||||
import org.apache.hadoop.fs.FSOutputSummer;
|
import org.apache.hadoop.fs.FSOutputSummer;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
import org.apache.hadoop.fs.FileEncryptionInfo;
|
import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
||||||
import org.apache.hadoop.fs.Syncable;
|
import org.apache.hadoop.fs.Syncable;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
@ -121,8 +120,9 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
private FileEncryptionInfo fileEncryptionInfo;
|
private FileEncryptionInfo fileEncryptionInfo;
|
||||||
|
|
||||||
/** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/
|
/** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/
|
||||||
protected DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock,
|
protected DFSPacket createPacket(int packetSize, int chunksPerPkt,
|
||||||
long seqno, boolean lastPacketInBlock) throws InterruptedIOException {
|
long offsetInBlock, long seqno, boolean lastPacketInBlock)
|
||||||
|
throws InterruptedIOException {
|
||||||
final byte[] buf;
|
final byte[] buf;
|
||||||
final int bufferSize = PacketHeader.PKT_MAX_HEADER_LEN + packetSize;
|
final int bufferSize = PacketHeader.PKT_MAX_HEADER_LEN + packetSize;
|
||||||
|
|
||||||
|
@ -159,9 +159,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
DatanodeInfo[] value = new DatanodeInfo[currentNodes.length];
|
DatanodeInfo[] value = new DatanodeInfo[currentNodes.length];
|
||||||
for (int i = 0; i < currentNodes.length; i++) {
|
System.arraycopy(currentNodes, 0, value, 0, currentNodes.length);
|
||||||
value[i] = currentNodes[i];
|
|
||||||
}
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,8 +177,8 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
return checksum;
|
return checksum;
|
||||||
}
|
}
|
||||||
|
|
||||||
private DFSOutputStream(DFSClient dfsClient, String src, Progressable progress,
|
private DFSOutputStream(DFSClient dfsClient, String src,
|
||||||
HdfsFileStatus stat, DataChecksum checksum) throws IOException {
|
Progressable progress, HdfsFileStatus stat, DataChecksum checksum) {
|
||||||
super(getChecksum4Compute(checksum, stat));
|
super(getChecksum4Compute(checksum, stat));
|
||||||
this.dfsClient = dfsClient;
|
this.dfsClient = dfsClient;
|
||||||
this.src = src;
|
this.src = src;
|
||||||
|
@ -188,7 +186,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
this.blockSize = stat.getBlockSize();
|
this.blockSize = stat.getBlockSize();
|
||||||
this.blockReplication = stat.getReplication();
|
this.blockReplication = stat.getReplication();
|
||||||
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
|
this.fileEncryptionInfo = stat.getFileEncryptionInfo();
|
||||||
this.cachingStrategy = new AtomicReference<CachingStrategy>(
|
this.cachingStrategy = new AtomicReference<>(
|
||||||
dfsClient.getDefaultWriteCachingStrategy());
|
dfsClient.getDefaultWriteCachingStrategy());
|
||||||
if (progress != null) {
|
if (progress != null) {
|
||||||
DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "
|
DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "
|
||||||
|
@ -202,8 +200,9 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
}
|
}
|
||||||
if (blockSize % bytesPerChecksum != 0) {
|
if (blockSize % bytesPerChecksum != 0) {
|
||||||
throw new HadoopIllegalArgumentException("Invalid values: "
|
throw new HadoopIllegalArgumentException("Invalid values: "
|
||||||
+ HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY + " (=" + bytesPerChecksum
|
+ HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY
|
||||||
+ ") must divide block size (=" + blockSize + ").");
|
+ " (=" + bytesPerChecksum + ") must divide block size (=" +
|
||||||
|
blockSize + ").");
|
||||||
}
|
}
|
||||||
this.byteArrayManager = dfsClient.getClientContext().getByteArrayManager();
|
this.byteArrayManager = dfsClient.getClientContext().getByteArrayManager();
|
||||||
}
|
}
|
||||||
|
@ -215,7 +214,8 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
this(dfsClient, src, progress, stat, checksum);
|
this(dfsClient, src, progress, stat, checksum);
|
||||||
this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK);
|
this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK);
|
||||||
|
|
||||||
computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);
|
computePacketChunkSize(dfsClient.getConf().getWritePacketSize(),
|
||||||
|
bytesPerChecksum);
|
||||||
|
|
||||||
streamer = new DataStreamer(stat, null, dfsClient, src, progress, checksum,
|
streamer = new DataStreamer(stat, null, dfsClient, src, progress, checksum,
|
||||||
cachingStrategy, byteArrayManager, favoredNodes);
|
cachingStrategy, byteArrayManager, favoredNodes);
|
||||||
|
@ -223,11 +223,10 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
|
|
||||||
static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
|
static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
|
||||||
FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
|
FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,
|
||||||
short replication, long blockSize, Progressable progress, int buffersize,
|
short replication, long blockSize, Progressable progress,
|
||||||
DataChecksum checksum, String[] favoredNodes) throws IOException {
|
DataChecksum checksum, String[] favoredNodes) throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored =
|
||||||
dfsClient.newPathTraceScope("newStreamForCreate", src);
|
dfsClient.newPathTraceScope("newStreamForCreate", src)) {
|
||||||
try {
|
|
||||||
HdfsFileStatus stat = null;
|
HdfsFileStatus stat = null;
|
||||||
|
|
||||||
// Retry the create if we get a RetryStartFileException up to a maximum
|
// Retry the create if we get a RetryStartFileException up to a maximum
|
||||||
|
@ -238,7 +237,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
shouldRetry = false;
|
shouldRetry = false;
|
||||||
try {
|
try {
|
||||||
stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
|
stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,
|
||||||
new EnumSetWritable<CreateFlag>(flag), createParent, replication,
|
new EnumSetWritable<>(flag), createParent, replication,
|
||||||
blockSize, SUPPORTED_CRYPTO_VERSIONS);
|
blockSize, SUPPORTED_CRYPTO_VERSIONS);
|
||||||
break;
|
break;
|
||||||
} catch (RemoteException re) {
|
} catch (RemoteException re) {
|
||||||
|
@ -273,8 +272,6 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
flag, progress, checksum, favoredNodes);
|
flag, progress, checksum, favoredNodes);
|
||||||
out.start();
|
out.start();
|
||||||
return out;
|
return out;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -294,17 +291,17 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
// The last partial block of the file has to be filled.
|
// The last partial block of the file has to be filled.
|
||||||
if (!toNewBlock && lastBlock != null) {
|
if (!toNewBlock && lastBlock != null) {
|
||||||
// indicate that we are appending to an existing block
|
// indicate that we are appending to an existing block
|
||||||
streamer = new DataStreamer(lastBlock, stat, dfsClient, src, progress, checksum,
|
streamer = new DataStreamer(lastBlock, stat, dfsClient, src, progress,
|
||||||
cachingStrategy, byteArrayManager);
|
checksum, cachingStrategy, byteArrayManager);
|
||||||
getStreamer().setBytesCurBlock(lastBlock.getBlockSize());
|
getStreamer().setBytesCurBlock(lastBlock.getBlockSize());
|
||||||
adjustPacketChunkSize(stat);
|
adjustPacketChunkSize(stat);
|
||||||
getStreamer().setPipelineInConstruction(lastBlock);
|
getStreamer().setPipelineInConstruction(lastBlock);
|
||||||
} else {
|
} else {
|
||||||
computePacketChunkSize(dfsClient.getConf().getWritePacketSize(),
|
computePacketChunkSize(dfsClient.getConf().getWritePacketSize(),
|
||||||
bytesPerChecksum);
|
bytesPerChecksum);
|
||||||
streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null,
|
streamer = new DataStreamer(stat,
|
||||||
dfsClient, src, progress, checksum, cachingStrategy, byteArrayManager,
|
lastBlock != null ? lastBlock.getBlock() : null, dfsClient, src,
|
||||||
favoredNodes);
|
progress, checksum, cachingStrategy, byteArrayManager, favoredNodes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -345,18 +342,15 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
}
|
}
|
||||||
|
|
||||||
static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
|
static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
|
||||||
EnumSet<CreateFlag> flags, int bufferSize, Progressable progress,
|
EnumSet<CreateFlag> flags, Progressable progress, LocatedBlock lastBlock,
|
||||||
LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum,
|
HdfsFileStatus stat, DataChecksum checksum, String[] favoredNodes)
|
||||||
String[] favoredNodes) throws IOException {
|
throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored =
|
||||||
dfsClient.newPathTraceScope("newStreamForAppend", src);
|
dfsClient.newPathTraceScope("newStreamForAppend", src)) {
|
||||||
try {
|
|
||||||
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, flags,
|
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, flags,
|
||||||
progress, lastBlock, stat, checksum, favoredNodes);
|
progress, lastBlock, stat, checksum, favoredNodes);
|
||||||
out.start();
|
out.start();
|
||||||
return out;
|
return out;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -486,23 +480,15 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void hflush() throws IOException {
|
public void hflush() throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored = dfsClient.newPathTraceScope("hflush", src)) {
|
||||||
dfsClient.newPathTraceScope("hflush", src);
|
|
||||||
try {
|
|
||||||
flushOrSync(false, EnumSet.noneOf(SyncFlag.class));
|
flushOrSync(false, EnumSet.noneOf(SyncFlag.class));
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void hsync() throws IOException {
|
public void hsync() throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored = dfsClient.newPathTraceScope("hsync", src)) {
|
||||||
dfsClient.newPathTraceScope("hsync", src);
|
|
||||||
try {
|
|
||||||
flushOrSync(true, EnumSet.noneOf(SyncFlag.class));
|
flushOrSync(true, EnumSet.noneOf(SyncFlag.class));
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -520,12 +506,8 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
* whether or not to update the block length in NameNode.
|
* whether or not to update the block length in NameNode.
|
||||||
*/
|
*/
|
||||||
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored = dfsClient.newPathTraceScope("hsync", src)) {
|
||||||
dfsClient.newPathTraceScope("hsync", src);
|
|
||||||
try {
|
|
||||||
flushOrSync(true, syncFlags);
|
flushOrSync(true, syncFlags);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -630,13 +612,14 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
dfsClient.namenode.fsync(src, fileId, dfsClient.clientName,
|
dfsClient.namenode.fsync(src, fileId, dfsClient.clientName,
|
||||||
lastBlockLength);
|
lastBlockLength);
|
||||||
} catch (IOException ioe) {
|
} catch (IOException ioe) {
|
||||||
DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe);
|
DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src,
|
||||||
// If we got an error here, it might be because some other thread called
|
ioe);
|
||||||
// close before our hflush completed. In that case, we should throw an
|
// If we got an error here, it might be because some other thread
|
||||||
// exception that the stream is closed.
|
// called close before our hflush completed. In that case, we should
|
||||||
|
// throw an exception that the stream is closed.
|
||||||
checkClosed();
|
checkClosed();
|
||||||
// If we aren't closed but failed to sync, we should expose that to the
|
// If we aren't closed but failed to sync, we should expose that to
|
||||||
// caller.
|
// the caller.
|
||||||
throw ioe;
|
throw ioe;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -647,9 +630,9 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (InterruptedIOException interrupt) {
|
} catch (InterruptedIOException interrupt) {
|
||||||
// This kind of error doesn't mean that the stream itself is broken - just the
|
// This kind of error doesn't mean that the stream itself is broken - just
|
||||||
// flushing thread got interrupted. So, we shouldn't close down the writer,
|
// the flushing thread got interrupted. So, we shouldn't close down the
|
||||||
// but instead just propagate the error
|
// writer, but instead just propagate the error
|
||||||
throw interrupt;
|
throw interrupt;
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
DFSClient.LOG.warn("Error while syncing", e);
|
DFSClient.LOG.warn("Error while syncing", e);
|
||||||
|
@ -723,7 +706,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
getStreamer().getLastException().set(new IOException("Lease timeout of "
|
getStreamer().getLastException().set(new IOException("Lease timeout of "
|
||||||
+ (dfsClient.getConf().getHdfsTimeout()/1000) + " seconds expired."));
|
+ (dfsClient.getConf().getHdfsTimeout() / 1000) + " seconds expired."));
|
||||||
closeThreads(true);
|
closeThreads(true);
|
||||||
dfsClient.endFileLease(fileId);
|
dfsClient.endFileLease(fileId);
|
||||||
}
|
}
|
||||||
|
@ -758,12 +741,9 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public synchronized void close() throws IOException {
|
public synchronized void close() throws IOException {
|
||||||
TraceScope scope =
|
try (TraceScope ignored =
|
||||||
dfsClient.newPathTraceScope("DFSOutputStream#close", src);
|
dfsClient.newPathTraceScope("DFSOutputStream#close", src)) {
|
||||||
try {
|
|
||||||
closeImpl();
|
closeImpl();
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -788,14 +768,12 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
// get last block before destroying the streamer
|
// get last block before destroying the streamer
|
||||||
ExtendedBlock lastBlock = getStreamer().getBlock();
|
ExtendedBlock lastBlock = getStreamer().getBlock();
|
||||||
closeThreads(false);
|
closeThreads(false);
|
||||||
TraceScope scope = dfsClient.getTracer().newScope("completeFile");
|
try (TraceScope ignored =
|
||||||
try {
|
dfsClient.getTracer().newScope("completeFile")) {
|
||||||
completeFile(lastBlock);
|
completeFile(lastBlock);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
dfsClient.endFileLease(fileId);
|
dfsClient.endFileLease(fileId);
|
||||||
} catch (ClosedChannelException e) {
|
} catch (ClosedChannelException ignored) {
|
||||||
} finally {
|
} finally {
|
||||||
setClosed();
|
setClosed();
|
||||||
}
|
}
|
||||||
|
@ -817,12 +795,11 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
if (!dfsClient.clientRunning
|
if (!dfsClient.clientRunning
|
||||||
|| (hdfsTimeout > 0
|
|| (hdfsTimeout > 0
|
||||||
&& localstart + hdfsTimeout < Time.monotonicNow())) {
|
&& localstart + hdfsTimeout < Time.monotonicNow())) {
|
||||||
String msg = "Unable to close file because dfsclient " +
|
String msg = "Unable to close file because dfsclient " +
|
||||||
" was unable to contact the HDFS servers." +
|
" was unable to contact the HDFS servers. clientRunning " +
|
||||||
" clientRunning " + dfsClient.clientRunning +
|
dfsClient.clientRunning + " hdfsTimeout " + hdfsTimeout;
|
||||||
" hdfsTimeout " + hdfsTimeout;
|
DFSClient.LOG.info(msg);
|
||||||
DFSClient.LOG.info(msg);
|
throw new IOException(msg);
|
||||||
throw new IOException(msg);
|
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
if (retries == 0) {
|
if (retries == 0) {
|
||||||
|
|
|
@ -139,7 +139,6 @@ class DFSPacket {
|
||||||
/**
|
/**
|
||||||
* Write the full packet, including the header, to the given output stream.
|
* Write the full packet, including the header, to the given output stream.
|
||||||
*
|
*
|
||||||
* @param stm
|
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
synchronized void writeTo(DataOutputStream stm) throws IOException {
|
synchronized void writeTo(DataOutputStream stm) throws IOException {
|
||||||
|
@ -173,15 +172,18 @@ class DFSPacket {
|
||||||
|
|
||||||
// corrupt the data for testing.
|
// corrupt the data for testing.
|
||||||
if (DFSClientFaultInjector.get().corruptPacket()) {
|
if (DFSClientFaultInjector.get().corruptPacket()) {
|
||||||
buf[headerStart+header.getSerializedSize() + checksumLen + dataLen-1] ^= 0xff;
|
buf[headerStart+header.getSerializedSize() + checksumLen + dataLen-1] ^=
|
||||||
|
0xff;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write the now contiguous full packet to the output stream.
|
// Write the now contiguous full packet to the output stream.
|
||||||
stm.write(buf, headerStart, header.getSerializedSize() + checksumLen + dataLen);
|
stm.write(buf, headerStart,
|
||||||
|
header.getSerializedSize() + checksumLen + dataLen);
|
||||||
|
|
||||||
// undo corruption.
|
// undo corruption.
|
||||||
if (DFSClientFaultInjector.get().uncorruptPacket()) {
|
if (DFSClientFaultInjector.get().uncorruptPacket()) {
|
||||||
buf[headerStart+header.getSerializedSize() + checksumLen + dataLen-1] ^= 0xff;
|
buf[headerStart+header.getSerializedSize() + checksumLen + dataLen-1] ^=
|
||||||
|
0xff;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -193,8 +195,6 @@ class DFSPacket {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Release the buffer in this packet to ByteArrayManager.
|
* Release the buffer in this packet to ByteArrayManager.
|
||||||
*
|
|
||||||
* @param bam
|
|
||||||
*/
|
*/
|
||||||
synchronized void releaseBuffer(ByteArrayManager bam) {
|
synchronized void releaseBuffer(ByteArrayManager bam) {
|
||||||
bam.release(buf);
|
bam.release(buf);
|
||||||
|
|
|
@ -303,8 +303,8 @@ public class DFSUtilClient {
|
||||||
* @param keys Set of keys to look for in the order of preference
|
* @param keys Set of keys to look for in the order of preference
|
||||||
* @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
|
* @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
|
||||||
*/
|
*/
|
||||||
static Map<String, Map<String, InetSocketAddress>>
|
static Map<String, Map<String, InetSocketAddress>> getAddresses(
|
||||||
getAddresses(Configuration conf, String defaultAddress, String... keys) {
|
Configuration conf, String defaultAddress, String... keys) {
|
||||||
Collection<String> nameserviceIds = getNameServiceIds(conf);
|
Collection<String> nameserviceIds = getNameServiceIds(conf);
|
||||||
return getAddressesForNsIds(conf, nameserviceIds, defaultAddress, keys);
|
return getAddressesForNsIds(conf, nameserviceIds, defaultAddress, keys);
|
||||||
}
|
}
|
||||||
|
@ -317,8 +317,7 @@ public class DFSUtilClient {
|
||||||
*
|
*
|
||||||
* @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
|
* @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
|
||||||
*/
|
*/
|
||||||
static Map<String, Map<String, InetSocketAddress>>
|
static Map<String, Map<String, InetSocketAddress>> getAddressesForNsIds(
|
||||||
getAddressesForNsIds(
|
|
||||||
Configuration conf, Collection<String> nsIds, String defaultAddress,
|
Configuration conf, Collection<String> nsIds, String defaultAddress,
|
||||||
String... keys) {
|
String... keys) {
|
||||||
// Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>]
|
// Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>]
|
||||||
|
@ -326,7 +325,7 @@ public class DFSUtilClient {
|
||||||
Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap();
|
Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap();
|
||||||
for (String nsId : emptyAsSingletonNull(nsIds)) {
|
for (String nsId : emptyAsSingletonNull(nsIds)) {
|
||||||
Map<String, InetSocketAddress> isas =
|
Map<String, InetSocketAddress> isas =
|
||||||
getAddressesForNameserviceId(conf, nsId, defaultAddress, keys);
|
getAddressesForNameserviceId(conf, nsId, defaultAddress, keys);
|
||||||
if (!isas.isEmpty()) {
|
if (!isas.isEmpty()) {
|
||||||
ret.put(nsId, isas);
|
ret.put(nsId, isas);
|
||||||
}
|
}
|
||||||
|
@ -533,7 +532,7 @@ public class DFSUtilClient {
|
||||||
|
|
||||||
public static Peer peerFromSocket(Socket socket)
|
public static Peer peerFromSocket(Socket socket)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Peer peer = null;
|
Peer peer;
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
try {
|
try {
|
||||||
// TCP_NODELAY is crucial here because of bad interactions between
|
// TCP_NODELAY is crucial here because of bad interactions between
|
||||||
|
@ -560,7 +559,7 @@ public class DFSUtilClient {
|
||||||
return peer;
|
return peer;
|
||||||
} finally {
|
} finally {
|
||||||
if (!success) {
|
if (!success) {
|
||||||
if (peer != null) peer.close();
|
// peer is always null so no need to call peer.close().
|
||||||
socket.close();
|
socket.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.StorageType;
|
import org.apache.hadoop.fs.StorageType;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite;
|
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite;
|
||||||
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
|
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
|
||||||
|
@ -80,7 +79,6 @@ import org.apache.hadoop.util.Daemon;
|
||||||
import org.apache.hadoop.util.DataChecksum;
|
import org.apache.hadoop.util.DataChecksum;
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
import org.apache.hadoop.util.Time;
|
import org.apache.hadoop.util.Time;
|
||||||
import org.apache.htrace.core.Sampler;
|
|
||||||
import org.apache.htrace.core.Span;
|
import org.apache.htrace.core.Span;
|
||||||
import org.apache.htrace.core.SpanId;
|
import org.apache.htrace.core.SpanId;
|
||||||
import org.apache.htrace.core.TraceScope;
|
import org.apache.htrace.core.TraceScope;
|
||||||
|
@ -95,6 +93,8 @@ import com.google.common.cache.RemovalNotification;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/*********************************************************************
|
/*********************************************************************
|
||||||
*
|
*
|
||||||
* The DataStreamer class is responsible for sending data packets to the
|
* The DataStreamer class is responsible for sending data packets to the
|
||||||
|
@ -136,7 +136,8 @@ class DataStreamer extends Daemon {
|
||||||
final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
|
final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
|
||||||
final Socket sock = client.socketFactory.createSocket();
|
final Socket sock = client.socketFactory.createSocket();
|
||||||
final int timeout = client.getDatanodeReadTimeout(length);
|
final int timeout = client.getDatanodeReadTimeout(length);
|
||||||
NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), conf.getSocketTimeout());
|
NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(),
|
||||||
|
conf.getSocketTimeout());
|
||||||
sock.setSoTimeout(timeout);
|
sock.setSoTimeout(timeout);
|
||||||
sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
|
sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
|
||||||
LOG.debug("Send buf size {}", sock.getSendBufferSize());
|
LOG.debug("Send buf size {}", sock.getSendBufferSize());
|
||||||
|
@ -427,7 +428,6 @@ class DataStreamer extends Daemon {
|
||||||
* Construct a data streamer for appending to the last partial block
|
* Construct a data streamer for appending to the last partial block
|
||||||
* @param lastBlock last block of the file to be appended
|
* @param lastBlock last block of the file to be appended
|
||||||
* @param stat status of the file to be appended
|
* @param stat status of the file to be appended
|
||||||
* @throws IOException if error occurs
|
|
||||||
*/
|
*/
|
||||||
DataStreamer(LocatedBlock lastBlock, HdfsFileStatus stat, DFSClient dfsClient,
|
DataStreamer(LocatedBlock lastBlock, HdfsFileStatus stat, DFSClient dfsClient,
|
||||||
String src, Progressable progress, DataChecksum checksum,
|
String src, Progressable progress, DataChecksum checksum,
|
||||||
|
@ -620,9 +620,8 @@ class DataStreamer extends Daemon {
|
||||||
LOG.debug("DataStreamer block {} sending packet {}", block, one);
|
LOG.debug("DataStreamer block {} sending packet {}", block, one);
|
||||||
|
|
||||||
// write out data to remote datanode
|
// write out data to remote datanode
|
||||||
TraceScope writeScope = dfsClient.getTracer().
|
try (TraceScope ignored = dfsClient.getTracer().
|
||||||
newScope("DataStreamer#writeTo", spanId);
|
newScope("DataStreamer#writeTo", spanId)) {
|
||||||
try {
|
|
||||||
one.writeTo(blockStream);
|
one.writeTo(blockStream);
|
||||||
blockStream.flush();
|
blockStream.flush();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -634,8 +633,6 @@ class DataStreamer extends Daemon {
|
||||||
// will be taken out then.
|
// will be taken out then.
|
||||||
errorState.markFirstNodeIfNotMarked();
|
errorState.markFirstNodeIfNotMarked();
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
|
||||||
writeScope.close();
|
|
||||||
}
|
}
|
||||||
lastPacket = Time.monotonicNow();
|
lastPacket = Time.monotonicNow();
|
||||||
|
|
||||||
|
@ -725,9 +722,8 @@ class DataStreamer extends Daemon {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
void waitForAckedSeqno(long seqno) throws IOException {
|
void waitForAckedSeqno(long seqno) throws IOException {
|
||||||
TraceScope scope = dfsClient.getTracer().
|
try (TraceScope ignored = dfsClient.getTracer().
|
||||||
newScope("waitForAckedSeqno");
|
newScope("waitForAckedSeqno")) {
|
||||||
try {
|
|
||||||
LOG.debug("Waiting for ack for: {}", seqno);
|
LOG.debug("Waiting for ack for: {}", seqno);
|
||||||
long begin = Time.monotonicNow();
|
long begin = Time.monotonicNow();
|
||||||
try {
|
try {
|
||||||
|
@ -747,15 +743,13 @@ class DataStreamer extends Daemon {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
checkClosed();
|
checkClosed();
|
||||||
} catch (ClosedChannelException e) {
|
} catch (ClosedChannelException cce) {
|
||||||
}
|
}
|
||||||
long duration = Time.monotonicNow() - begin;
|
long duration = Time.monotonicNow() - begin;
|
||||||
if (duration > dfsclientSlowLogThresholdMs) {
|
if (duration > dfsclientSlowLogThresholdMs) {
|
||||||
LOG.warn("Slow waitForAckedSeqno took " + duration
|
LOG.warn("Slow waitForAckedSeqno took " + duration
|
||||||
+ "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms)");
|
+ "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms)");
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -802,7 +796,7 @@ class DataStreamer extends Daemon {
|
||||||
}
|
}
|
||||||
checkClosed();
|
checkClosed();
|
||||||
queuePacket(packet);
|
queuePacket(packet);
|
||||||
} catch (ClosedChannelException e) {
|
} catch (ClosedChannelException ignored) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -901,10 +895,7 @@ class DataStreamer extends Daemon {
|
||||||
assert false;
|
assert false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (addr != null && NetUtils.isLocalAddress(addr)) {
|
return addr != null && NetUtils.isLocalAddress(addr);
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
|
@ -1137,17 +1128,15 @@ class DataStreamer extends Daemon {
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
if (nodes.length != original.length + 1) {
|
if (nodes.length != original.length + 1) {
|
||||||
throw new IOException(
|
throw new IOException(
|
||||||
new StringBuilder()
|
"Failed to replace a bad datanode on the existing pipeline "
|
||||||
.append("Failed to replace a bad datanode on the existing pipeline ")
|
+ "due to no more good datanodes being available to try. "
|
||||||
.append("due to no more good datanodes being available to try. ")
|
+ "(Nodes: current=" + Arrays.asList(nodes)
|
||||||
.append("(Nodes: current=").append(Arrays.asList(nodes))
|
+ ", original=" + Arrays.asList(original) + "). "
|
||||||
.append(", original=").append(Arrays.asList(original)).append("). ")
|
+ "The current failed datanode replacement policy is "
|
||||||
.append("The current failed datanode replacement policy is ")
|
+ dfsClient.dtpReplaceDatanodeOnFailure
|
||||||
.append(dfsClient.dtpReplaceDatanodeOnFailure).append(", and ")
|
+ ", and a client may configure this via '"
|
||||||
.append("a client may configure this via '")
|
+ BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY
|
||||||
.append(BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY)
|
+ "' in its configuration.");
|
||||||
.append("' in its configuration.")
|
|
||||||
.toString());
|
|
||||||
}
|
}
|
||||||
for(int i = 0; i < nodes.length; i++) {
|
for(int i = 0; i < nodes.length; i++) {
|
||||||
int j = 0;
|
int j = 0;
|
||||||
|
@ -1196,7 +1185,7 @@ class DataStreamer extends Daemon {
|
||||||
final StorageType[] originalTypes = storageTypes;
|
final StorageType[] originalTypes = storageTypes;
|
||||||
final String[] originalIDs = storageIDs;
|
final String[] originalIDs = storageIDs;
|
||||||
IOException caughtException = null;
|
IOException caughtException = null;
|
||||||
ArrayList<DatanodeInfo> exclude = new ArrayList<DatanodeInfo>(failed);
|
ArrayList<DatanodeInfo> exclude = new ArrayList<>(failed);
|
||||||
while (tried < 3) {
|
while (tried < 3) {
|
||||||
LocatedBlock lb;
|
LocatedBlock lb;
|
||||||
//get a new datanode
|
//get a new datanode
|
||||||
|
@ -1235,7 +1224,8 @@ class DataStreamer extends Daemon {
|
||||||
|
|
||||||
private void transfer(final DatanodeInfo src, final DatanodeInfo[] targets,
|
private void transfer(final DatanodeInfo src, final DatanodeInfo[] targets,
|
||||||
final StorageType[] targetStorageTypes,
|
final StorageType[] targetStorageTypes,
|
||||||
final Token<BlockTokenIdentifier> blockToken) throws IOException {
|
final Token<BlockTokenIdentifier> blockToken)
|
||||||
|
throws IOException {
|
||||||
//transfer replica to the new datanode
|
//transfer replica to the new datanode
|
||||||
Socket sock = null;
|
Socket sock = null;
|
||||||
DataOutputStream out = null;
|
DataOutputStream out = null;
|
||||||
|
@ -1246,7 +1236,8 @@ class DataStreamer extends Daemon {
|
||||||
|
|
||||||
// transfer timeout multiplier based on the transfer size
|
// transfer timeout multiplier based on the transfer size
|
||||||
// One per 200 packets = 12.8MB. Minimum is 2.
|
// One per 200 packets = 12.8MB. Minimum is 2.
|
||||||
int multi = 2 + (int)(bytesSent/dfsClient.getConf().getWritePacketSize())/200;
|
int multi = 2 + (int)(bytesSent /dfsClient.getConf().getWritePacketSize())
|
||||||
|
/ 200;
|
||||||
final long readTimeout = dfsClient.getDatanodeReadTimeout(multi);
|
final long readTimeout = dfsClient.getDatanodeReadTimeout(multi);
|
||||||
|
|
||||||
OutputStream unbufOut = NetUtils.getOutputStream(sock, writeTimeout);
|
OutputStream unbufOut = NetUtils.getOutputStream(sock, writeTimeout);
|
||||||
|
@ -1413,13 +1404,14 @@ class DataStreamer extends Daemon {
|
||||||
// good reports should follow bad ones, if client committed
|
// good reports should follow bad ones, if client committed
|
||||||
// with those nodes.
|
// with those nodes.
|
||||||
Thread.sleep(2000);
|
Thread.sleep(2000);
|
||||||
} catch (InterruptedException ie) {}
|
} catch (InterruptedException ignored) {
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
LocatedBlock updateBlockForPipeline() throws IOException {
|
private LocatedBlock updateBlockForPipeline() throws IOException {
|
||||||
return dfsClient.namenode.updateBlockForPipeline(
|
return dfsClient.namenode.updateBlockForPipeline(block,
|
||||||
block, dfsClient.clientName);
|
dfsClient.clientName);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** update pipeline at the namenode */
|
/** update pipeline at the namenode */
|
||||||
|
@ -1437,12 +1429,12 @@ class DataStreamer extends Daemon {
|
||||||
* Must get block ID and the IDs of the destinations from the namenode.
|
* Must get block ID and the IDs of the destinations from the namenode.
|
||||||
* Returns the list of target datanodes.
|
* Returns the list of target datanodes.
|
||||||
*/
|
*/
|
||||||
private LocatedBlock nextBlockOutputStream() throws IOException {
|
protected LocatedBlock nextBlockOutputStream() throws IOException {
|
||||||
LocatedBlock lb = null;
|
LocatedBlock lb;
|
||||||
DatanodeInfo[] nodes = null;
|
DatanodeInfo[] nodes;
|
||||||
StorageType[] storageTypes = null;
|
StorageType[] storageTypes;
|
||||||
int count = dfsClient.getConf().getNumBlockWriteRetry();
|
int count = dfsClient.getConf().getNumBlockWriteRetry();
|
||||||
boolean success = false;
|
boolean success;
|
||||||
ExtendedBlock oldBlock = block;
|
ExtendedBlock oldBlock = block;
|
||||||
do {
|
do {
|
||||||
errorState.reset();
|
errorState.reset();
|
||||||
|
@ -1493,7 +1485,6 @@ class DataStreamer extends Daemon {
|
||||||
LOG.info("nodes are empty for write pipeline of " + block);
|
LOG.info("nodes are empty for write pipeline of " + block);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
Status pipelineStatus = SUCCESS;
|
|
||||||
String firstBadLink = "";
|
String firstBadLink = "";
|
||||||
boolean checkRestart = false;
|
boolean checkRestart = false;
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
|
@ -1528,25 +1519,26 @@ class DataStreamer extends Daemon {
|
||||||
// Xmit header info to datanode
|
// Xmit header info to datanode
|
||||||
//
|
//
|
||||||
|
|
||||||
BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage(): stage;
|
BlockConstructionStage bcs = recoveryFlag ?
|
||||||
|
stage.getRecoveryStage() : stage;
|
||||||
|
|
||||||
// We cannot change the block length in 'block' as it counts the number
|
// We cannot change the block length in 'block' as it counts the number
|
||||||
// of bytes ack'ed.
|
// of bytes ack'ed.
|
||||||
ExtendedBlock blockCopy = new ExtendedBlock(block);
|
ExtendedBlock blockCopy = new ExtendedBlock(block);
|
||||||
blockCopy.setNumBytes(stat.getBlockSize());
|
blockCopy.setNumBytes(stat.getBlockSize());
|
||||||
|
|
||||||
boolean[] targetPinnings = getPinnings(nodes, true);
|
boolean[] targetPinnings = getPinnings(nodes);
|
||||||
// send the request
|
// send the request
|
||||||
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
|
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
|
||||||
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
||||||
nodes.length, block.getNumBytes(), bytesSent, newGS,
|
nodes.length, block.getNumBytes(), bytesSent, newGS,
|
||||||
checksum4WriteBlock, cachingStrategy.get(), isLazyPersistFile,
|
checksum4WriteBlock, cachingStrategy.get(), isLazyPersistFile,
|
||||||
(targetPinnings == null ? false : targetPinnings[0]), targetPinnings);
|
(targetPinnings != null && targetPinnings[0]), targetPinnings);
|
||||||
|
|
||||||
// receive ack for connect
|
// receive ack for connect
|
||||||
BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(
|
BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(
|
||||||
PBHelperClient.vintPrefixed(blockReplyStream));
|
PBHelperClient.vintPrefixed(blockReplyStream));
|
||||||
pipelineStatus = resp.getStatus();
|
Status pipelineStatus = resp.getStatus();
|
||||||
firstBadLink = resp.getFirstBadLink();
|
firstBadLink = resp.getFirstBadLink();
|
||||||
|
|
||||||
// Got an restart OOB ack.
|
// Got an restart OOB ack.
|
||||||
|
@ -1571,7 +1563,8 @@ class DataStreamer extends Daemon {
|
||||||
if (!errorState.isRestartingNode()) {
|
if (!errorState.isRestartingNode()) {
|
||||||
LOG.info("Exception in createBlockOutputStream", ie);
|
LOG.info("Exception in createBlockOutputStream", ie);
|
||||||
}
|
}
|
||||||
if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
|
if (ie instanceof InvalidEncryptionKeyException &&
|
||||||
|
refetchEncryptionKey > 0) {
|
||||||
LOG.info("Will fetch a new encryption key and retry, "
|
LOG.info("Will fetch a new encryption key and retry, "
|
||||||
+ "encryption key was invalid when connecting to "
|
+ "encryption key was invalid when connecting to "
|
||||||
+ nodes[0] + " : " + ie);
|
+ nodes[0] + " : " + ie);
|
||||||
|
@ -1593,14 +1586,15 @@ class DataStreamer extends Daemon {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
assert checkRestart == false;
|
assert !checkRestart;
|
||||||
errorState.setBadNodeIndex(0);
|
errorState.setBadNodeIndex(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
final int i = errorState.getBadNodeIndex();
|
final int i = errorState.getBadNodeIndex();
|
||||||
// Check whether there is a restart worth waiting for.
|
// Check whether there is a restart worth waiting for.
|
||||||
if (checkRestart && shouldWaitForRestart(i)) {
|
if (checkRestart && shouldWaitForRestart(i)) {
|
||||||
errorState.initRestartingNode(i, "Datanode " + i + " is restarting: " + nodes[i]);
|
errorState.initRestartingNode(i, "Datanode " + i + " is restarting: "
|
||||||
|
+ nodes[i]);
|
||||||
}
|
}
|
||||||
errorState.setError(true);
|
errorState.setError(true);
|
||||||
lastException.set(ie);
|
lastException.set(ie);
|
||||||
|
@ -1610,7 +1604,6 @@ class DataStreamer extends Daemon {
|
||||||
IOUtils.closeSocket(s);
|
IOUtils.closeSocket(s);
|
||||||
s = null;
|
s = null;
|
||||||
IOUtils.closeStream(out);
|
IOUtils.closeStream(out);
|
||||||
out = null;
|
|
||||||
IOUtils.closeStream(blockReplyStream);
|
IOUtils.closeStream(blockReplyStream);
|
||||||
blockReplyStream = null;
|
blockReplyStream = null;
|
||||||
}
|
}
|
||||||
|
@ -1619,19 +1612,18 @@ class DataStreamer extends Daemon {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
|
private boolean[] getPinnings(DatanodeInfo[] nodes) {
|
||||||
if (favoredNodes == null) {
|
if (favoredNodes == null) {
|
||||||
return null;
|
return null;
|
||||||
} else {
|
} else {
|
||||||
boolean[] pinnings = new boolean[nodes.length];
|
boolean[] pinnings = new boolean[nodes.length];
|
||||||
HashSet<String> favoredSet =
|
HashSet<String> favoredSet = new HashSet<>(Arrays.asList(favoredNodes));
|
||||||
new HashSet<String>(Arrays.asList(favoredNodes));
|
|
||||||
for (int i = 0; i < nodes.length; i++) {
|
for (int i = 0; i < nodes.length; i++) {
|
||||||
pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
|
pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
|
||||||
LOG.debug("{} was chosen by name node (favored={}).",
|
LOG.debug("{} was chosen by name node (favored={}).",
|
||||||
nodes[i].getXferAddrWithHostname(), pinnings[i]);
|
nodes[i].getXferAddrWithHostname(), pinnings[i]);
|
||||||
}
|
}
|
||||||
if (shouldLog && !favoredSet.isEmpty()) {
|
if (!favoredSet.isEmpty()) {
|
||||||
// There is one or more favored nodes that were not allocated.
|
// There is one or more favored nodes that were not allocated.
|
||||||
LOG.warn("These favored nodes were specified but not chosen: "
|
LOG.warn("These favored nodes were specified but not chosen: "
|
||||||
+ favoredSet + " Specified favored nodes: "
|
+ favoredSet + " Specified favored nodes: "
|
||||||
|
@ -1777,7 +1769,7 @@ class DataStreamer extends Daemon {
|
||||||
* For heartbeat packets, create buffer directly by new byte[]
|
* For heartbeat packets, create buffer directly by new byte[]
|
||||||
* since heartbeats should not be blocked.
|
* since heartbeats should not be blocked.
|
||||||
*/
|
*/
|
||||||
private DFSPacket createHeartbeatPacket() throws InterruptedIOException {
|
private DFSPacket createHeartbeatPacket() {
|
||||||
final byte[] buf = new byte[PacketHeader.PKT_MAX_HEADER_LEN];
|
final byte[] buf = new byte[PacketHeader.PKT_MAX_HEADER_LEN];
|
||||||
return new DFSPacket(buf, 0, 0, DFSPacket.HEART_BEAT_SEQNO, 0, false);
|
return new DFSPacket(buf, 0, 0, DFSPacket.HEART_BEAT_SEQNO, 0, false);
|
||||||
}
|
}
|
||||||
|
@ -1789,7 +1781,8 @@ class DataStreamer extends Daemon {
|
||||||
.removalListener(new RemovalListener<DatanodeInfo, DatanodeInfo>() {
|
.removalListener(new RemovalListener<DatanodeInfo, DatanodeInfo>() {
|
||||||
@Override
|
@Override
|
||||||
public void onRemoval(
|
public void onRemoval(
|
||||||
RemovalNotification<DatanodeInfo, DatanodeInfo> notification) {
|
@Nonnull RemovalNotification<DatanodeInfo, DatanodeInfo>
|
||||||
|
notification) {
|
||||||
LOG.info("Removing node " + notification.getKey()
|
LOG.info("Removing node " + notification.getKey()
|
||||||
+ " from the excluded nodes list");
|
+ " from the excluded nodes list");
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,6 @@ import org.apache.hadoop.fs.CreateFlag;
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.FSLinkResolver;
|
import org.apache.hadoop.fs.FSLinkResolver;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
|
||||||
import org.apache.hadoop.fs.FileChecksum;
|
import org.apache.hadoop.fs.FileChecksum;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
@ -52,7 +51,6 @@ import org.apache.hadoop.fs.LocatedFileStatus;
|
||||||
import org.apache.hadoop.fs.Options;
|
import org.apache.hadoop.fs.Options;
|
||||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||||
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
||||||
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.PathFilter;
|
import org.apache.hadoop.fs.PathFilter;
|
||||||
import org.apache.hadoop.fs.RemoteIterator;
|
import org.apache.hadoop.fs.RemoteIterator;
|
||||||
|
@ -88,7 +86,6 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
|
||||||
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.net.NetUtils;
|
import org.apache.hadoop.net.NetUtils;
|
||||||
import org.apache.hadoop.security.AccessControlException;
|
|
||||||
import org.apache.hadoop.security.Credentials;
|
import org.apache.hadoop.security.Credentials;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
|
@ -174,7 +171,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
String result = fixRelativePart(dir).toUri().getPath();
|
String result = fixRelativePart(dir).toUri().getPath();
|
||||||
if (!DFSUtilClient.isValidName(result)) {
|
if (!DFSUtilClient.isValidName(result)) {
|
||||||
throw new IllegalArgumentException("Invalid DFS directory name " +
|
throw new IllegalArgumentException("Invalid DFS directory name " +
|
||||||
result);
|
result);
|
||||||
}
|
}
|
||||||
workingDir = fixRelativePart(dir);
|
workingDir = fixRelativePart(dir);
|
||||||
}
|
}
|
||||||
|
@ -198,7 +195,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
String result = file.toUri().getPath();
|
String result = file.toUri().getPath();
|
||||||
if (!DFSUtilClient.isValidName(result)) {
|
if (!DFSUtilClient.isValidName(result)) {
|
||||||
throw new IllegalArgumentException("Pathname " + result + " from " +
|
throw new IllegalArgumentException("Pathname " + result + " from " +
|
||||||
file+" is not a valid DFS filename.");
|
file+" is not a valid DFS filename.");
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
@ -282,8 +279,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException{
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.recoverLease(getPathName(p));
|
return dfs.recoverLease(getPathName(p));
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -306,10 +302,9 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FSDataInputStream>() {
|
return new FileSystemLinkResolver<FSDataInputStream>() {
|
||||||
@Override
|
@Override
|
||||||
public FSDataInputStream doCall(final Path p)
|
public FSDataInputStream doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
final DFSInputStream dfsis =
|
final DFSInputStream dfsis =
|
||||||
dfs.open(getPathName(p), bufferSize, verifyChecksum);
|
dfs.open(getPathName(p), bufferSize, verifyChecksum);
|
||||||
return dfs.createWrappedInputStream(dfsis);
|
return dfs.createWrappedInputStream(dfsis);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -413,13 +408,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
final FsPermission permission, final boolean overwrite,
|
final FsPermission permission, final boolean overwrite,
|
||||||
final int bufferSize, final short replication, final long blockSize,
|
final int bufferSize, final short replication, final long blockSize,
|
||||||
final Progressable progress, final InetSocketAddress[] favoredNodes)
|
final Progressable progress, final InetSocketAddress[] favoredNodes)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<HdfsDataOutputStream>() {
|
return new FileSystemLinkResolver<HdfsDataOutputStream>() {
|
||||||
@Override
|
@Override
|
||||||
public HdfsDataOutputStream doCall(final Path p)
|
public HdfsDataOutputStream doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
final DFSOutputStream out = dfs.create(getPathName(f), permission,
|
final DFSOutputStream out = dfs.create(getPathName(f), permission,
|
||||||
overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
|
overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
|
||||||
: EnumSet.of(CreateFlag.CREATE),
|
: EnumSet.of(CreateFlag.CREATE),
|
||||||
|
@ -444,18 +438,18 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream create(final Path f, final FsPermission permission,
|
public FSDataOutputStream create(final Path f, final FsPermission permission,
|
||||||
final EnumSet<CreateFlag> cflags, final int bufferSize,
|
final EnumSet<CreateFlag> cflags, final int bufferSize,
|
||||||
final short replication, final long blockSize, final Progressable progress,
|
final short replication, final long blockSize,
|
||||||
final ChecksumOpt checksumOpt) throws IOException {
|
final Progressable progress, final ChecksumOpt checksumOpt)
|
||||||
|
throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FSDataOutputStream>() {
|
return new FileSystemLinkResolver<FSDataOutputStream>() {
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream doCall(final Path p)
|
public FSDataOutputStream doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
||||||
cflags, replication, blockSize, progress, bufferSize,
|
cflags, replication, blockSize, progress, bufferSize,
|
||||||
checksumOpt);
|
checksumOpt);
|
||||||
return dfs.createWrappedOutputStream(dfsos, statistics);
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -469,14 +463,14 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected HdfsDataOutputStream primitiveCreate(Path f,
|
protected HdfsDataOutputStream primitiveCreate(Path f,
|
||||||
FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
|
FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
|
||||||
short replication, long blockSize, Progressable progress,
|
short replication, long blockSize, Progressable progress,
|
||||||
ChecksumOpt checksumOpt) throws IOException {
|
ChecksumOpt checksumOpt) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final DFSOutputStream dfsos = dfs.primitiveCreate(
|
final DFSOutputStream dfsos = dfs.primitiveCreate(
|
||||||
getPathName(fixRelativePart(f)),
|
getPathName(fixRelativePart(f)),
|
||||||
absolutePermission, flag, true, replication, blockSize,
|
absolutePermission, flag, true, replication, blockSize,
|
||||||
progress, bufferSize, checksumOpt);
|
progress, bufferSize, checksumOpt);
|
||||||
return dfs.createWrappedOutputStream(dfsos, statistics);
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -495,10 +489,9 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FSDataOutputStream>() {
|
return new FileSystemLinkResolver<FSDataOutputStream>() {
|
||||||
@Override
|
@Override
|
||||||
public FSDataOutputStream doCall(final Path p) throws IOException,
|
public FSDataOutputStream doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
final DFSOutputStream dfsos = dfs.create(getPathName(p), permission,
|
||||||
flag, false, replication, blockSize, progress, bufferSize, null);
|
flag, false, replication, blockSize, progress, bufferSize, null);
|
||||||
return dfs.createWrappedOutputStream(dfsos, statistics);
|
return dfs.createWrappedOutputStream(dfsos, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -512,15 +505,13 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean setReplication(Path src,
|
public boolean setReplication(Path src, final short replication)
|
||||||
final short replication
|
throws IOException {
|
||||||
) throws IOException {
|
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.setReplication(getPathName(p), replication);
|
return dfs.setReplication(getPathName(p), replication);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -544,8 +535,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setStoragePolicy(getPathName(p), policyName);
|
dfs.setStoragePolicy(getPathName(p), policyName);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -571,7 +561,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public BlockStoragePolicySpi next(final FileSystem fs, final Path p)
|
public BlockStoragePolicySpi next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.getStoragePolicy(p);
|
return fs.getStoragePolicy(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -585,7 +575,6 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Deprecated. Prefer {@link FileSystem#getAllStoragePolicies()}
|
* Deprecated. Prefer {@link FileSystem#getAllStoragePolicies()}
|
||||||
* @return
|
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
@Deprecated
|
@Deprecated
|
||||||
|
@ -662,8 +651,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
// Keep trying to resolve the destination
|
// Keep trying to resolve the destination
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.rename(getPathName(source), getPathName(p));
|
return dfs.rename(getPathName(source), getPathName(p));
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -695,8 +683,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
// Keep trying to resolve the destination
|
// Keep trying to resolve the destination
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.rename(getPathName(source), getPathName(p), options);
|
dfs.rename(getPathName(source), getPathName(p), options);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -716,8 +703,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.truncate(getPathName(p), newLength);
|
return dfs.truncate(getPathName(p), newLength);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -734,8 +720,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.delete(getPathName(p), recursive);
|
return dfs.delete(getPathName(p), recursive);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -752,8 +737,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<ContentSummary>() {
|
return new FileSystemLinkResolver<ContentSummary>() {
|
||||||
@Override
|
@Override
|
||||||
public ContentSummary doCall(final Path p)
|
public ContentSummary doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.getContentSummary(getPathName(p));
|
return dfs.getContentSummary(getPathName(p));
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -765,15 +749,15 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Set a directory's quotas
|
/** Set a directory's quotas
|
||||||
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long, StorageType)
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String,
|
||||||
|
* long, long, StorageType)
|
||||||
*/
|
*/
|
||||||
public void setQuota(Path src, final long namespaceQuota,
|
public void setQuota(Path src, final long namespaceQuota,
|
||||||
final long storagespaceQuota) throws IOException {
|
final long storagespaceQuota) throws IOException {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setQuota(getPathName(p), namespaceQuota, storagespaceQuota);
|
dfs.setQuota(getPathName(p), namespaceQuota, storagespaceQuota);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -795,22 +779,21 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* @param quota value of the specific storage type quota to be modified.
|
* @param quota value of the specific storage type quota to be modified.
|
||||||
* Maybe {@link HdfsConstants#QUOTA_RESET} to clear quota by storage type.
|
* Maybe {@link HdfsConstants#QUOTA_RESET} to clear quota by storage type.
|
||||||
*/
|
*/
|
||||||
public void setQuotaByStorageType(
|
public void setQuotaByStorageType(Path src, final StorageType type,
|
||||||
Path src, final StorageType type, final long quota)
|
final long quota)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setQuotaByStorageType(getPathName(p), type, quota);
|
dfs.setQuotaByStorageType(getPathName(p), type, quota);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Void next(final FileSystem fs, final Path p)
|
public Void next(final FileSystem fs, final Path p)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// setQuotaByStorageType is not defined in FileSystem, so we only can resolve
|
// setQuotaByStorageType is not defined in FileSystem, so we only can
|
||||||
// within this DFS
|
// resolve within this DFS
|
||||||
return doCall(p);
|
return doCall(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -840,9 +823,9 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
// The directory size is too big that it needs to fetch more
|
// The directory size is too big that it needs to fetch more
|
||||||
// estimate the total number of entries in the directory
|
// estimate the total number of entries in the directory
|
||||||
int totalNumEntries =
|
int totalNumEntries =
|
||||||
partialListing.length + thisListing.getRemainingEntries();
|
partialListing.length + thisListing.getRemainingEntries();
|
||||||
ArrayList<FileStatus> listing =
|
ArrayList<FileStatus> listing =
|
||||||
new ArrayList<FileStatus>(totalNumEntries);
|
new ArrayList<>(totalNumEntries);
|
||||||
// add the first batch of entries to the array list
|
// add the first batch of entries to the array list
|
||||||
for (HdfsFileStatus fileStatus : partialListing) {
|
for (HdfsFileStatus fileStatus : partialListing) {
|
||||||
listing.add(fileStatus.makeQualified(getUri(), p));
|
listing.add(fileStatus.makeQualified(getUri(), p));
|
||||||
|
@ -880,8 +863,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
return new FileSystemLinkResolver<FileStatus[]>() {
|
return new FileSystemLinkResolver<FileStatus[]>() {
|
||||||
@Override
|
@Override
|
||||||
public FileStatus[] doCall(final Path p)
|
public FileStatus[] doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return listStatusInternal(p);
|
return listStatusInternal(p);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -895,18 +877,18 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@Override
|
@Override
|
||||||
protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path p,
|
protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path p,
|
||||||
final PathFilter filter)
|
final PathFilter filter)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
return new FileSystemLinkResolver<RemoteIterator<LocatedFileStatus>>() {
|
return new FileSystemLinkResolver<RemoteIterator<LocatedFileStatus>>() {
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<LocatedFileStatus> doCall(final Path p)
|
public RemoteIterator<LocatedFileStatus> doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return new DirListingIterator<LocatedFileStatus>(p, filter, true);
|
return new DirListingIterator<>(p, filter, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<LocatedFileStatus> next(final FileSystem fs, final Path p)
|
public RemoteIterator<LocatedFileStatus> next(final FileSystem fs,
|
||||||
throws IOException {
|
final Path p) throws IOException {
|
||||||
if (fs instanceof DistributedFileSystem) {
|
if (fs instanceof DistributedFileSystem) {
|
||||||
return ((DistributedFileSystem)fs).listLocatedStatus(p, filter);
|
return ((DistributedFileSystem)fs).listLocatedStatus(p, filter);
|
||||||
}
|
}
|
||||||
|
@ -929,19 +911,19 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<FileStatus> listStatusIterator(final Path p)
|
public RemoteIterator<FileStatus> listStatusIterator(final Path p)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
return new FileSystemLinkResolver<RemoteIterator<FileStatus>>() {
|
return new FileSystemLinkResolver<RemoteIterator<FileStatus>>() {
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<FileStatus> doCall(final Path p)
|
public RemoteIterator<FileStatus> doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return new DirListingIterator<FileStatus>(p, false);
|
return new DirListingIterator<>(p, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<FileStatus> next(final FileSystem fs, final Path p)
|
public RemoteIterator<FileStatus> next(final FileSystem fs, final Path p)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return ((DistributedFileSystem)fs).listStatusIterator(p);
|
return ((DistributedFileSystem)fs).listStatusIterator(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
|
||||||
|
@ -957,7 +939,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* @param <T> the type of the file status
|
* @param <T> the type of the file status
|
||||||
*/
|
*/
|
||||||
private class DirListingIterator<T extends FileStatus>
|
private class DirListingIterator<T extends FileStatus>
|
||||||
implements RemoteIterator<T> {
|
implements RemoteIterator<T> {
|
||||||
private DirectoryListing thisListing;
|
private DirectoryListing thisListing;
|
||||||
private int i;
|
private int i;
|
||||||
private Path p;
|
private Path p;
|
||||||
|
@ -999,7 +981,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
} else {
|
} else {
|
||||||
next = (T)fileStat.makeQualified(getUri(), p);
|
next = (T)fileStat.makeQualified(getUri(), p);
|
||||||
}
|
}
|
||||||
// apply filter if not null
|
// apply filter if not null
|
||||||
if (filter == null || filter.accept(next.getPath())) {
|
if (filter == null || filter.accept(next.getPath())) {
|
||||||
curStat = next;
|
curStat = next;
|
||||||
}
|
}
|
||||||
|
@ -1074,8 +1056,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.mkdirs(getPathName(p), permission, createParent);
|
return dfs.mkdirs(getPathName(p), permission, createParent);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1096,7 +1077,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation")
|
||||||
@Override
|
@Override
|
||||||
protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
|
protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
return dfs.primitiveMkdir(getPathName(f), absolutePermission);
|
return dfs.primitiveMkdir(getPathName(f), absolutePermission);
|
||||||
}
|
}
|
||||||
|
@ -1213,7 +1194,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
|
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return new CorruptFileBlockIterator(dfs, path);
|
return new CorruptFileBlockIterator(dfs, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1223,8 +1204,8 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return datanode statistics for the given type. */
|
/** @return datanode statistics for the given type. */
|
||||||
public DatanodeInfo[] getDataNodeStats(final DatanodeReportType type
|
public DatanodeInfo[] getDataNodeStats(final DatanodeReportType type)
|
||||||
) throws IOException {
|
throws IOException {
|
||||||
return dfs.datanodeReport(type);
|
return dfs.datanodeReport(type);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1235,7 +1216,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* HdfsConstants.SafeModeAction,boolean)
|
* HdfsConstants.SafeModeAction,boolean)
|
||||||
*/
|
*/
|
||||||
public boolean setSafeMode(HdfsConstants.SafeModeAction action)
|
public boolean setSafeMode(HdfsConstants.SafeModeAction action)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return setSafeMode(action, false);
|
return setSafeMode(action, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1260,7 +1241,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
*
|
*
|
||||||
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace()
|
||||||
*/
|
*/
|
||||||
public void saveNamespace() throws AccessControlException, IOException {
|
public void saveNamespace() throws IOException {
|
||||||
dfs.saveNamespace();
|
dfs.saveNamespace();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1270,7 +1251,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits()
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits()
|
||||||
* @return the transaction ID of the newly created segment
|
* @return the transaction ID of the newly created segment
|
||||||
*/
|
*/
|
||||||
public long rollEdits() throws AccessControlException, IOException {
|
public long rollEdits() throws IOException {
|
||||||
return dfs.rollEdits();
|
return dfs.rollEdits();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1279,8 +1260,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
*
|
*
|
||||||
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#restoreFailedStorage(String arg)
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#restoreFailedStorage(String arg)
|
||||||
*/
|
*/
|
||||||
public boolean restoreFailedStorage(String arg)
|
public boolean restoreFailedStorage(String arg) throws IOException {
|
||||||
throws AccessControlException, IOException {
|
|
||||||
return dfs.restoreFailedStorage(arg);
|
return dfs.restoreFailedStorage(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1332,8 +1312,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FileStatus>() {
|
return new FileSystemLinkResolver<FileStatus>() {
|
||||||
@Override
|
@Override
|
||||||
public FileStatus doCall(final Path p) throws IOException,
|
public FileStatus doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
HdfsFileStatus fi = dfs.getFileInfo(getPathName(p));
|
HdfsFileStatus fi = dfs.getFileInfo(getPathName(p));
|
||||||
if (fi != null) {
|
if (fi != null) {
|
||||||
return fi.makeQualified(getUri(), p);
|
return fi.makeQualified(getUri(), p);
|
||||||
|
@ -1352,10 +1331,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@SuppressWarnings("deprecation")
|
@SuppressWarnings("deprecation")
|
||||||
@Override
|
@Override
|
||||||
public void createSymlink(final Path target, final Path link,
|
public void createSymlink(final Path target, final Path link,
|
||||||
final boolean createParent) throws AccessControlException,
|
final boolean createParent) throws IOException {
|
||||||
FileAlreadyExistsException, FileNotFoundException,
|
|
||||||
ParentNotDirectoryException, UnsupportedFileSystemException,
|
|
||||||
IOException {
|
|
||||||
if (!FileSystem.areSymlinksEnabled()) {
|
if (!FileSystem.areSymlinksEnabled()) {
|
||||||
throw new UnsupportedOperationException("Symlinks not supported");
|
throw new UnsupportedOperationException("Symlinks not supported");
|
||||||
}
|
}
|
||||||
|
@ -1363,14 +1339,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
final Path absF = fixRelativePart(link);
|
final Path absF = fixRelativePart(link);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p) throws IOException,
|
public Void doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
dfs.createSymlink(target.toString(), getPathName(p), createParent);
|
dfs.createSymlink(target.toString(), getPathName(p), createParent);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Void next(final FileSystem fs, final Path p)
|
public Void next(final FileSystem fs, final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
fs.createSymlink(target, p, createParent);
|
fs.createSymlink(target, p, createParent);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1383,15 +1357,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public FileStatus getFileLinkStatus(final Path f)
|
public FileStatus getFileLinkStatus(final Path f) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
UnsupportedFileSystemException, IOException {
|
|
||||||
statistics.incrementReadOps(1);
|
statistics.incrementReadOps(1);
|
||||||
final Path absF = fixRelativePart(f);
|
final Path absF = fixRelativePart(f);
|
||||||
FileStatus status = new FileSystemLinkResolver<FileStatus>() {
|
FileStatus status = new FileSystemLinkResolver<FileStatus>() {
|
||||||
@Override
|
@Override
|
||||||
public FileStatus doCall(final Path p) throws IOException,
|
public FileStatus doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
HdfsFileStatus fi = dfs.getFileLinkInfo(getPathName(p));
|
HdfsFileStatus fi = dfs.getFileLinkInfo(getPathName(p));
|
||||||
if (fi != null) {
|
if (fi != null) {
|
||||||
return fi.makeQualified(getUri(), p);
|
return fi.makeQualified(getUri(), p);
|
||||||
|
@ -1401,7 +1372,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public FileStatus next(final FileSystem fs, final Path p)
|
public FileStatus next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.getFileLinkStatus(p);
|
return fs.getFileLinkStatus(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -1415,14 +1386,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Path getLinkTarget(final Path f) throws AccessControlException,
|
public Path getLinkTarget(final Path f) throws IOException {
|
||||||
FileNotFoundException, UnsupportedFileSystemException, IOException {
|
|
||||||
statistics.incrementReadOps(1);
|
statistics.incrementReadOps(1);
|
||||||
final Path absF = fixRelativePart(f);
|
final Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<Path>() {
|
return new FileSystemLinkResolver<Path>() {
|
||||||
@Override
|
@Override
|
||||||
public Path doCall(final Path p) throws IOException,
|
public Path doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
HdfsFileStatus fi = dfs.getFileLinkInfo(getPathName(p));
|
HdfsFileStatus fi = dfs.getFileLinkInfo(getPathName(p));
|
||||||
if (fi != null) {
|
if (fi != null) {
|
||||||
return fi.makeQualified(getUri(), p).getSymlink();
|
return fi.makeQualified(getUri(), p).getSymlink();
|
||||||
|
@ -1431,8 +1400,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Path next(final FileSystem fs, final Path p)
|
public Path next(final FileSystem fs, final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return fs.getLinkTarget(p);
|
return fs.getLinkTarget(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -1454,8 +1422,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FileChecksum>() {
|
return new FileSystemLinkResolver<FileChecksum>() {
|
||||||
@Override
|
@Override
|
||||||
public FileChecksum doCall(final Path p)
|
public FileChecksum doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE);
|
return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1474,8 +1441,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(f);
|
Path absF = fixRelativePart(f);
|
||||||
return new FileSystemLinkResolver<FileChecksum>() {
|
return new FileSystemLinkResolver<FileChecksum>() {
|
||||||
@Override
|
@Override
|
||||||
public FileChecksum doCall(final Path p)
|
public FileChecksum doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.getFileChecksum(getPathName(p), length);
|
return dfs.getFileChecksum(getPathName(p), length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1483,7 +1449,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
public FileChecksum next(final FileSystem fs, final Path p)
|
public FileChecksum next(final FileSystem fs, final Path p)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (fs instanceof DistributedFileSystem) {
|
if (fs instanceof DistributedFileSystem) {
|
||||||
return ((DistributedFileSystem) fs).getFileChecksum(p, length);
|
return fs.getFileChecksum(p, length);
|
||||||
} else {
|
} else {
|
||||||
throw new UnsupportedFileSystemException(
|
throw new UnsupportedFileSystemException(
|
||||||
"getFileChecksum(Path, long) is not supported by "
|
"getFileChecksum(Path, long) is not supported by "
|
||||||
|
@ -1495,13 +1461,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setPermission(Path p, final FsPermission permission
|
public void setPermission(Path p, final FsPermission permission
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setPermission(getPathName(p), permission);
|
dfs.setPermission(getPathName(p), permission);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1516,8 +1481,8 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setOwner(Path p, final String username, final String groupname
|
public void setOwner(Path p, final String username, final String groupname)
|
||||||
) throws IOException {
|
throws IOException {
|
||||||
if (username == null && groupname == null) {
|
if (username == null && groupname == null) {
|
||||||
throw new IOException("username == null && groupname == null");
|
throw new IOException("username == null && groupname == null");
|
||||||
}
|
}
|
||||||
|
@ -1525,8 +1490,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setOwner(getPathName(p), username, groupname);
|
dfs.setOwner(getPathName(p), username, groupname);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1541,14 +1505,13 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setTimes(Path p, final long mtime, final long atime
|
public void setTimes(Path p, final long mtime, final long atime)
|
||||||
) throws IOException {
|
throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
Path absF = fixRelativePart(p);
|
Path absF = fixRelativePart(p);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.setTimes(getPathName(p), mtime, atime);
|
dfs.setTimes(getPathName(p), mtime, atime);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1571,9 +1534,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
@Override
|
@Override
|
||||||
public Token<DelegationTokenIdentifier> getDelegationToken(String renewer)
|
public Token<DelegationTokenIdentifier> getDelegationToken(String renewer)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Token<DelegationTokenIdentifier> result =
|
return dfs.getDelegationToken(renewer == null ? null : new Text(renewer));
|
||||||
dfs.getDelegationToken(renewer == null ? null : new Text(renewer));
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1627,8 +1588,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.allowSnapshot(getPathName(p));
|
dfs.allowSnapshot(getPathName(p));
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1654,8 +1614,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.disallowSnapshot(getPathName(p));
|
dfs.disallowSnapshot(getPathName(p));
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1682,8 +1641,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
return new FileSystemLinkResolver<Path>() {
|
return new FileSystemLinkResolver<Path>() {
|
||||||
@Override
|
@Override
|
||||||
public Path doCall(final Path p)
|
public Path doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return new Path(dfs.createSnapshot(getPathName(p), snapshotName));
|
return new Path(dfs.createSnapshot(getPathName(p), snapshotName));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1708,8 +1666,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.renameSnapshot(getPathName(p), snapshotOldName, snapshotNewName);
|
dfs.renameSnapshot(getPathName(p), snapshotOldName, snapshotNewName);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1745,8 +1702,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(snapshotDir);
|
Path absF = fixRelativePart(snapshotDir);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
dfs.deleteSnapshot(getPathName(p), snapshotName);
|
dfs.deleteSnapshot(getPathName(p), snapshotName);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1778,8 +1734,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(snapshotDir);
|
Path absF = fixRelativePart(snapshotDir);
|
||||||
return new FileSystemLinkResolver<SnapshotDiffReport>() {
|
return new FileSystemLinkResolver<SnapshotDiffReport>() {
|
||||||
@Override
|
@Override
|
||||||
public SnapshotDiffReport doCall(final Path p)
|
public SnapshotDiffReport doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.getSnapshotDiffReport(getPathName(p), fromSnapshot,
|
return dfs.getSnapshotDiffReport(getPathName(p), fromSnapshot,
|
||||||
toSnapshot);
|
toSnapshot);
|
||||||
}
|
}
|
||||||
|
@ -1812,8 +1767,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
return new FileSystemLinkResolver<Boolean>() {
|
return new FileSystemLinkResolver<Boolean>() {
|
||||||
@Override
|
@Override
|
||||||
public Boolean doCall(final Path p)
|
public Boolean doCall(final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return dfs.isFileClosed(getPathName(p));
|
return dfs.isFileClosed(getPathName(p));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2043,8 +1997,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Void next(final FileSystem fs, final Path p)
|
public Void next(final FileSystem fs, final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
fs.removeDefaultAcl(p);
|
fs.removeDefaultAcl(p);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -2064,8 +2017,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Void next(final FileSystem fs, final Path p)
|
public Void next(final FileSystem fs, final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
fs.removeAcl(p);
|
fs.removeAcl(p);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -2076,7 +2028,8 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void setAcl(Path path, final List<AclEntry> aclSpec) throws IOException {
|
public void setAcl(Path path, final List<AclEntry> aclSpec)
|
||||||
|
throws IOException {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -2106,7 +2059,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public AclStatus next(final FileSystem fs, final Path p)
|
public AclStatus next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.getAclStatus(p);
|
return fs.getAclStatus(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -2114,12 +2067,11 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
/* HDFS only */
|
/* HDFS only */
|
||||||
public void createEncryptionZone(final Path path, final String keyName)
|
public void createEncryptionZone(final Path path, final String keyName)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p) throws IOException,
|
public Void doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
dfs.createEncryptionZone(getPathName(p), keyName);
|
dfs.createEncryptionZone(getPathName(p), keyName);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -2142,13 +2094,12 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
/* HDFS only */
|
/* HDFS only */
|
||||||
public EncryptionZone getEZForPath(final Path path)
|
public EncryptionZone getEZForPath(final Path path)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Preconditions.checkNotNull(path);
|
Preconditions.checkNotNull(path);
|
||||||
Path absF = fixRelativePart(path);
|
Path absF = fixRelativePart(path);
|
||||||
return new FileSystemLinkResolver<EncryptionZone>() {
|
return new FileSystemLinkResolver<EncryptionZone>() {
|
||||||
@Override
|
@Override
|
||||||
public EncryptionZone doCall(final Path p) throws IOException,
|
public EncryptionZone doCall(final Path p) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
return dfs.getEZForPath(getPathName(p));
|
return dfs.getEZForPath(getPathName(p));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2203,8 +2154,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
return dfs.getXAttr(getPathName(p), name);
|
return dfs.getXAttr(getPathName(p), name);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public byte[] next(final FileSystem fs, final Path p)
|
public byte[] next(final FileSystem fs, final Path p) throws IOException {
|
||||||
throws IOException, UnresolvedLinkException {
|
|
||||||
return fs.getXAttr(p, name);
|
return fs.getXAttr(p, name);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -2220,7 +2170,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Map<String, byte[]> next(final FileSystem fs, final Path p)
|
public Map<String, byte[]> next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.getXAttrs(p);
|
return fs.getXAttrs(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -2237,7 +2187,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public Map<String, byte[]> next(final FileSystem fs, final Path p)
|
public Map<String, byte[]> next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.getXAttrs(p, names);
|
return fs.getXAttrs(p, names);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
@ -2245,7 +2195,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> listXAttrs(Path path)
|
public List<String> listXAttrs(Path path)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final Path absF = fixRelativePart(path);
|
final Path absF = fixRelativePart(path);
|
||||||
return new FileSystemLinkResolver<List<String>>() {
|
return new FileSystemLinkResolver<List<String>>() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -2254,7 +2204,7 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public List<String> next(final FileSystem fs, final Path p)
|
public List<String> next(final FileSystem fs, final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
return fs.listXAttrs(p);
|
return fs.listXAttrs(p);
|
||||||
}
|
}
|
||||||
}.resolve(this, absF);
|
}.resolve(this, absF);
|
||||||
|
|
|
@ -76,7 +76,6 @@ final public class ExtendedBlockId {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return new StringBuilder().append(blockId).
|
return String.valueOf(blockId) + "_" + bpId;
|
||||||
append("_").append(bpId).toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -79,7 +79,7 @@ public final class ExternalBlockReader implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int available() throws IOException {
|
public int available() {
|
||||||
// We return the amount of bytes between the current offset and the visible
|
// We return the amount of bytes between the current offset and the visible
|
||||||
// length. Some of the other block readers return a shorter length than
|
// length. Some of the other block readers return a shorter length than
|
||||||
// that. The only advantage to returning a shorter length is that the
|
// that. The only advantage to returning a shorter length is that the
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs;
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
|
@ -37,10 +36,13 @@ import com.google.common.cache.RemovalNotification;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class KeyProviderCache {
|
public class KeyProviderCache {
|
||||||
|
|
||||||
public static final Logger LOG = LoggerFactory.getLogger(KeyProviderCache.class);
|
public static final Logger LOG = LoggerFactory.getLogger(
|
||||||
|
KeyProviderCache.class);
|
||||||
|
|
||||||
private final Cache<URI, KeyProvider> cache;
|
private final Cache<URI, KeyProvider> cache;
|
||||||
|
|
||||||
|
@ -50,14 +52,14 @@ public class KeyProviderCache {
|
||||||
.removalListener(new RemovalListener<URI, KeyProvider>() {
|
.removalListener(new RemovalListener<URI, KeyProvider>() {
|
||||||
@Override
|
@Override
|
||||||
public void onRemoval(
|
public void onRemoval(
|
||||||
RemovalNotification<URI, KeyProvider> notification) {
|
@Nonnull RemovalNotification<URI, KeyProvider> notification) {
|
||||||
try {
|
try {
|
||||||
|
assert notification.getValue() != null;
|
||||||
notification.getValue().close();
|
notification.getValue().close();
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
LOG.error(
|
LOG.error(
|
||||||
"Error closing KeyProvider with uri ["
|
"Error closing KeyProvider with uri ["
|
||||||
+ notification.getKey() + "]", e);
|
+ notification.getKey() + "]", e);
|
||||||
;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -83,8 +85,8 @@ public class KeyProviderCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
private URI createKeyProviderURI(Configuration conf) {
|
private URI createKeyProviderURI(Configuration conf) {
|
||||||
final String providerUriStr =
|
final String providerUriStr = conf.getTrimmed(
|
||||||
conf.getTrimmed(HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "");
|
HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "");
|
||||||
// No provider set in conf
|
// No provider set in conf
|
||||||
if (providerUriStr.isEmpty()) {
|
if (providerUriStr.isEmpty()) {
|
||||||
LOG.error("Could not find uri with key ["
|
LOG.error("Could not find uri with key ["
|
||||||
|
@ -104,9 +106,9 @@ public class KeyProviderCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public void setKeyProvider(Configuration conf, KeyProvider keyProvider)
|
public void setKeyProvider(Configuration conf, KeyProvider keyProvider) {
|
||||||
throws IOException {
|
|
||||||
URI uri = createKeyProviderURI(conf);
|
URI uri = createKeyProviderURI(conf);
|
||||||
|
assert uri != null;
|
||||||
cache.put(uri, keyProvider);
|
cache.put(uri, keyProvider);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -297,13 +297,11 @@ public class NameNodeProxiesClient {
|
||||||
* @param failoverProxyProvider Failover proxy provider
|
* @param failoverProxyProvider Failover proxy provider
|
||||||
* @return an object containing both the proxy and the associated
|
* @return an object containing both the proxy and the associated
|
||||||
* delegation token service it corresponds to
|
* delegation token service it corresponds to
|
||||||
* @throws IOException
|
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public static <T> ProxyAndInfo<T> createHAProxy(
|
public static <T> ProxyAndInfo<T> createHAProxy(
|
||||||
Configuration conf, URI nameNodeUri, Class<T> xface,
|
Configuration conf, URI nameNodeUri, Class<T> xface,
|
||||||
AbstractNNFailoverProxyProvider<T> failoverProxyProvider)
|
AbstractNNFailoverProxyProvider<T> failoverProxyProvider) {
|
||||||
throws IOException {
|
|
||||||
Preconditions.checkNotNull(failoverProxyProvider);
|
Preconditions.checkNotNull(failoverProxyProvider);
|
||||||
// HA case
|
// HA case
|
||||||
DfsClientConf config = new DfsClientConf(conf);
|
DfsClientConf config = new DfsClientConf(conf);
|
||||||
|
|
|
@ -32,7 +32,6 @@ import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.hdfs.net.Peer;
|
import org.apache.hadoop.hdfs.net.Peer;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
import org.apache.hadoop.hdfs.util.IOUtilsClient;
|
import org.apache.hadoop.hdfs.util.IOUtilsClient;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
|
||||||
import org.apache.hadoop.util.Daemon;
|
import org.apache.hadoop.util.Daemon;
|
||||||
import org.apache.hadoop.util.Time;
|
import org.apache.hadoop.util.Time;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -92,7 +91,7 @@ public class PeerCache {
|
||||||
private Daemon daemon;
|
private Daemon daemon;
|
||||||
/** A map for per user per datanode. */
|
/** A map for per user per datanode. */
|
||||||
private final LinkedListMultimap<Key, Value> multimap =
|
private final LinkedListMultimap<Key, Value> multimap =
|
||||||
LinkedListMultimap.create();
|
LinkedListMultimap.create();
|
||||||
private final int capacity;
|
private final int capacity;
|
||||||
private final long expiryPeriod;
|
private final long expiryPeriod;
|
||||||
|
|
||||||
|
@ -109,12 +108,12 @@ public class PeerCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isDaemonStarted() {
|
private boolean isDaemonStarted() {
|
||||||
return (daemon == null)? false: true;
|
return daemon != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized void startExpiryDaemon() {
|
private synchronized void startExpiryDaemon() {
|
||||||
// start daemon only if not already started
|
// start daemon only if not already started
|
||||||
if (isDaemonStarted() == true) {
|
if (isDaemonStarted()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -215,12 +214,11 @@ public class PeerCache {
|
||||||
private synchronized void evictExpired(long expiryPeriod) {
|
private synchronized void evictExpired(long expiryPeriod) {
|
||||||
while (multimap.size() != 0) {
|
while (multimap.size() != 0) {
|
||||||
Iterator<Entry<Key, Value>> iter =
|
Iterator<Entry<Key, Value>> iter =
|
||||||
multimap.entries().iterator();
|
multimap.entries().iterator();
|
||||||
Entry<Key, Value> entry = iter.next();
|
Entry<Key, Value> entry = iter.next();
|
||||||
// if oldest socket expired, remove it
|
// if oldest socket expired, remove it
|
||||||
if (entry == null ||
|
if (entry == null ||
|
||||||
Time.monotonicNow() - entry.getValue().getTime() <
|
Time.monotonicNow() - entry.getValue().getTime() < expiryPeriod) {
|
||||||
expiryPeriod) {
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
IOUtilsClient.cleanup(LOG, entry.getValue().getPeer());
|
IOUtilsClient.cleanup(LOG, entry.getValue().getPeer());
|
||||||
|
@ -235,8 +233,7 @@ public class PeerCache {
|
||||||
// We can get the oldest element immediately, because of an interesting
|
// We can get the oldest element immediately, because of an interesting
|
||||||
// property of LinkedListMultimap: its iterator traverses entries in the
|
// property of LinkedListMultimap: its iterator traverses entries in the
|
||||||
// order that they were added.
|
// order that they were added.
|
||||||
Iterator<Entry<Key, Value>> iter =
|
Iterator<Entry<Key, Value>> iter = multimap.entries().iterator();
|
||||||
multimap.entries().iterator();
|
|
||||||
if (!iter.hasNext()) {
|
if (!iter.hasNext()) {
|
||||||
throw new IllegalStateException("Cannot evict from empty cache! " +
|
throw new IllegalStateException("Cannot evict from empty cache! " +
|
||||||
"capacity: " + capacity);
|
"capacity: " + capacity);
|
||||||
|
@ -247,8 +244,8 @@ public class PeerCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Periodically check in the cache and expire the entries
|
* Periodically check in the cache and expire the entries older than
|
||||||
* older than expiryPeriod minutes
|
* expiryPeriod minutes.
|
||||||
*/
|
*/
|
||||||
private void run() throws InterruptedException {
|
private void run() throws InterruptedException {
|
||||||
for(long lastExpiryTime = Time.monotonicNow();
|
for(long lastExpiryTime = Time.monotonicNow();
|
||||||
|
|
|
@ -55,7 +55,8 @@ import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated this is an old implementation that is being left around
|
* @deprecated this is an old implementation that is being left around
|
||||||
* in case any issues spring up with the new {@link RemoteBlockReader2} implementation.
|
* in case any issues spring up with the new {@link RemoteBlockReader2}
|
||||||
|
* implementation.
|
||||||
* It will be removed in the next release.
|
* It will be removed in the next release.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
|
@ -79,7 +80,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
private final long blockId;
|
private final long blockId;
|
||||||
|
|
||||||
/** offset in block of of first chunk - may be less than startOffset
|
/** offset in block of of first chunk - may be less than startOffset
|
||||||
if startOffset is not chunk-aligned */
|
if startOffset is not chunk-aligned */
|
||||||
private final long firstChunkOffset;
|
private final long firstChunkOffset;
|
||||||
|
|
||||||
private final int bytesPerChecksum;
|
private final int bytesPerChecksum;
|
||||||
|
@ -119,7 +120,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(byte[] buf, int off, int len)
|
public synchronized int read(byte[] buf, int off, int len)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
// This has to be set here, *before* the skip, since we can
|
// This has to be set here, *before* the skip, since we can
|
||||||
// hit EOS during the skip, in the case that our entire read
|
// hit EOS during the skip, in the case that our entire read
|
||||||
|
@ -168,7 +169,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
@Override
|
@Override
|
||||||
public int read() throws IOException {
|
public int read() throws IOException {
|
||||||
throw new IOException("read() is not expected to be invoked. " +
|
throw new IOException("read() is not expected to be invoked. " +
|
||||||
"Use read(buf, off, len) instead.");
|
"Use read(buf, off, len) instead.");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -188,7 +189,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
@Override
|
@Override
|
||||||
protected long getChunkPosition(long pos) {
|
protected long getChunkPosition(long pos) {
|
||||||
throw new RuntimeException("getChunkPosition() is not supported, " +
|
throw new RuntimeException("getChunkPosition() is not supported, " +
|
||||||
"since seek is not required");
|
"since seek is not required");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -198,7 +199,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
*/
|
*/
|
||||||
private void adjustChecksumBytes(int dataLen) {
|
private void adjustChecksumBytes(int dataLen) {
|
||||||
int requiredSize =
|
int requiredSize =
|
||||||
((dataLen + bytesPerChecksum - 1)/bytesPerChecksum)*checksumSize;
|
((dataLen + bytesPerChecksum - 1)/bytesPerChecksum)*checksumSize;
|
||||||
if (checksumBytes == null || requiredSize > checksumBytes.capacity()) {
|
if (checksumBytes == null || requiredSize > checksumBytes.capacity()) {
|
||||||
checksumBytes = ByteBuffer.wrap(new byte[requiredSize]);
|
checksumBytes = ByteBuffer.wrap(new byte[requiredSize]);
|
||||||
} else {
|
} else {
|
||||||
|
@ -209,20 +210,17 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected synchronized int readChunk(long pos, byte[] buf, int offset,
|
protected synchronized int readChunk(long pos, byte[] buf, int offset,
|
||||||
int len, byte[] checksumBuf)
|
int len, byte[] checksumBuf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = tracer.
|
try (TraceScope ignored = tracer.newScope(
|
||||||
newScope("RemoteBlockReader#readChunk(" + blockId + ")");
|
"RemoteBlockReader#readChunk(" + blockId + ")")) {
|
||||||
try {
|
|
||||||
return readChunkImpl(pos, buf, offset, len, checksumBuf);
|
return readChunkImpl(pos, buf, offset, len, checksumBuf);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized int readChunkImpl(long pos, byte[] buf, int offset,
|
private synchronized int readChunkImpl(long pos, byte[] buf, int offset,
|
||||||
int len, byte[] checksumBuf)
|
int len, byte[] checksumBuf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// Read one chunk.
|
// Read one chunk.
|
||||||
if (eos) {
|
if (eos) {
|
||||||
// Already hit EOF
|
// Already hit EOF
|
||||||
|
@ -241,7 +239,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
// for the same chunk we expect to be reading from the DN.
|
// for the same chunk we expect to be reading from the DN.
|
||||||
if ( (pos + firstChunkOffset) != chunkOffset ) {
|
if ( (pos + firstChunkOffset) != chunkOffset ) {
|
||||||
throw new IOException("Mismatch in pos : " + pos + " + " +
|
throw new IOException("Mismatch in pos : " + pos + " + " +
|
||||||
firstChunkOffset + " != " + chunkOffset);
|
firstChunkOffset + " != " + chunkOffset);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read next packet if the previous packet has been read completely.
|
// Read next packet if the previous packet has been read completely.
|
||||||
|
@ -254,8 +252,8 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
|
|
||||||
// Sanity check the lengths
|
// Sanity check the lengths
|
||||||
if (!header.sanityCheck(lastSeqNo)) {
|
if (!header.sanityCheck(lastSeqNo)) {
|
||||||
throw new IOException("BlockReader: error in packet header " +
|
throw new IOException("BlockReader: error in packet header " +
|
||||||
header);
|
header);
|
||||||
}
|
}
|
||||||
|
|
||||||
lastSeqNo = header.getSeqno();
|
lastSeqNo = header.getSeqno();
|
||||||
|
@ -263,7 +261,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
adjustChecksumBytes(header.getDataLen());
|
adjustChecksumBytes(header.getDataLen());
|
||||||
if (header.getDataLen() > 0) {
|
if (header.getDataLen() > 0) {
|
||||||
IOUtils.readFully(in, checksumBytes.array(), 0,
|
IOUtils.readFully(in, checksumBytes.array(), 0,
|
||||||
checksumBytes.limit());
|
checksumBytes.limit());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -284,14 +282,14 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
// How many chunks we can fit in databuffer
|
// How many chunks we can fit in databuffer
|
||||||
// - note this is a floor since we always read full chunks
|
// - note this is a floor since we always read full chunks
|
||||||
int chunksCanFit = Math.min(len / bytesPerChecksum,
|
int chunksCanFit = Math.min(len / bytesPerChecksum,
|
||||||
checksumBuf.length / checksumSize);
|
checksumBuf.length / checksumSize);
|
||||||
|
|
||||||
// How many chunks should we read
|
// How many chunks should we read
|
||||||
checksumsToRead = Math.min(chunksLeft, chunksCanFit);
|
checksumsToRead = Math.min(chunksLeft, chunksCanFit);
|
||||||
// How many bytes should we actually read
|
// How many bytes should we actually read
|
||||||
bytesToRead = Math.min(
|
bytesToRead = Math.min(
|
||||||
checksumsToRead * bytesPerChecksum, // full chunks
|
checksumsToRead * bytesPerChecksum, // full chunks
|
||||||
dataLeft); // in case we have a partial
|
dataLeft); // in case we have a partial
|
||||||
} else {
|
} else {
|
||||||
// no checksum
|
// no checksum
|
||||||
bytesToRead = Math.min(dataLeft, len);
|
bytesToRead = Math.min(dataLeft, len);
|
||||||
|
@ -328,7 +326,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
if (!hdr.isLastPacketInBlock() ||
|
if (!hdr.isLastPacketInBlock() ||
|
||||||
hdr.getDataLen() != 0) {
|
hdr.getDataLen() != 0) {
|
||||||
throw new IOException("Expected empty end-of-read packet! Header: " +
|
throw new IOException("Expected empty end-of-read packet! Header: " +
|
||||||
hdr);
|
hdr);
|
||||||
}
|
}
|
||||||
|
|
||||||
eos = true;
|
eos = true;
|
||||||
|
@ -347,11 +345,11 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
DatanodeID datanodeID, PeerCache peerCache, Tracer tracer) {
|
DatanodeID datanodeID, PeerCache peerCache, Tracer tracer) {
|
||||||
// Path is used only for printing block and file information in debug
|
// Path is used only for printing block and file information in debug
|
||||||
super(new Path("/" + Block.BLOCK_FILE_PREFIX + blockId +
|
super(new Path("/" + Block.BLOCK_FILE_PREFIX + blockId +
|
||||||
":" + bpid + ":of:"+ file)/*too non path-like?*/,
|
":" + bpid + ":of:"+ file)/*too non path-like?*/,
|
||||||
1, verifyChecksum,
|
1, verifyChecksum,
|
||||||
checksum.getChecksumSize() > 0? checksum : null,
|
checksum.getChecksumSize() > 0? checksum : null,
|
||||||
checksum.getBytesPerChecksum(),
|
checksum.getBytesPerChecksum(),
|
||||||
checksum.getChecksumSize());
|
checksum.getChecksumSize());
|
||||||
|
|
||||||
this.isLocal = DFSUtilClient.isLocalAddress(NetUtils.
|
this.isLocal = DFSUtilClient.isLocalAddress(NetUtils.
|
||||||
createSocketAddr(datanodeID.getXferAddr()));
|
createSocketAddr(datanodeID.getXferAddr()));
|
||||||
|
@ -394,16 +392,16 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
* @return New BlockReader instance, or null on error.
|
* @return New BlockReader instance, or null on error.
|
||||||
*/
|
*/
|
||||||
public static RemoteBlockReader newBlockReader(String file,
|
public static RemoteBlockReader newBlockReader(String file,
|
||||||
ExtendedBlock block,
|
ExtendedBlock block,
|
||||||
Token<BlockTokenIdentifier> blockToken,
|
Token<BlockTokenIdentifier> blockToken,
|
||||||
long startOffset, long len,
|
long startOffset, long len,
|
||||||
int bufferSize, boolean verifyChecksum,
|
int bufferSize, boolean verifyChecksum,
|
||||||
String clientName, Peer peer,
|
String clientName, Peer peer,
|
||||||
DatanodeID datanodeID,
|
DatanodeID datanodeID,
|
||||||
PeerCache peerCache,
|
PeerCache peerCache,
|
||||||
CachingStrategy cachingStrategy,
|
CachingStrategy cachingStrategy,
|
||||||
Tracer tracer)
|
Tracer tracer)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// in and out will be closed when sock is closed (by the caller)
|
// in and out will be closed when sock is closed (by the caller)
|
||||||
final DataOutputStream out =
|
final DataOutputStream out =
|
||||||
new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
|
new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
|
||||||
|
@ -421,7 +419,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
PBHelperClient.vintPrefixed(in));
|
PBHelperClient.vintPrefixed(in));
|
||||||
RemoteBlockReader2.checkSuccess(status, peer, block, file);
|
RemoteBlockReader2.checkSuccess(status, peer, block, file);
|
||||||
ReadOpChecksumInfoProto checksumInfo =
|
ReadOpChecksumInfoProto checksumInfo =
|
||||||
status.getReadOpChecksumInfo();
|
status.getReadOpChecksumInfo();
|
||||||
DataChecksum checksum = DataTransferProtoUtil.fromProto(
|
DataChecksum checksum = DataTransferProtoUtil.fromProto(
|
||||||
checksumInfo.getChecksum());
|
checksumInfo.getChecksum());
|
||||||
//Warning when we get CHECKSUM_NULL?
|
//Warning when we get CHECKSUM_NULL?
|
||||||
|
@ -432,8 +430,8 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
|
if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
|
||||||
firstChunkOffset <= (startOffset - checksum.getBytesPerChecksum())) {
|
firstChunkOffset <= (startOffset - checksum.getBytesPerChecksum())) {
|
||||||
throw new IOException("BlockReader: error in first chunk offset (" +
|
throw new IOException("BlockReader: error in first chunk offset (" +
|
||||||
firstChunkOffset + ") startOffset is " +
|
firstChunkOffset + ") startOffset is " +
|
||||||
startOffset + " for file " + file);
|
startOffset + " for file " + file);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new RemoteBlockReader(file, block.getBlockPoolId(), block.getBlockId(),
|
return new RemoteBlockReader(file, block.getBlockPoolId(), block.getBlockId(),
|
||||||
|
@ -479,7 +477,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
// It's ok not to be able to send this. But something is probably wrong.
|
// It's ok not to be able to send this. But something is probably wrong.
|
||||||
LOG.info("Could not send read status (" + statusCode + ") to datanode " +
|
LOG.info("Could not send read status (" + statusCode + ") to datanode " +
|
||||||
peer.getRemoteAddressString() + ": " + e.getMessage());
|
peer.getRemoteAddressString() + ": " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -489,7 +487,7 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int available() throws IOException {
|
public int available() {
|
||||||
// An optimistic estimate of how much data is available
|
// An optimistic estimate of how much data is available
|
||||||
// to us without doing network I/O.
|
// to us without doing network I/O.
|
||||||
return RemoteBlockReader2.TCP_WINDOW_SIZE;
|
return RemoteBlockReader2.TCP_WINDOW_SIZE;
|
||||||
|
|
|
@ -134,18 +134,16 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(byte[] buf, int off, int len)
|
public synchronized int read(byte[] buf, int off, int len)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
UUID randomId = (LOG.isTraceEnabled() ? UUID.randomUUID() : null);
|
UUID randomId = (LOG.isTraceEnabled() ? UUID.randomUUID() : null);
|
||||||
LOG.trace("Starting read #{} file {} from datanode {}",
|
LOG.trace("Starting read #{} file {} from datanode {}",
|
||||||
randomId, filename, datanodeID.getHostName());
|
randomId, filename, datanodeID.getHostName());
|
||||||
|
|
||||||
if (curDataSlice == null || curDataSlice.remaining() == 0 && bytesNeededToFinish > 0) {
|
if (curDataSlice == null ||
|
||||||
TraceScope scope = tracer.newScope(
|
curDataSlice.remaining() == 0 && bytesNeededToFinish > 0) {
|
||||||
"RemoteBlockReader2#readNextPacket(" + blockId + ")");
|
try (TraceScope ignored = tracer.newScope(
|
||||||
try {
|
"RemoteBlockReader2#readNextPacket(" + blockId + ")")) {
|
||||||
readNextPacket();
|
readNextPacket();
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -165,13 +163,11 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized int read(ByteBuffer buf) throws IOException {
|
public synchronized int read(ByteBuffer buf) throws IOException {
|
||||||
if (curDataSlice == null || curDataSlice.remaining() == 0 && bytesNeededToFinish > 0) {
|
if (curDataSlice == null ||
|
||||||
TraceScope scope = tracer.newScope(
|
(curDataSlice.remaining() == 0 && bytesNeededToFinish > 0)) {
|
||||||
"RemoteBlockReader2#readNextPacket(" + blockId + ")");
|
try (TraceScope ignored = tracer.newScope(
|
||||||
try {
|
"RemoteBlockReader2#readNextPacket(" + blockId + ")")) {
|
||||||
readNextPacket();
|
readNextPacket();
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (curDataSlice.remaining() == 0) {
|
if (curDataSlice.remaining() == 0) {
|
||||||
|
@ -200,8 +196,8 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
|
|
||||||
// Sanity check the lengths
|
// Sanity check the lengths
|
||||||
if (!curHeader.sanityCheck(lastSeqNo)) {
|
if (!curHeader.sanityCheck(lastSeqNo)) {
|
||||||
throw new IOException("BlockReader: error in packet header " +
|
throw new IOException("BlockReader: error in packet header " +
|
||||||
curHeader);
|
curHeader);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (curHeader.getDataLen() > 0) {
|
if (curHeader.getDataLen() > 0) {
|
||||||
|
@ -209,8 +205,9 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
int checksumsLen = chunks * checksumSize;
|
int checksumsLen = chunks * checksumSize;
|
||||||
|
|
||||||
assert packetReceiver.getChecksumSlice().capacity() == checksumsLen :
|
assert packetReceiver.getChecksumSlice().capacity() == checksumsLen :
|
||||||
"checksum slice capacity=" + packetReceiver.getChecksumSlice().capacity() +
|
"checksum slice capacity=" +
|
||||||
" checksumsLen=" + checksumsLen;
|
packetReceiver.getChecksumSlice().capacity() +
|
||||||
|
" checksumsLen=" + checksumsLen;
|
||||||
|
|
||||||
lastSeqNo = curHeader.getSeqno();
|
lastSeqNo = curHeader.getSeqno();
|
||||||
if (verifyChecksum && curDataSlice.remaining() > 0) {
|
if (verifyChecksum && curDataSlice.remaining() > 0) {
|
||||||
|
@ -251,7 +248,8 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
long skipped = 0;
|
long skipped = 0;
|
||||||
while (skipped < n) {
|
while (skipped < n) {
|
||||||
long needToSkip = n - skipped;
|
long needToSkip = n - skipped;
|
||||||
if (curDataSlice == null || curDataSlice.remaining() == 0 && bytesNeededToFinish > 0) {
|
if (curDataSlice == null ||
|
||||||
|
curDataSlice.remaining() == 0 && bytesNeededToFinish > 0) {
|
||||||
readNextPacket();
|
readNextPacket();
|
||||||
}
|
}
|
||||||
if (curDataSlice.remaining() == 0) {
|
if (curDataSlice.remaining() == 0) {
|
||||||
|
@ -273,13 +271,13 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
|
|
||||||
PacketHeader trailer = packetReceiver.getHeader();
|
PacketHeader trailer = packetReceiver.getHeader();
|
||||||
if (!trailer.isLastPacketInBlock() ||
|
if (!trailer.isLastPacketInBlock() ||
|
||||||
trailer.getDataLen() != 0) {
|
trailer.getDataLen() != 0) {
|
||||||
throw new IOException("Expected empty end-of-read packet! Header: " +
|
throw new IOException("Expected empty end-of-read packet! Header: " +
|
||||||
trailer);
|
trailer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected RemoteBlockReader2(String file, String bpid, long blockId,
|
protected RemoteBlockReader2(String file, long blockId,
|
||||||
DataChecksum checksum, boolean verifyChecksum,
|
DataChecksum checksum, boolean verifyChecksum,
|
||||||
long startOffset, long firstChunkOffset, long bytesToRead, Peer peer,
|
long startOffset, long firstChunkOffset, long bytesToRead, Peer peer,
|
||||||
DatanodeID datanodeID, PeerCache peerCache, Tracer tracer) {
|
DatanodeID datanodeID, PeerCache peerCache, Tracer tracer) {
|
||||||
|
@ -335,7 +333,7 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
// It's ok not to be able to send this. But something is probably wrong.
|
// It's ok not to be able to send this. But something is probably wrong.
|
||||||
LOG.info("Could not send read status (" + statusCode + ") to datanode " +
|
LOG.info("Could not send read status (" + statusCode + ") to datanode " +
|
||||||
peer.getRemoteAddressString() + ": " + e.getMessage());
|
peer.getRemoteAddressString() + ": " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -346,9 +344,9 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
ClientReadStatusProto.newBuilder()
|
ClientReadStatusProto.newBuilder()
|
||||||
.setStatus(statusCode)
|
.setStatus(statusCode)
|
||||||
.build()
|
.build()
|
||||||
.writeDelimitedTo(out);
|
.writeDelimitedTo(out);
|
||||||
|
|
||||||
out.flush();
|
out.flush();
|
||||||
}
|
}
|
||||||
|
@ -391,18 +389,18 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
* @return New BlockReader instance, or null on error.
|
* @return New BlockReader instance, or null on error.
|
||||||
*/
|
*/
|
||||||
public static BlockReader newBlockReader(String file,
|
public static BlockReader newBlockReader(String file,
|
||||||
ExtendedBlock block,
|
ExtendedBlock block,
|
||||||
Token<BlockTokenIdentifier> blockToken,
|
Token<BlockTokenIdentifier> blockToken,
|
||||||
long startOffset, long len,
|
long startOffset, long len,
|
||||||
boolean verifyChecksum,
|
boolean verifyChecksum,
|
||||||
String clientName,
|
String clientName,
|
||||||
Peer peer, DatanodeID datanodeID,
|
Peer peer, DatanodeID datanodeID,
|
||||||
PeerCache peerCache,
|
PeerCache peerCache,
|
||||||
CachingStrategy cachingStrategy,
|
CachingStrategy cachingStrategy,
|
||||||
Tracer tracer) throws IOException {
|
Tracer tracer) throws IOException {
|
||||||
// in and out will be closed when sock is closed (by the caller)
|
// in and out will be closed when sock is closed (by the caller)
|
||||||
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
|
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
|
||||||
peer.getOutputStream()));
|
peer.getOutputStream()));
|
||||||
new Sender(out).readBlock(block, blockToken, clientName, startOffset, len,
|
new Sender(out).readBlock(block, blockToken, clientName, startOffset, len,
|
||||||
verifyChecksum, cachingStrategy);
|
verifyChecksum, cachingStrategy);
|
||||||
|
|
||||||
|
@ -415,7 +413,7 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
PBHelperClient.vintPrefixed(in));
|
PBHelperClient.vintPrefixed(in));
|
||||||
checkSuccess(status, peer, block, file);
|
checkSuccess(status, peer, block, file);
|
||||||
ReadOpChecksumInfoProto checksumInfo =
|
ReadOpChecksumInfoProto checksumInfo =
|
||||||
status.getReadOpChecksumInfo();
|
status.getReadOpChecksumInfo();
|
||||||
DataChecksum checksum = DataTransferProtoUtil.fromProto(
|
DataChecksum checksum = DataTransferProtoUtil.fromProto(
|
||||||
checksumInfo.getChecksum());
|
checksumInfo.getChecksum());
|
||||||
//Warning when we get CHECKSUM_NULL?
|
//Warning when we get CHECKSUM_NULL?
|
||||||
|
@ -426,13 +424,13 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
|
if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
|
||||||
firstChunkOffset <= (startOffset - checksum.getBytesPerChecksum())) {
|
firstChunkOffset <= (startOffset - checksum.getBytesPerChecksum())) {
|
||||||
throw new IOException("BlockReader: error in first chunk offset (" +
|
throw new IOException("BlockReader: error in first chunk offset (" +
|
||||||
firstChunkOffset + ") startOffset is " +
|
firstChunkOffset + ") startOffset is " +
|
||||||
startOffset + " for file " + file);
|
startOffset + " for file " + file);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new RemoteBlockReader2(file, block.getBlockPoolId(), block.getBlockId(),
|
return new RemoteBlockReader2(file, block.getBlockId(), checksum,
|
||||||
checksum, verifyChecksum, startOffset, firstChunkOffset, len, peer,
|
verifyChecksum, startOffset, firstChunkOffset, len, peer, datanodeID,
|
||||||
datanodeID, peerCache, tracer);
|
peerCache, tracer);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void checkSuccess(
|
static void checkSuccess(
|
||||||
|
@ -440,16 +438,16 @@ public class RemoteBlockReader2 implements BlockReader {
|
||||||
ExtendedBlock block, String file)
|
ExtendedBlock block, String file)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
String logInfo = "for OP_READ_BLOCK"
|
String logInfo = "for OP_READ_BLOCK"
|
||||||
+ ", self=" + peer.getLocalAddressString()
|
+ ", self=" + peer.getLocalAddressString()
|
||||||
+ ", remote=" + peer.getRemoteAddressString()
|
+ ", remote=" + peer.getRemoteAddressString()
|
||||||
+ ", for file " + file
|
+ ", for file " + file
|
||||||
+ ", for pool " + block.getBlockPoolId()
|
+ ", for pool " + block.getBlockPoolId()
|
||||||
+ " block " + block.getBlockId() + "_" + block.getGenerationStamp();
|
+ " block " + block.getBlockId() + "_" + block.getGenerationStamp();
|
||||||
DataTransferProtoUtil.checkBlockOpStatus(status, logInfo);
|
DataTransferProtoUtil.checkBlockOpStatus(status, logInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int available() throws IOException {
|
public int available() {
|
||||||
// An optimistic estimate of how much data is available
|
// An optimistic estimate of how much data is available
|
||||||
// to us without doing network I/O.
|
// to us without doing network I/O.
|
||||||
return TCP_WINDOW_SIZE;
|
return TCP_WINDOW_SIZE;
|
||||||
|
|
|
@ -77,10 +77,9 @@ public class XAttrHelper {
|
||||||
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
throw new HadoopIllegalArgumentException("An XAttr name must be " +
|
||||||
"prefixed with user/trusted/security/system/raw, followed by a '.'");
|
"prefixed with user/trusted/security/system/raw, followed by a '.'");
|
||||||
}
|
}
|
||||||
XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name.
|
|
||||||
substring(prefixIndex + 1)).setValue(value).build();
|
|
||||||
|
|
||||||
return xAttr;
|
return (new XAttr.Builder()).setNameSpace(ns).setName(name.
|
||||||
|
substring(prefixIndex + 1)).setValue(value).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -31,7 +31,8 @@ public interface HdfsClientConfigKeys {
|
||||||
long DFS_BLOCK_SIZE_DEFAULT = 128*1024*1024;
|
long DFS_BLOCK_SIZE_DEFAULT = 128*1024*1024;
|
||||||
String DFS_REPLICATION_KEY = "dfs.replication";
|
String DFS_REPLICATION_KEY = "dfs.replication";
|
||||||
short DFS_REPLICATION_DEFAULT = 3;
|
short DFS_REPLICATION_DEFAULT = 3;
|
||||||
String DFS_WEBHDFS_USER_PATTERN_KEY = "dfs.webhdfs.user.provider.user.pattern";
|
String DFS_WEBHDFS_USER_PATTERN_KEY =
|
||||||
|
"dfs.webhdfs.user.provider.user.pattern";
|
||||||
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
|
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
|
||||||
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
|
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
|
||||||
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
|
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
|
||||||
|
@ -125,7 +126,8 @@ public interface HdfsClientConfigKeys {
|
||||||
"dfs.datanode.hdfs-blocks-metadata.enabled";
|
"dfs.datanode.hdfs-blocks-metadata.enabled";
|
||||||
boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT = false;
|
boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT = false;
|
||||||
|
|
||||||
String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
|
String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY =
|
||||||
|
"dfs.datanode.kerberos.principal";
|
||||||
String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes";
|
String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes";
|
||||||
long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB
|
long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB
|
||||||
String DFS_ENCRYPTION_KEY_PROVIDER_URI = "dfs.encryption.key.provider.uri";
|
String DFS_ENCRYPTION_KEY_PROVIDER_URI = "dfs.encryption.key.provider.uri";
|
||||||
|
@ -142,7 +144,8 @@ public interface HdfsClientConfigKeys {
|
||||||
"dfs.encrypt.data.transfer.cipher.key.bitlength";
|
"dfs.encrypt.data.transfer.cipher.key.bitlength";
|
||||||
int DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_DEFAULT = 128;
|
int DFS_ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH_DEFAULT = 128;
|
||||||
|
|
||||||
String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class";
|
String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS =
|
||||||
|
"dfs.trustedchannel.resolver.class";
|
||||||
|
|
||||||
String REPLICA_ACCESSOR_BUILDER_CLASSES_KEY =
|
String REPLICA_ACCESSOR_BUILDER_CLASSES_KEY =
|
||||||
PREFIX + "replica.accessor.builder.classes";
|
PREFIX + "replica.accessor.builder.classes";
|
||||||
|
@ -165,9 +168,11 @@ public interface HdfsClientConfigKeys {
|
||||||
String POLICY_SPEC_KEY = PREFIX + "policy.spec";
|
String POLICY_SPEC_KEY = PREFIX + "policy.spec";
|
||||||
String POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,...
|
String POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,...
|
||||||
|
|
||||||
String TIMES_GET_LAST_BLOCK_LENGTH_KEY = PREFIX + "times.get-last-block-length";
|
String TIMES_GET_LAST_BLOCK_LENGTH_KEY =
|
||||||
|
PREFIX + "times.get-last-block-length";
|
||||||
int TIMES_GET_LAST_BLOCK_LENGTH_DEFAULT = 3;
|
int TIMES_GET_LAST_BLOCK_LENGTH_DEFAULT = 3;
|
||||||
String INTERVAL_GET_LAST_BLOCK_LENGTH_KEY = PREFIX + "interval-ms.get-last-block-length";
|
String INTERVAL_GET_LAST_BLOCK_LENGTH_KEY =
|
||||||
|
PREFIX + "interval-ms.get-last-block-length";
|
||||||
int INTERVAL_GET_LAST_BLOCK_LENGTH_DEFAULT = 4000;
|
int INTERVAL_GET_LAST_BLOCK_LENGTH_DEFAULT = 4000;
|
||||||
|
|
||||||
String MAX_ATTEMPTS_KEY = PREFIX + "max.attempts";
|
String MAX_ATTEMPTS_KEY = PREFIX + "max.attempts";
|
||||||
|
@ -190,7 +195,8 @@ public interface HdfsClientConfigKeys {
|
||||||
int SLEEPTIME_MAX_DEFAULT = 15000;
|
int SLEEPTIME_MAX_DEFAULT = 15000;
|
||||||
String CONNECTION_RETRIES_KEY = PREFIX + "connection.retries";
|
String CONNECTION_RETRIES_KEY = PREFIX + "connection.retries";
|
||||||
int CONNECTION_RETRIES_DEFAULT = 0;
|
int CONNECTION_RETRIES_DEFAULT = 0;
|
||||||
String CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY = PREFIX + "connection.retries.on.timeouts";
|
String CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY =
|
||||||
|
PREFIX + "connection.retries.on.timeouts";
|
||||||
int CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0;
|
int CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -200,7 +206,8 @@ public interface HdfsClientConfigKeys {
|
||||||
|
|
||||||
String MAX_PACKETS_IN_FLIGHT_KEY = PREFIX + "max-packets-in-flight";
|
String MAX_PACKETS_IN_FLIGHT_KEY = PREFIX + "max-packets-in-flight";
|
||||||
int MAX_PACKETS_IN_FLIGHT_DEFAULT = 80;
|
int MAX_PACKETS_IN_FLIGHT_DEFAULT = 80;
|
||||||
String EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_KEY = PREFIX + "exclude.nodes.cache.expiry.interval.millis";
|
String EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_KEY =
|
||||||
|
PREFIX + "exclude.nodes.cache.expiry.interval.millis";
|
||||||
long EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10*MINUTE;
|
long EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10*MINUTE;
|
||||||
|
|
||||||
interface ByteArrayManager {
|
interface ByteArrayManager {
|
||||||
|
@ -212,7 +219,8 @@ public interface HdfsClientConfigKeys {
|
||||||
int COUNT_THRESHOLD_DEFAULT = 128;
|
int COUNT_THRESHOLD_DEFAULT = 128;
|
||||||
String COUNT_LIMIT_KEY = PREFIX + "count-limit";
|
String COUNT_LIMIT_KEY = PREFIX + "count-limit";
|
||||||
int COUNT_LIMIT_DEFAULT = 2048;
|
int COUNT_LIMIT_DEFAULT = 2048;
|
||||||
String COUNT_RESET_TIME_PERIOD_MS_KEY = PREFIX + "count-reset-time-period-ms";
|
String COUNT_RESET_TIME_PERIOD_MS_KEY =
|
||||||
|
PREFIX + "count-reset-time-period-ms";
|
||||||
long COUNT_RESET_TIME_PERIOD_MS_DEFAULT = 10*SECOND;
|
long COUNT_RESET_TIME_PERIOD_MS_DEFAULT = 10*SECOND;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -223,9 +231,11 @@ public interface HdfsClientConfigKeys {
|
||||||
|
|
||||||
String RETRIES_KEY = PREFIX + "retries";
|
String RETRIES_KEY = PREFIX + "retries";
|
||||||
int RETRIES_DEFAULT = 3;
|
int RETRIES_DEFAULT = 3;
|
||||||
String LOCATEFOLLOWINGBLOCK_RETRIES_KEY = PREFIX + "locateFollowingBlock.retries";
|
String LOCATEFOLLOWINGBLOCK_RETRIES_KEY =
|
||||||
|
PREFIX + "locateFollowingBlock.retries";
|
||||||
int LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT = 5;
|
int LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT = 5;
|
||||||
String LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY = PREFIX + "locateFollowingBlock.initial.delay.ms";
|
String LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_KEY =
|
||||||
|
PREFIX + "locateFollowingBlock.initial.delay.ms";
|
||||||
int LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT = 400;
|
int LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_MS_DEFAULT = 400;
|
||||||
|
|
||||||
interface ReplaceDatanodeOnFailure {
|
interface ReplaceDatanodeOnFailure {
|
||||||
|
@ -267,7 +277,8 @@ public interface HdfsClientConfigKeys {
|
||||||
interface ShortCircuit {
|
interface ShortCircuit {
|
||||||
String PREFIX = Read.PREFIX + "short.circuit.";
|
String PREFIX = Read.PREFIX + "short.circuit.";
|
||||||
|
|
||||||
String REPLICA_STALE_THRESHOLD_MS_KEY = PREFIX + "replica.stale.threshold.ms";
|
String REPLICA_STALE_THRESHOLD_MS_KEY =
|
||||||
|
PREFIX + "replica.stale.threshold.ms";
|
||||||
long REPLICA_STALE_THRESHOLD_MS_DEFAULT = 30*MINUTE;
|
long REPLICA_STALE_THRESHOLD_MS_DEFAULT = 30*MINUTE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -38,11 +38,11 @@ import com.google.common.base.Preconditions;
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class HdfsDataInputStream extends FSDataInputStream {
|
public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
public HdfsDataInputStream(DFSInputStream in) throws IOException {
|
public HdfsDataInputStream(DFSInputStream in) {
|
||||||
super(in);
|
super(in);
|
||||||
}
|
}
|
||||||
|
|
||||||
public HdfsDataInputStream(CryptoInputStream in) throws IOException {
|
public HdfsDataInputStream(CryptoInputStream in) {
|
||||||
super(in);
|
super(in);
|
||||||
Preconditions.checkArgument(in.getWrappedStream() instanceof DFSInputStream,
|
Preconditions.checkArgument(in.getWrappedStream() instanceof DFSInputStream,
|
||||||
"CryptoInputStream should wrap a DFSInputStream");
|
"CryptoInputStream should wrap a DFSInputStream");
|
||||||
|
@ -63,7 +63,7 @@ public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
* @return the underlying output stream
|
* @return the underlying output stream
|
||||||
*/
|
*/
|
||||||
public InputStream getWrappedStream() {
|
public InputStream getWrappedStream() {
|
||||||
return in;
|
return in;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -93,7 +93,7 @@ public class HdfsDataInputStream extends FSDataInputStream {
|
||||||
*
|
*
|
||||||
* @return The visible length of the file.
|
* @return The visible length of the file.
|
||||||
*/
|
*/
|
||||||
public long getVisibleLength() throws IOException {
|
public long getVisibleLength() {
|
||||||
return getDFSInputStream().getFileLength();
|
return getDFSInputStream().getFileLength();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -41,20 +41,21 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
super(out, stats, startPosition);
|
super(out, stats, startPosition);
|
||||||
}
|
}
|
||||||
|
|
||||||
public HdfsDataOutputStream(DFSOutputStream out, FileSystem.Statistics stats
|
public HdfsDataOutputStream(DFSOutputStream out, FileSystem.Statistics stats)
|
||||||
) throws IOException {
|
throws IOException {
|
||||||
this(out, stats, 0L);
|
this(out, stats, 0L);
|
||||||
}
|
}
|
||||||
|
|
||||||
public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats,
|
public HdfsDataOutputStream(CryptoOutputStream out,
|
||||||
long startPosition) throws IOException {
|
FileSystem.Statistics stats, long startPosition) throws IOException {
|
||||||
super(out, stats, startPosition);
|
super(out, stats, startPosition);
|
||||||
Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream,
|
Preconditions.checkArgument(
|
||||||
|
out.getWrappedStream() instanceof DFSOutputStream,
|
||||||
"CryptoOutputStream should wrap a DFSOutputStream");
|
"CryptoOutputStream should wrap a DFSOutputStream");
|
||||||
}
|
}
|
||||||
|
|
||||||
public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats)
|
public HdfsDataOutputStream(CryptoOutputStream out,
|
||||||
throws IOException {
|
FileSystem.Statistics stats) throws IOException {
|
||||||
this(out, stats, 0L);
|
this(out, stats, 0L);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -89,13 +90,13 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
|
||||||
OutputStream wrappedStream = getWrappedStream();
|
OutputStream wrappedStream = getWrappedStream();
|
||||||
if (wrappedStream instanceof CryptoOutputStream) {
|
if (wrappedStream instanceof CryptoOutputStream) {
|
||||||
((CryptoOutputStream) wrappedStream).flush();
|
wrappedStream.flush();
|
||||||
wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
|
wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
|
||||||
}
|
}
|
||||||
((DFSOutputStream) wrappedStream).hsync(syncFlags);
|
((DFSOutputStream) wrappedStream).hsync(syncFlags);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static enum SyncFlag {
|
public enum SyncFlag {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* When doing sync to DataNodes, also update the metadata (block length) in
|
* When doing sync to DataNodes, also update the metadata (block length) in
|
||||||
|
@ -107,6 +108,6 @@ public class HdfsDataOutputStream extends FSDataOutputStream {
|
||||||
* Sync the data to DataNode, close the current block, and allocate a new
|
* Sync the data to DataNode, close the current block, and allocate a new
|
||||||
* block
|
* block
|
||||||
*/
|
*/
|
||||||
END_BLOCK;
|
END_BLOCK
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -79,7 +79,6 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
|
||||||
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.ShortCircuit;
|
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.ShortCircuit;
|
||||||
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write;
|
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write;
|
||||||
|
|
||||||
import java.lang.Class;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -254,9 +253,8 @@ public class DfsClientConf {
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private List<Class<? extends ReplicaAccessorBuilder>>
|
private List<Class<? extends ReplicaAccessorBuilder>>
|
||||||
loadReplicaAccessorBuilderClasses(Configuration conf)
|
loadReplicaAccessorBuilderClasses(Configuration conf) {
|
||||||
{
|
String[] classNames = conf.getTrimmedStrings(
|
||||||
String classNames[] = conf.getTrimmedStrings(
|
|
||||||
HdfsClientConfigKeys.REPLICA_ACCESSOR_BUILDER_CLASSES_KEY);
|
HdfsClientConfigKeys.REPLICA_ACCESSOR_BUILDER_CLASSES_KEY);
|
||||||
if (classNames.length == 0) {
|
if (classNames.length == 0) {
|
||||||
return Collections.emptyList();
|
return Collections.emptyList();
|
||||||
|
@ -267,8 +265,8 @@ public class DfsClientConf {
|
||||||
for (String className: classNames) {
|
for (String className: classNames) {
|
||||||
try {
|
try {
|
||||||
Class<? extends ReplicaAccessorBuilder> cls =
|
Class<? extends ReplicaAccessorBuilder> cls =
|
||||||
(Class<? extends ReplicaAccessorBuilder>)
|
(Class<? extends ReplicaAccessorBuilder>)
|
||||||
classLoader.loadClass(className);
|
classLoader.loadClass(className);
|
||||||
classes.add(cls);
|
classes.add(cls);
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
LOG.warn("Unable to load " + className, t);
|
LOG.warn("Unable to load " + className, t);
|
||||||
|
|
|
@ -42,8 +42,8 @@ import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
* Used by {@link org.apache.hadoop.hdfs.DFSClient} for renewing file-being-written leases
|
* Used by {@link org.apache.hadoop.hdfs.DFSClient} for renewing
|
||||||
* on the namenode.
|
* file-being-written leases on the namenode.
|
||||||
* When a file is opened for write (create or append),
|
* When a file is opened for write (create or append),
|
||||||
* namenode stores a file lease for recording the identity of the writer.
|
* namenode stores a file lease for recording the identity of the writer.
|
||||||
* The writer (i.e. the DFSClient) is required to renew the lease periodically.
|
* The writer (i.e. the DFSClient) is required to renew the lease periodically.
|
||||||
|
@ -57,7 +57,8 @@ import org.slf4j.LoggerFactory;
|
||||||
* <li>
|
* <li>
|
||||||
* It maintains a map from (namenode, user) pairs to lease renewers.
|
* It maintains a map from (namenode, user) pairs to lease renewers.
|
||||||
* The same {@link LeaseRenewer} instance is used for renewing lease
|
* The same {@link LeaseRenewer} instance is used for renewing lease
|
||||||
* for all the {@link org.apache.hadoop.hdfs.DFSClient} to the same namenode and the same user.
|
* for all the {@link org.apache.hadoop.hdfs.DFSClient} to the same namenode and
|
||||||
|
* the same user.
|
||||||
* </li>
|
* </li>
|
||||||
* <li>
|
* <li>
|
||||||
* Each renewer maintains a list of {@link org.apache.hadoop.hdfs.DFSClient}.
|
* Each renewer maintains a list of {@link org.apache.hadoop.hdfs.DFSClient}.
|
||||||
|
@ -80,7 +81,7 @@ public class LeaseRenewer {
|
||||||
|
|
||||||
/** Get a {@link LeaseRenewer} instance */
|
/** Get a {@link LeaseRenewer} instance */
|
||||||
public static LeaseRenewer getInstance(final String authority,
|
public static LeaseRenewer getInstance(final String authority,
|
||||||
final UserGroupInformation ugi, final DFSClient dfsc) throws IOException {
|
final UserGroupInformation ugi, final DFSClient dfsc) {
|
||||||
final LeaseRenewer r = Factory.INSTANCE.get(authority, ugi);
|
final LeaseRenewer r = Factory.INSTANCE.get(authority, ugi);
|
||||||
r.addClient(dfsc);
|
r.addClient(dfsc);
|
||||||
return r;
|
return r;
|
||||||
|
@ -136,7 +137,7 @@ public class LeaseRenewer {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A map for per user per namenode renewers. */
|
/** A map for per user per namenode renewers. */
|
||||||
private final Map<Key, LeaseRenewer> renewers = new HashMap<Key, LeaseRenewer>();
|
private final Map<Key, LeaseRenewer> renewers = new HashMap<>();
|
||||||
|
|
||||||
/** Get a renewer. */
|
/** Get a renewer. */
|
||||||
private synchronized LeaseRenewer get(final String authority,
|
private synchronized LeaseRenewer get(final String authority,
|
||||||
|
@ -189,7 +190,7 @@ public class LeaseRenewer {
|
||||||
private final Factory.Key factorykey;
|
private final Factory.Key factorykey;
|
||||||
|
|
||||||
/** A list of clients corresponding to this renewer. */
|
/** A list of clients corresponding to this renewer. */
|
||||||
private final List<DFSClient> dfsclients = new ArrayList<DFSClient>();
|
private final List<DFSClient> dfsclients = new ArrayList<>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A stringified stack trace of the call stack when the Lease Renewer
|
* A stringified stack trace of the call stack when the Lease Renewer
|
||||||
|
@ -404,7 +405,7 @@ public class LeaseRenewer {
|
||||||
private void renew() throws IOException {
|
private void renew() throws IOException {
|
||||||
final List<DFSClient> copies;
|
final List<DFSClient> copies;
|
||||||
synchronized(this) {
|
synchronized(this) {
|
||||||
copies = new ArrayList<DFSClient>(dfsclients);
|
copies = new ArrayList<>(dfsclients);
|
||||||
}
|
}
|
||||||
//sort the client names for finding out repeated names.
|
//sort the client names for finding out repeated names.
|
||||||
Collections.sort(copies, new Comparator<DFSClient>() {
|
Collections.sort(copies, new Comparator<DFSClient>() {
|
||||||
|
@ -414,8 +415,7 @@ public class LeaseRenewer {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
String previousName = "";
|
String previousName = "";
|
||||||
for(int i = 0; i < copies.size(); i++) {
|
for (final DFSClient c : copies) {
|
||||||
final DFSClient c = copies.get(i);
|
|
||||||
//skip if current client name is the same as the previous name.
|
//skip if current client name is the same as the previous name.
|
||||||
if (!c.getClientName().equals(previousName)) {
|
if (!c.getClientName().equals(previousName)) {
|
||||||
if (!c.renewLease()) {
|
if (!c.renewLease()) {
|
||||||
|
@ -470,7 +470,7 @@ public class LeaseRenewer {
|
||||||
LOG.debug("Lease renewer daemon for " + clientsString()
|
LOG.debug("Lease renewer daemon for " + clientsString()
|
||||||
+ " with renew id " + id + " is not current");
|
+ " with renew id " + id + " is not current");
|
||||||
} else {
|
} else {
|
||||||
LOG.debug("Lease renewer daemon for " + clientsString()
|
LOG.debug("Lease renewer daemon for " + clientsString()
|
||||||
+ " with renew id " + id + " expired");
|
+ " with renew id " + id + " expired");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,7 +34,7 @@ import java.util.List;
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public abstract class Event {
|
public abstract class Event {
|
||||||
public static enum EventType {
|
public enum EventType {
|
||||||
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE
|
CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,8 +98,8 @@ public abstract class Event {
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
public static class CreateEvent extends Event {
|
public static class CreateEvent extends Event {
|
||||||
|
|
||||||
public static enum INodeType {
|
public enum INodeType {
|
||||||
FILE, DIRECTORY, SYMLINK;
|
FILE, DIRECTORY, SYMLINK
|
||||||
}
|
}
|
||||||
|
|
||||||
private INodeType iNodeType;
|
private INodeType iNodeType;
|
||||||
|
@ -247,17 +247,21 @@ public abstract class Event {
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder content = new StringBuilder();
|
StringBuilder content = new StringBuilder();
|
||||||
content.append("CreateEvent [INodeType=" + iNodeType + ", path=" + path
|
content.append("CreateEvent [INodeType=").append(iNodeType)
|
||||||
+ ", ctime=" + ctime + ", replication=" + replication
|
.append(", path=").append(path)
|
||||||
+ ", ownerName=" + ownerName + ", groupName=" + groupName
|
.append(", ctime=").append(ctime)
|
||||||
+ ", perms=" + perms + ", ");
|
.append(", replication=").append(replication)
|
||||||
|
.append(", ownerName=").append(ownerName)
|
||||||
|
.append(", groupName=").append(groupName)
|
||||||
|
.append(", perms=").append(perms).append(", ");
|
||||||
|
|
||||||
if (symlinkTarget != null) {
|
if (symlinkTarget != null) {
|
||||||
content.append("symlinkTarget=" + symlinkTarget + ", ");
|
content.append("symlinkTarget=").append(symlinkTarget).append(", ");
|
||||||
}
|
}
|
||||||
|
|
||||||
content.append("overwrite=" + overwrite + ", defaultBlockSize="
|
content.append("overwrite=").append(overwrite)
|
||||||
+ defaultBlockSize + "]");
|
.append(", defaultBlockSize=").append(defaultBlockSize)
|
||||||
|
.append("]");
|
||||||
return content.toString();
|
return content.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -274,8 +278,8 @@ public abstract class Event {
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
public static class MetadataUpdateEvent extends Event {
|
public static class MetadataUpdateEvent extends Event {
|
||||||
|
|
||||||
public static enum MetadataType {
|
public enum MetadataType {
|
||||||
TIMES, REPLICATION, OWNER, PERMS, ACLS, XATTRS;
|
TIMES, REPLICATION, OWNER, PERMS, ACLS, XATTRS
|
||||||
}
|
}
|
||||||
|
|
||||||
private String path;
|
private String path;
|
||||||
|
@ -434,28 +438,29 @@ public abstract class Event {
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder content = new StringBuilder();
|
StringBuilder content = new StringBuilder();
|
||||||
content.append("MetadataUpdateEvent [path=" + path + ", metadataType="
|
content.append("MetadataUpdateEvent [path=").append(path)
|
||||||
+ metadataType);
|
.append(", metadataType=").append(metadataType);
|
||||||
switch (metadataType) {
|
switch (metadataType) {
|
||||||
case TIMES:
|
case TIMES:
|
||||||
content.append(", mtime=" + mtime + ", atime=" + atime);
|
content.append(", mtime=").append(mtime)
|
||||||
|
.append(", atime=").append(atime);
|
||||||
break;
|
break;
|
||||||
case REPLICATION:
|
case REPLICATION:
|
||||||
content.append(", replication=" + replication);
|
content.append(", replication=").append(replication);
|
||||||
break;
|
break;
|
||||||
case OWNER:
|
case OWNER:
|
||||||
content.append(", ownerName=" + ownerName
|
content.append(", ownerName=").append(ownerName)
|
||||||
+ ", groupName=" + groupName);
|
.append(", groupName=").append(groupName);
|
||||||
break;
|
break;
|
||||||
case PERMS:
|
case PERMS:
|
||||||
content.append(", perms=" + perms);
|
content.append(", perms=").append(perms);
|
||||||
break;
|
break;
|
||||||
case ACLS:
|
case ACLS:
|
||||||
content.append(", acls=" + acls);
|
content.append(", acls=").append(acls);
|
||||||
break;
|
break;
|
||||||
case XATTRS:
|
case XATTRS:
|
||||||
content.append(", xAttrs=" + xAttrs + ", xAttrsRemoved="
|
content.append(", xAttrs=").append(xAttrs)
|
||||||
+ xAttrsRemoved);
|
.append(", xAttrsRemoved=").append(xAttrsRemoved);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -34,24 +34,24 @@ public interface Peer extends Closeable {
|
||||||
* @return The input stream channel associated with this
|
* @return The input stream channel associated with this
|
||||||
* peer, or null if it has none.
|
* peer, or null if it has none.
|
||||||
*/
|
*/
|
||||||
public ReadableByteChannel getInputStreamChannel();
|
ReadableByteChannel getInputStreamChannel();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the read timeout on this peer.
|
* Set the read timeout on this peer.
|
||||||
*
|
*
|
||||||
* @param timeoutMs The timeout in milliseconds.
|
* @param timeoutMs The timeout in milliseconds.
|
||||||
*/
|
*/
|
||||||
public void setReadTimeout(int timeoutMs) throws IOException;
|
void setReadTimeout(int timeoutMs) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return The receive buffer size.
|
* @return The receive buffer size.
|
||||||
*/
|
*/
|
||||||
public int getReceiveBufferSize() throws IOException;
|
int getReceiveBufferSize() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return True if TCP_NODELAY is turned on.
|
* @return True if TCP_NODELAY is turned on.
|
||||||
*/
|
*/
|
||||||
public boolean getTcpNoDelay() throws IOException;
|
boolean getTcpNoDelay() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the write timeout on this peer.
|
* Set the write timeout on this peer.
|
||||||
|
@ -61,57 +61,57 @@ public interface Peer extends Closeable {
|
||||||
*
|
*
|
||||||
* @param timeoutMs The timeout in milliseconds.
|
* @param timeoutMs The timeout in milliseconds.
|
||||||
*/
|
*/
|
||||||
public void setWriteTimeout(int timeoutMs) throws IOException;
|
void setWriteTimeout(int timeoutMs) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return true only if the peer is closed.
|
* @return true only if the peer is closed.
|
||||||
*/
|
*/
|
||||||
public boolean isClosed();
|
boolean isClosed();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Close the peer.
|
* Close the peer.
|
||||||
*
|
*
|
||||||
* It's safe to re-close a Peer that is already closed.
|
* It's safe to re-close a Peer that is already closed.
|
||||||
*/
|
*/
|
||||||
public void close() throws IOException;
|
void close() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return A string representing the remote end of our
|
* @return A string representing the remote end of our
|
||||||
* connection to the peer.
|
* connection to the peer.
|
||||||
*/
|
*/
|
||||||
public String getRemoteAddressString();
|
String getRemoteAddressString();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return A string representing the local end of our
|
* @return A string representing the local end of our
|
||||||
* connection to the peer.
|
* connection to the peer.
|
||||||
*/
|
*/
|
||||||
public String getLocalAddressString();
|
String getLocalAddressString();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return An InputStream associated with the Peer.
|
* @return An InputStream associated with the Peer.
|
||||||
* This InputStream will be valid until you close
|
* This InputStream will be valid until you close
|
||||||
* this peer with Peer#close.
|
* this peer with Peer#close.
|
||||||
*/
|
*/
|
||||||
public InputStream getInputStream() throws IOException;
|
InputStream getInputStream() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return An OutputStream associated with the Peer.
|
* @return An OutputStream associated with the Peer.
|
||||||
* This OutputStream will be valid until you close
|
* This OutputStream will be valid until you close
|
||||||
* this peer with Peer#close.
|
* this peer with Peer#close.
|
||||||
*/
|
*/
|
||||||
public OutputStream getOutputStream() throws IOException;
|
OutputStream getOutputStream() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return True if the peer resides on the same
|
* @return True if the peer resides on the same
|
||||||
* computer as we.
|
* computer as we.
|
||||||
*/
|
*/
|
||||||
public boolean isLocal();
|
boolean isLocal();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return The DomainSocket associated with the current
|
* @return The DomainSocket associated with the current
|
||||||
* peer, or null if there is none.
|
* peer, or null if there is none.
|
||||||
*/
|
*/
|
||||||
public DomainSocket getDomainSocket();
|
DomainSocket getDomainSocket();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return true if the channel is secure.
|
* Return true if the channel is secure.
|
||||||
|
@ -119,5 +119,5 @@ public interface Peer extends Closeable {
|
||||||
* @return True if our channel to this peer is not
|
* @return True if our channel to this peer is not
|
||||||
* susceptible to man-in-the-middle attacks.
|
* susceptible to man-in-the-middle attacks.
|
||||||
*/
|
*/
|
||||||
public boolean hasSecureChannel();
|
boolean hasSecureChannel();
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.io.*;
|
import org.apache.hadoop.io.*;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/**************************************************
|
/**************************************************
|
||||||
* A Block is a Hadoop FS primitive, identified by a
|
* A Block is a Hadoop FS primitive, identified by a
|
||||||
* long.
|
* long.
|
||||||
|
@ -36,12 +38,10 @@ public class Block implements Writable, Comparable<Block> {
|
||||||
public static final String BLOCK_FILE_PREFIX = "blk_";
|
public static final String BLOCK_FILE_PREFIX = "blk_";
|
||||||
public static final String METADATA_EXTENSION = ".meta";
|
public static final String METADATA_EXTENSION = ".meta";
|
||||||
static { // register a ctor
|
static { // register a ctor
|
||||||
WritableFactories.setFactory
|
WritableFactories.setFactory(Block.class, new WritableFactory() {
|
||||||
(Block.class,
|
@Override
|
||||||
new WritableFactory() {
|
public Writable newInstance() { return new Block(); }
|
||||||
@Override
|
});
|
||||||
public Writable newInstance() { return new Block(); }
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final Pattern blockFilePattern = Pattern
|
public static final Pattern blockFilePattern = Pattern
|
||||||
|
@ -208,20 +208,14 @@ public class Block implements Writable, Comparable<Block> {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // Comparable
|
@Override // Comparable
|
||||||
public int compareTo(Block b) {
|
public int compareTo(@Nonnull Block b) {
|
||||||
return blockId < b.blockId ? -1 :
|
return blockId < b.blockId ? -1 :
|
||||||
blockId > b.blockId ? 1 : 0;
|
blockId > b.blockId ? 1 : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // Object
|
@Override // Object
|
||||||
public boolean equals(Object o) {
|
public boolean equals(Object o) {
|
||||||
if (this == o) {
|
return this == o || o instanceof Block && compareTo((Block) o) == 0;
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (!(o instanceof Block)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return compareTo((Block)o) == 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -230,9 +224,10 @@ public class Block implements Writable, Comparable<Block> {
|
||||||
*/
|
*/
|
||||||
public static boolean matchingIdAndGenStamp(Block a, Block b) {
|
public static boolean matchingIdAndGenStamp(Block a, Block b) {
|
||||||
if (a == b) return true; // same block, or both null
|
if (a == b) return true; // same block, or both null
|
||||||
if (a == null || b == null) return false; // only one null
|
// only one null
|
||||||
return a.blockId == b.blockId &&
|
return !(a == null || b == null) &&
|
||||||
a.generationStamp == b.generationStamp;
|
a.blockId == b.blockId &&
|
||||||
|
a.generationStamp == b.generationStamp;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // Object
|
@Override // Object
|
||||||
|
|
|
@ -79,7 +79,7 @@ public class BlockStoragePolicy implements BlockStoragePolicySpi {
|
||||||
* @return a list of {@link StorageType}s for storing the replicas of a block.
|
* @return a list of {@link StorageType}s for storing the replicas of a block.
|
||||||
*/
|
*/
|
||||||
public List<StorageType> chooseStorageTypes(final short replication) {
|
public List<StorageType> chooseStorageTypes(final short replication) {
|
||||||
final List<StorageType> types = new LinkedList<StorageType>();
|
final List<StorageType> types = new LinkedList<>();
|
||||||
int i = 0, j = 0;
|
int i = 0, j = 0;
|
||||||
|
|
||||||
// Do not return transient storage types. We will not have accurate
|
// Do not return transient storage types. We will not have accurate
|
||||||
|
@ -136,11 +136,11 @@ public class BlockStoragePolicy implements BlockStoragePolicySpi {
|
||||||
final Iterable<StorageType> chosen,
|
final Iterable<StorageType> chosen,
|
||||||
final EnumSet<StorageType> unavailables,
|
final EnumSet<StorageType> unavailables,
|
||||||
final boolean isNewBlock) {
|
final boolean isNewBlock) {
|
||||||
final List<StorageType> excess = new LinkedList<StorageType>();
|
final List<StorageType> excess = new LinkedList<>();
|
||||||
final List<StorageType> storageTypes = chooseStorageTypes(
|
final List<StorageType> storageTypes = chooseStorageTypes(
|
||||||
replication, chosen, excess);
|
replication, chosen, excess);
|
||||||
final int expectedSize = storageTypes.size() - excess.size();
|
final int expectedSize = storageTypes.size() - excess.size();
|
||||||
final List<StorageType> removed = new LinkedList<StorageType>();
|
final List<StorageType> removed = new LinkedList<>();
|
||||||
for(int i = storageTypes.size() - 1; i >= 0; i--) {
|
for(int i = storageTypes.size() - 1; i >= 0; i--) {
|
||||||
// replace/remove unavailable storage types.
|
// replace/remove unavailable storage types.
|
||||||
final StorageType t = storageTypes.get(i);
|
final StorageType t = storageTypes.get(i);
|
||||||
|
@ -195,7 +195,7 @@ public class BlockStoragePolicy implements BlockStoragePolicySpi {
|
||||||
public List<StorageType> chooseExcess(final short replication,
|
public List<StorageType> chooseExcess(final short replication,
|
||||||
final Iterable<StorageType> chosen) {
|
final Iterable<StorageType> chosen) {
|
||||||
final List<StorageType> types = chooseStorageTypes(replication);
|
final List<StorageType> types = chooseStorageTypes(replication);
|
||||||
final List<StorageType> excess = new LinkedList<StorageType>();
|
final List<StorageType> excess = new LinkedList<>();
|
||||||
diff(types, chosen, excess);
|
diff(types, chosen, excess);
|
||||||
return excess;
|
return excess;
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,4 +42,4 @@ public class CacheDirectiveEntry {
|
||||||
public CacheDirectiveStats getStats() {
|
public CacheDirectiveStats getStats() {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
|
@ -350,9 +350,8 @@ public class CacheDirectiveInfo {
|
||||||
}
|
}
|
||||||
if (expiration != null) {
|
if (expiration != null) {
|
||||||
builder.append(prefix).append("expiration: ").append(expiration);
|
builder.append(prefix).append("expiration: ").append(expiration);
|
||||||
prefix = ", ";
|
|
||||||
}
|
}
|
||||||
builder.append("}");
|
builder.append("}");
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
|
@ -92,9 +92,8 @@ public class CacheDirectiveIterator
|
||||||
@Override
|
@Override
|
||||||
public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
|
public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
BatchedEntries<CacheDirectiveEntry> entries = null;
|
BatchedEntries<CacheDirectiveEntry> entries;
|
||||||
TraceScope scope = tracer.newScope("listCacheDirectives");
|
try (TraceScope ignored = tracer.newScope("listCacheDirectives")) {
|
||||||
try {
|
|
||||||
entries = namenode.listCacheDirectives(prevKey, filter);
|
entries = namenode.listCacheDirectives(prevKey, filter);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
if (e.getMessage().contains("Filtering by ID is unsupported")) {
|
if (e.getMessage().contains("Filtering by ID is unsupported")) {
|
||||||
|
@ -105,9 +104,9 @@ public class CacheDirectiveIterator
|
||||||
// This is somewhat brittle, since it depends on directives being
|
// This is somewhat brittle, since it depends on directives being
|
||||||
// returned in order of ascending ID.
|
// returned in order of ascending ID.
|
||||||
entries = namenode.listCacheDirectives(id - 1, filter);
|
entries = namenode.listCacheDirectives(id - 1, filter);
|
||||||
for (int i=0; i<entries.size(); i++) {
|
for (int i = 0; i < entries.size(); i++) {
|
||||||
CacheDirectiveEntry entry = entries.get(i);
|
CacheDirectiveEntry entry = entries.get(i);
|
||||||
if (entry.getInfo().getId().equals((Long)id)) {
|
if (entry.getInfo().getId().equals(id)) {
|
||||||
return new SingleEntry(entry);
|
return new SingleEntry(entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -115,8 +114,6 @@ public class CacheDirectiveIterator
|
||||||
"Did not find requested id " + id);
|
"Did not find requested id " + id);
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
Preconditions.checkNotNull(entries);
|
Preconditions.checkNotNull(entries);
|
||||||
return entries;
|
return entries;
|
||||||
|
|
|
@ -156,14 +156,10 @@ public class CacheDirectiveStats {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder builder = new StringBuilder();
|
return "{" + "bytesNeeded: " + bytesNeeded + ", "
|
||||||
builder.append("{");
|
+ "bytesCached: " + bytesCached + ", "
|
||||||
builder.append("bytesNeeded: ").append(bytesNeeded);
|
+ "filesNeeded: " + filesNeeded + ", "
|
||||||
builder.append(", ").append("bytesCached: ").append(bytesCached);
|
+ "filesCached: " + filesCached + ", "
|
||||||
builder.append(", ").append("filesNeeded: ").append(filesNeeded);
|
+ "hasExpired: " + hasExpired + "}";
|
||||||
builder.append(", ").append("filesCached: ").append(filesCached);
|
|
||||||
builder.append(", ").append("hasExpired: ").append(hasExpired);
|
|
||||||
builder.append("}");
|
|
||||||
return builder.toString();
|
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
|
@ -155,15 +155,13 @@ public class CachePoolInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return new StringBuilder().append("{").
|
return "{" + "poolName:" + poolName
|
||||||
append("poolName:").append(poolName).
|
+ ", ownerName:" + ownerName
|
||||||
append(", ownerName:").append(ownerName).
|
+ ", groupName:" + groupName
|
||||||
append(", groupName:").append(groupName).
|
+ ", mode:"
|
||||||
append(", mode:").append((mode == null) ? "null" :
|
+ ((mode == null) ? "null" : String.format("0%03o", mode.toShort()))
|
||||||
String.format("0%03o", mode.toShort())).
|
+ ", limit:" + limit
|
||||||
append(", limit:").append(limit).
|
+ ", maxRelativeExpiryMs:" + maxRelativeExpiryMs + "}";
|
||||||
append(", maxRelativeExpiryMs:").append(maxRelativeExpiryMs).
|
|
||||||
append("}").toString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -47,11 +47,8 @@ public class CachePoolIterator
|
||||||
@Override
|
@Override
|
||||||
public BatchedEntries<CachePoolEntry> makeRequest(String prevKey)
|
public BatchedEntries<CachePoolEntry> makeRequest(String prevKey)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = tracer.newScope("listCachePools");
|
try (TraceScope ignored = tracer.newScope("listCachePools")) {
|
||||||
try {
|
|
||||||
return namenode.listCachePools(prevKey);
|
return namenode.listCachePools(prevKey);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -66,7 +66,7 @@ public class CachePoolStats {
|
||||||
return new CachePoolStats(bytesNeeded, bytesCached, bytesOverlimit,
|
return new CachePoolStats(bytesNeeded, bytesCached, bytesOverlimit,
|
||||||
filesNeeded, filesCached);
|
filesNeeded, filesCached);
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
private final long bytesNeeded;
|
private final long bytesNeeded;
|
||||||
private final long bytesCached;
|
private final long bytesCached;
|
||||||
|
@ -104,12 +104,10 @@ public class CachePoolStats {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return new StringBuilder().append("{").
|
return "{" + "bytesNeeded:" + bytesNeeded
|
||||||
append("bytesNeeded:").append(bytesNeeded).
|
+ ", bytesCached:" + bytesCached
|
||||||
append(", bytesCached:").append(bytesCached).
|
+ ", bytesOverlimit:" + bytesOverlimit
|
||||||
append(", bytesOverlimit:").append(bytesOverlimit).
|
+ ", filesNeeded:" + filesNeeded
|
||||||
append(", filesNeeded:").append(filesNeeded).
|
+ ", filesCached:" + filesCached + "}";
|
||||||
append(", filesCached:").append(filesCached).
|
|
||||||
append("}").toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,7 @@ public interface ClientDatanodeProtocol {
|
||||||
* 9 is the last version id when this class was used for protocols
|
* 9 is the last version id when this class was used for protocols
|
||||||
* serialization. DO not update this version any further.
|
* serialization. DO not update this version any further.
|
||||||
*/
|
*/
|
||||||
public static final long versionID = 9L;
|
long versionID = 9L;
|
||||||
|
|
||||||
/** Return the visible length of a replica. */
|
/** Return the visible length of a replica. */
|
||||||
long getReplicaVisibleLength(ExtendedBlock b) throws IOException;
|
long getReplicaVisibleLength(ExtendedBlock b) throws IOException;
|
||||||
|
|
|
@ -47,6 +47,7 @@ import org.apache.hadoop.io.EnumSetWritable;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.io.retry.AtMostOnce;
|
import org.apache.hadoop.io.retry.AtMostOnce;
|
||||||
import org.apache.hadoop.io.retry.Idempotent;
|
import org.apache.hadoop.io.retry.Idempotent;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.security.KerberosInfo;
|
import org.apache.hadoop.security.KerberosInfo;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.security.token.TokenInfo;
|
import org.apache.hadoop.security.token.TokenInfo;
|
||||||
|
|
|
@ -42,8 +42,10 @@ public class DSQuotaExceededException extends QuotaExceededException {
|
||||||
String msg = super.getMessage();
|
String msg = super.getMessage();
|
||||||
if (msg == null) {
|
if (msg == null) {
|
||||||
return "The DiskSpace quota" + (pathName==null?"": " of " + pathName)
|
return "The DiskSpace quota" + (pathName==null?"": " of " + pathName)
|
||||||
+ " is exceeded: quota = " + quota + " B = " + long2String(quota, "B", 2)
|
+ " is exceeded: quota = " + quota
|
||||||
+ " but diskspace consumed = " + count + " B = " + long2String(count, "B", 2);
|
+ " B = " + long2String(quota, "B", 2)
|
||||||
|
+ " but diskspace consumed = " + count
|
||||||
|
+ " B = " + long2String(count, "B", 2);
|
||||||
} else {
|
} else {
|
||||||
return msg;
|
return msg;
|
||||||
}
|
}
|
||||||
|
|
|
@ -233,14 +233,10 @@ public class DatanodeID implements Comparable<DatanodeID> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object to) {
|
public boolean equals(Object to) {
|
||||||
if (this == to) {
|
return this == to ||
|
||||||
return true;
|
(to instanceof DatanodeID &&
|
||||||
}
|
getXferAddr().equals(((DatanodeID) to).getXferAddr()) &&
|
||||||
if (!(to instanceof DatanodeID)) {
|
datanodeUuid.equals(((DatanodeID) to).getDatanodeUuid()));
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return (getXferAddr().equals(((DatanodeID)to).getXferAddr()) &&
|
|
||||||
datanodeUuid.equals(((DatanodeID)to).getDatanodeUuid()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -52,7 +52,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
|
||||||
private int xceiverCount;
|
private int xceiverCount;
|
||||||
private String location = NetworkTopology.DEFAULT_RACK;
|
private String location = NetworkTopology.DEFAULT_RACK;
|
||||||
private String softwareVersion;
|
private String softwareVersion;
|
||||||
private List<String> dependentHostNames = new LinkedList<String>();
|
private List<String> dependentHostNames = new LinkedList<>();
|
||||||
private String upgradeDomain;
|
private String upgradeDomain;
|
||||||
|
|
||||||
// Datanode administrative states
|
// Datanode administrative states
|
||||||
|
@ -358,18 +358,18 @@ public class DatanodeInfo extends DatanodeID implements Node {
|
||||||
float cacheRemainingPercent = getCacheRemainingPercent();
|
float cacheRemainingPercent = getCacheRemainingPercent();
|
||||||
String lookupName = NetUtils.getHostNameOfIP(getName());
|
String lookupName = NetUtils.getHostNameOfIP(getName());
|
||||||
|
|
||||||
buffer.append("Name: "+ getName());
|
buffer.append("Name: ").append(getName());
|
||||||
if (lookupName != null) {
|
if (lookupName != null) {
|
||||||
buffer.append(" (" + lookupName + ")");
|
buffer.append(" (").append(lookupName).append(")");
|
||||||
}
|
}
|
||||||
buffer.append("\n");
|
buffer.append("\n");
|
||||||
buffer.append("Hostname: " + getHostName() + "\n");
|
buffer.append("Hostname: ").append(getHostName()).append("\n");
|
||||||
|
|
||||||
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
|
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
|
||||||
buffer.append("Rack: "+location+"\n");
|
buffer.append("Rack: ").append(location).append("\n");
|
||||||
}
|
}
|
||||||
if (upgradeDomain != null) {
|
if (upgradeDomain != null) {
|
||||||
buffer.append("Upgrade domain: "+ upgradeDomain +"\n");
|
buffer.append("Upgrade domain: ").append(upgradeDomain).append("\n");
|
||||||
}
|
}
|
||||||
buffer.append("Decommission Status : ");
|
buffer.append("Decommission Status : ");
|
||||||
if (isDecommissioned()) {
|
if (isDecommissioned()) {
|
||||||
|
@ -379,19 +379,30 @@ public class DatanodeInfo extends DatanodeID implements Node {
|
||||||
} else {
|
} else {
|
||||||
buffer.append("Normal\n");
|
buffer.append("Normal\n");
|
||||||
}
|
}
|
||||||
buffer.append("Configured Capacity: "+c+" ("+StringUtils.byteDesc(c)+")"+"\n");
|
buffer.append("Configured Capacity: ").append(c).append(" (")
|
||||||
buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n");
|
.append(StringUtils.byteDesc(c)).append(")").append("\n");
|
||||||
buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n");
|
buffer.append("DFS Used: ").append(u).append(" (")
|
||||||
buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n");
|
.append(StringUtils.byteDesc(u)).append(")").append("\n");
|
||||||
buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n");
|
buffer.append("Non DFS Used: ").append(nonDFSUsed).append(" (")
|
||||||
buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n");
|
.append(StringUtils.byteDesc(nonDFSUsed)).append(")").append("\n");
|
||||||
buffer.append("Configured Cache Capacity: "+cc+" ("+StringUtils.byteDesc(cc)+")"+"\n");
|
buffer.append("DFS Remaining: ").append(r).append(" (")
|
||||||
buffer.append("Cache Used: "+cu+" ("+StringUtils.byteDesc(cu)+")"+"\n");
|
.append(StringUtils.byteDesc(r)).append(")").append("\n");
|
||||||
buffer.append("Cache Remaining: " +cr+ " ("+StringUtils.byteDesc(cr)+")"+"\n");
|
buffer.append("DFS Used%: ").append(percent2String(usedPercent))
|
||||||
buffer.append("Cache Used%: "+percent2String(cacheUsedPercent) + "\n");
|
.append("\n");
|
||||||
buffer.append("Cache Remaining%: "+percent2String(cacheRemainingPercent) + "\n");
|
buffer.append("DFS Remaining%: ").append(percent2String(remainingPercent))
|
||||||
buffer.append("Xceivers: "+getXceiverCount()+"\n");
|
.append("\n");
|
||||||
buffer.append("Last contact: "+new Date(lastUpdate)+"\n");
|
buffer.append("Configured Cache Capacity: ").append(cc).append(" (")
|
||||||
|
.append(StringUtils.byteDesc(cc)).append(")").append("\n");
|
||||||
|
buffer.append("Cache Used: ").append(cu).append(" (")
|
||||||
|
.append(StringUtils.byteDesc(cu)).append(")").append("\n");
|
||||||
|
buffer.append("Cache Remaining: ").append(cr).append(" (")
|
||||||
|
.append(StringUtils.byteDesc(cr)).append(")").append("\n");
|
||||||
|
buffer.append("Cache Used%: ").append(percent2String(cacheUsedPercent))
|
||||||
|
.append("\n");
|
||||||
|
buffer.append("Cache Remaining%: ")
|
||||||
|
.append(percent2String(cacheRemainingPercent)).append("\n");
|
||||||
|
buffer.append("Xceivers: ").append(getXceiverCount()).append("\n");
|
||||||
|
buffer.append("Last contact: ").append(new Date(lastUpdate)).append("\n");
|
||||||
return buffer.toString();
|
return buffer.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -408,10 +419,10 @@ public class DatanodeInfo extends DatanodeID implements Node {
|
||||||
float cacheUsedPercent = getCacheUsedPercent();
|
float cacheUsedPercent = getCacheUsedPercent();
|
||||||
buffer.append(getName());
|
buffer.append(getName());
|
||||||
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
|
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
|
||||||
buffer.append(" "+location);
|
buffer.append(" ").append(location);
|
||||||
}
|
}
|
||||||
if (upgradeDomain != null) {
|
if (upgradeDomain != null) {
|
||||||
buffer.append(" " + upgradeDomain);
|
buffer.append(" ").append(upgradeDomain);
|
||||||
}
|
}
|
||||||
if (isDecommissioned()) {
|
if (isDecommissioned()) {
|
||||||
buffer.append(" DD");
|
buffer.append(" DD");
|
||||||
|
@ -420,15 +431,21 @@ public class DatanodeInfo extends DatanodeID implements Node {
|
||||||
} else {
|
} else {
|
||||||
buffer.append(" IN");
|
buffer.append(" IN");
|
||||||
}
|
}
|
||||||
buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
|
buffer.append(" ").append(c).append("(").append(StringUtils.byteDesc(c))
|
||||||
buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
|
.append(")");
|
||||||
buffer.append(" " + percent2String(usedPercent));
|
buffer.append(" ").append(u).append("(").append(StringUtils.byteDesc(u))
|
||||||
buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
|
.append(")");
|
||||||
buffer.append(" " + cc + "(" + StringUtils.byteDesc(cc)+")");
|
buffer.append(" ").append(percent2String(usedPercent));
|
||||||
buffer.append(" " + cu + "(" + StringUtils.byteDesc(cu)+")");
|
buffer.append(" ").append(r).append("(").append(StringUtils.byteDesc(r))
|
||||||
buffer.append(" " + percent2String(cacheUsedPercent));
|
.append(")");
|
||||||
buffer.append(" " + cr + "(" + StringUtils.byteDesc(cr)+")");
|
buffer.append(" ").append(cc).append("(").append(StringUtils.byteDesc(cc))
|
||||||
buffer.append(" " + new Date(lastUpdate));
|
.append(")");
|
||||||
|
buffer.append(" ").append(cu).append("(").append(StringUtils.byteDesc(cu))
|
||||||
|
.append(")");
|
||||||
|
buffer.append(" ").append(percent2String(cacheUsedPercent));
|
||||||
|
buffer.append(" ").append(cr).append("(").append(StringUtils.byteDesc(cr))
|
||||||
|
.append(")");
|
||||||
|
buffer.append(" ").append(new Date(lastUpdate));
|
||||||
return buffer.toString();
|
return buffer.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.protocol;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.StorageType;
|
import org.apache.hadoop.fs.StorageType;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
|
||||||
|
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
|
@ -29,7 +28,7 @@ public class DatanodeInfoWithStorage extends DatanodeInfo {
|
||||||
private final StorageType storageType;
|
private final StorageType storageType;
|
||||||
|
|
||||||
public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
|
public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
|
||||||
StorageType storageType) {
|
StorageType storageType) {
|
||||||
super(from);
|
super(from);
|
||||||
this.storageID = storageID;
|
this.storageID = storageID;
|
||||||
this.storageType = storageType;
|
this.storageType = storageType;
|
||||||
|
|
|
@ -55,10 +55,8 @@ public class DatanodeLocalInfo {
|
||||||
|
|
||||||
/** A formatted string for printing the status of the DataNode. */
|
/** A formatted string for printing the status of the DataNode. */
|
||||||
public String getDatanodeLocalReport() {
|
public String getDatanodeLocalReport() {
|
||||||
StringBuilder buffer = new StringBuilder();
|
return ("Uptime: " + getUptime())
|
||||||
buffer.append("Uptime: " + getUptime());
|
+ ", Software version: " + getSoftwareVersion()
|
||||||
buffer.append(", Software version: " + getSoftwareVersion());
|
+ ", Config version: " + getConfigVersion();
|
||||||
buffer.append(", Config version: " + getConfigVersion());
|
|
||||||
return buffer.toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,7 +39,7 @@ public class EncryptionZoneIterator
|
||||||
private final Tracer tracer;
|
private final Tracer tracer;
|
||||||
|
|
||||||
public EncryptionZoneIterator(ClientProtocol namenode, Tracer tracer) {
|
public EncryptionZoneIterator(ClientProtocol namenode, Tracer tracer) {
|
||||||
super(Long.valueOf(0));
|
super((long) 0);
|
||||||
this.namenode = namenode;
|
this.namenode = namenode;
|
||||||
this.tracer = tracer;
|
this.tracer = tracer;
|
||||||
}
|
}
|
||||||
|
@ -47,11 +47,8 @@ public class EncryptionZoneIterator
|
||||||
@Override
|
@Override
|
||||||
public BatchedEntries<EncryptionZone> makeRequest(Long prevId)
|
public BatchedEntries<EncryptionZone> makeRequest(Long prevId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
TraceScope scope = tracer.newScope("listEncryptionZones");
|
try (TraceScope ignored = tracer.newScope("listEncryptionZones")) {
|
||||||
try {
|
|
||||||
return namenode.listEncryptionZones(prevId);
|
return namenode.listEncryptionZones(prevId);
|
||||||
} finally {
|
|
||||||
scope.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,7 +33,8 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class HdfsFileStatus {
|
public class HdfsFileStatus {
|
||||||
|
|
||||||
private final byte[] path; // local name of the inode that's encoded in java UTF8
|
// local name of the inode that's encoded in java UTF8
|
||||||
|
private final byte[] path;
|
||||||
private final byte[] symlink; // symlink target encoded in java UTF8 or null
|
private final byte[] symlink; // symlink target encoded in java UTF8 or null
|
||||||
private final long length;
|
private final long length;
|
||||||
private final boolean isdir;
|
private final boolean isdir;
|
||||||
|
|
|
@ -31,7 +31,8 @@ public class LastBlockWithStatus {
|
||||||
|
|
||||||
private final HdfsFileStatus fileStatus;
|
private final HdfsFileStatus fileStatus;
|
||||||
|
|
||||||
public LastBlockWithStatus(LocatedBlock lastBlock, HdfsFileStatus fileStatus) {
|
public LastBlockWithStatus(LocatedBlock lastBlock,
|
||||||
|
HdfsFileStatus fileStatus) {
|
||||||
this.lastBlock = lastBlock;
|
this.lastBlock = lastBlock;
|
||||||
this.fileStatus = fileStatus;
|
this.fileStatus = fileStatus;
|
||||||
}
|
}
|
||||||
|
|
|
@ -65,13 +65,13 @@ public class LocatedBlock {
|
||||||
}
|
}
|
||||||
|
|
||||||
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs,
|
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs,
|
||||||
String[] storageIDs, StorageType[] storageTypes) {
|
String[] storageIDs, StorageType[] storageTypes) {
|
||||||
this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS);
|
this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS);
|
||||||
}
|
}
|
||||||
|
|
||||||
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs,
|
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs,
|
||||||
StorageType[] storageTypes, long startOffset,
|
StorageType[] storageTypes, long startOffset,
|
||||||
boolean corrupt, DatanodeInfo[] cachedLocs) {
|
boolean corrupt, DatanodeInfo[] cachedLocs) {
|
||||||
this.b = b;
|
this.b = b;
|
||||||
this.offset = startOffset;
|
this.offset = startOffset;
|
||||||
this.corrupt = corrupt;
|
this.corrupt = corrupt;
|
||||||
|
|
|
@ -32,7 +32,8 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class LocatedBlocks {
|
public class LocatedBlocks {
|
||||||
private final long fileLength;
|
private final long fileLength;
|
||||||
private final List<LocatedBlock> blocks; // array of blocks with prioritized locations
|
// array of blocks with prioritized locations
|
||||||
|
private final List<LocatedBlock> blocks;
|
||||||
private final boolean underConstruction;
|
private final boolean underConstruction;
|
||||||
private final LocatedBlock lastLocatedBlock;
|
private final LocatedBlock lastLocatedBlock;
|
||||||
private final boolean isLastBlockComplete;
|
private final boolean isLastBlockComplete;
|
||||||
|
@ -123,22 +124,22 @@ public class LocatedBlocks {
|
||||||
key.setStartOffset(offset);
|
key.setStartOffset(offset);
|
||||||
key.getBlock().setNumBytes(1);
|
key.getBlock().setNumBytes(1);
|
||||||
Comparator<LocatedBlock> comp =
|
Comparator<LocatedBlock> comp =
|
||||||
new Comparator<LocatedBlock>() {
|
new Comparator<LocatedBlock>() {
|
||||||
// Returns 0 iff a is inside b or b is inside a
|
// Returns 0 iff a is inside b or b is inside a
|
||||||
@Override
|
@Override
|
||||||
public int compare(LocatedBlock a, LocatedBlock b) {
|
public int compare(LocatedBlock a, LocatedBlock b) {
|
||||||
long aBeg = a.getStartOffset();
|
long aBeg = a.getStartOffset();
|
||||||
long bBeg = b.getStartOffset();
|
long bBeg = b.getStartOffset();
|
||||||
long aEnd = aBeg + a.getBlockSize();
|
long aEnd = aBeg + a.getBlockSize();
|
||||||
long bEnd = bBeg + b.getBlockSize();
|
long bEnd = bBeg + b.getBlockSize();
|
||||||
if(aBeg <= bBeg && bEnd <= aEnd
|
if(aBeg <= bBeg && bEnd <= aEnd
|
||||||
|| bBeg <= aBeg && aEnd <= bEnd)
|
|| bBeg <= aBeg && aEnd <= bEnd)
|
||||||
return 0; // one of the blocks is inside the other
|
return 0; // one of the blocks is inside the other
|
||||||
if(aBeg < bBeg)
|
if(aBeg < bBeg)
|
||||||
return -1; // a's left bound is to the left of the b's
|
return -1; // a's left bound is to the left of the b's
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return Collections.binarySearch(blocks, key, comp);
|
return Collections.binarySearch(blocks, key, comp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,14 +177,10 @@ public class LocatedBlocks {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
final StringBuilder b = new StringBuilder(getClass().getSimpleName());
|
return getClass().getSimpleName() + "{" + "\n fileLength=" + fileLength
|
||||||
b.append("{")
|
+ "\n underConstruction=" + underConstruction
|
||||||
.append("\n fileLength=").append(fileLength)
|
+ "\n blocks=" + blocks
|
||||||
.append("\n underConstruction=").append(underConstruction)
|
+ "\n lastLocatedBlock=" + lastLocatedBlock
|
||||||
.append("\n blocks=").append(blocks)
|
+ "\n isLastBlockComplete=" + isLastBlockComplete + "}";
|
||||||
.append("\n lastLocatedBlock=").append(lastLocatedBlock)
|
|
||||||
.append("\n isLastBlockComplete=").append(isLastBlockComplete)
|
|
||||||
.append("}");
|
|
||||||
return b.toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,7 +43,7 @@ public final class NSQuotaExceededException extends QuotaExceededException {
|
||||||
String msg = super.getMessage();
|
String msg = super.getMessage();
|
||||||
if (msg == null) {
|
if (msg == null) {
|
||||||
msg = "The NameSpace quota (directories and files)" +
|
msg = "The NameSpace quota (directories and files)" +
|
||||||
(pathName==null?"":(" of directory " + pathName)) +
|
(pathName==null?"":(" of directory " + pathName)) +
|
||||||
" is exceeded: quota=" + quota + " file count=" + count;
|
" is exceeded: quota=" + quota + " file count=" + count;
|
||||||
|
|
||||||
if (prefix != null) {
|
if (prefix != null) {
|
||||||
|
|
|
@ -64,7 +64,6 @@ public class RollingUpgradeInfo extends RollingUpgradeStatus {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finalize the upgrade if not already finalized
|
* Finalize the upgrade if not already finalized
|
||||||
* @param finalizeTime
|
|
||||||
*/
|
*/
|
||||||
public void finalize(long finalizeTime) {
|
public void finalize(long finalizeTime) {
|
||||||
if (finalizeTime != 0) {
|
if (finalizeTime != 0) {
|
||||||
|
@ -99,8 +98,11 @@ public class RollingUpgradeInfo extends RollingUpgradeStatus {
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return super.toString()
|
return super.toString()
|
||||||
+ "\n Start Time: " + (startTime == 0? "<NOT STARTED>": timestamp2String(startTime))
|
+ "\n Start Time: "
|
||||||
+ "\n Finalize Time: " + (finalizeTime == 0? "<NOT FINALIZED>": timestamp2String(finalizeTime));
|
+ (startTime == 0 ? "<NOT STARTED>" : timestamp2String(startTime))
|
||||||
|
+ "\n Finalize Time: "
|
||||||
|
+ (finalizeTime == 0 ? "<NOT FINALIZED>" :
|
||||||
|
timestamp2String(finalizeTime));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String timestamp2String(long timestamp) {
|
private static String timestamp2String(long timestamp) {
|
||||||
|
|
|
@ -49,7 +49,7 @@ public class SnapshotDiffReport {
|
||||||
|
|
||||||
private final String label;
|
private final String label;
|
||||||
|
|
||||||
private DiffType(String label) {
|
DiffType(String label) {
|
||||||
this.label = label;
|
this.label = label;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -69,7 +69,7 @@ public class SnapshotDiffReport {
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Representing the full path and diff type of a file/directory where changes
|
* Representing the full path and diff type of a file/directory where changes
|
||||||
|
@ -205,10 +205,11 @@ public class SnapshotDiffReport {
|
||||||
"current directory" : "snapshot " + fromSnapshot;
|
"current directory" : "snapshot " + fromSnapshot;
|
||||||
String to = toSnapshot == null || toSnapshot.isEmpty() ? "current directory"
|
String to = toSnapshot == null || toSnapshot.isEmpty() ? "current directory"
|
||||||
: "snapshot " + toSnapshot;
|
: "snapshot " + toSnapshot;
|
||||||
str.append("Difference between " + from + " and " + to
|
str.append("Difference between ").append(from).append(" and ").append(to)
|
||||||
+ " under directory " + snapshotRoot + ":" + LINE_SEPARATOR);
|
.append(" under directory ").append(snapshotRoot).append(":")
|
||||||
|
.append(LINE_SEPARATOR);
|
||||||
for (DiffReportEntry entry : diffList) {
|
for (DiffReportEntry entry : diffList) {
|
||||||
str.append(entry.toString() + LINE_SEPARATOR);
|
str.append(entry.toString()).append(LINE_SEPARATOR);
|
||||||
}
|
}
|
||||||
return str.toString();
|
return str.toString();
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,17 +31,18 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
|
||||||
*/
|
*/
|
||||||
public class SnapshottableDirectoryStatus {
|
public class SnapshottableDirectoryStatus {
|
||||||
/** Compare the statuses by full paths. */
|
/** Compare the statuses by full paths. */
|
||||||
public static final Comparator<SnapshottableDirectoryStatus> COMPARATOR
|
public static final Comparator<SnapshottableDirectoryStatus> COMPARATOR =
|
||||||
= new Comparator<SnapshottableDirectoryStatus>() {
|
new Comparator<SnapshottableDirectoryStatus>() {
|
||||||
@Override
|
@Override
|
||||||
public int compare(SnapshottableDirectoryStatus left,
|
public int compare(SnapshottableDirectoryStatus left,
|
||||||
SnapshottableDirectoryStatus right) {
|
SnapshottableDirectoryStatus right) {
|
||||||
int d = DFSUtilClient.compareBytes(left.parentFullPath, right.parentFullPath);
|
int d = DFSUtilClient.compareBytes(left.parentFullPath,
|
||||||
return d != 0? d
|
right.parentFullPath);
|
||||||
: DFSUtilClient.compareBytes(left.dirStatus.getLocalNameInBytes(),
|
return d != 0? d
|
||||||
|
: DFSUtilClient.compareBytes(left.dirStatus.getLocalNameInBytes(),
|
||||||
right.dirStatus.getLocalNameInBytes());
|
right.dirStatus.getLocalNameInBytes());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Basic information of the snapshottable directory */
|
/** Basic information of the snapshottable directory */
|
||||||
private final HdfsFileStatus dirStatus;
|
private final HdfsFileStatus dirStatus;
|
||||||
|
@ -134,18 +135,16 @@ public class SnapshottableDirectoryStatus {
|
||||||
maxSnapshotQuota = maxLength(maxSnapshotQuota, status.snapshotQuota);
|
maxSnapshotQuota = maxLength(maxSnapshotQuota, status.snapshotQuota);
|
||||||
}
|
}
|
||||||
|
|
||||||
StringBuilder fmt = new StringBuilder();
|
String lineFormat = "%s%s " // permission string
|
||||||
fmt.append("%s%s "); // permission string
|
+ "%" + maxRepl + "s "
|
||||||
fmt.append("%" + maxRepl + "s ");
|
+ (maxOwner > 0 ? "%-" + maxOwner + "s " : "%s")
|
||||||
fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
|
+ (maxGroup > 0 ? "%-" + maxGroup + "s " : "%s")
|
||||||
fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
|
+ "%" + maxLen + "s "
|
||||||
fmt.append("%" + maxLen + "s ");
|
+ "%s " // mod time
|
||||||
fmt.append("%s "); // mod time
|
+ "%" + maxSnapshotNum + "s "
|
||||||
fmt.append("%" + maxSnapshotNum + "s ");
|
+ "%" + maxSnapshotQuota + "s "
|
||||||
fmt.append("%" + maxSnapshotQuota + "s ");
|
+ "%s"; // path
|
||||||
fmt.append("%s"); // path
|
|
||||||
|
|
||||||
String lineFormat = fmt.toString();
|
|
||||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
|
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
|
||||||
|
|
||||||
for (SnapshottableDirectoryStatus status : stats) {
|
for (SnapshottableDirectoryStatus status : stats) {
|
||||||
|
|
|
@ -18,8 +18,6 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hdfs.protocol;
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.UnresolvedLinkException;
|
import org.apache.hadoop.fs.UnresolvedLinkException;
|
||||||
|
@ -55,7 +53,7 @@ public final class UnresolvedPathException extends UnresolvedLinkException {
|
||||||
/**
|
/**
|
||||||
* Return a path with the link resolved with the target.
|
* Return a path with the link resolved with the target.
|
||||||
*/
|
*/
|
||||||
public Path getResolvedPath() throws IOException {
|
public Path getResolvedPath() {
|
||||||
// If the path is absolute we cam throw out the preceding part and
|
// If the path is absolute we cam throw out the preceding part and
|
||||||
// just append the remainder to the target, otherwise append each
|
// just append the remainder to the target, otherwise append each
|
||||||
// piece to resolve the link in path.
|
// piece to resolve the link in path.
|
||||||
|
@ -76,12 +74,6 @@ public final class UnresolvedPathException extends UnresolvedLinkException {
|
||||||
if (msg != null) {
|
if (msg != null) {
|
||||||
return msg;
|
return msg;
|
||||||
}
|
}
|
||||||
String myMsg = "Unresolved path " + path;
|
return getResolvedPath().toString();
|
||||||
try {
|
|
||||||
return getResolvedPath().toString();
|
|
||||||
} catch (IOException e) {
|
|
||||||
// Ignore
|
|
||||||
}
|
|
||||||
return myMsg;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -65,7 +65,9 @@ public abstract class DataTransferProtoUtil {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DataChecksum fromProto(ChecksumProto proto) {
|
public static DataChecksum fromProto(ChecksumProto proto) {
|
||||||
if (proto == null) return null;
|
if (proto == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
int bytesPerChecksum = proto.getBytesPerChecksum();
|
int bytesPerChecksum = proto.getBytesPerChecksum();
|
||||||
DataChecksum.Type type = PBHelperClient.convert(proto.getType());
|
DataChecksum.Type type = PBHelperClient.convert(proto.getType());
|
||||||
|
@ -74,19 +76,17 @@ public abstract class DataTransferProtoUtil {
|
||||||
|
|
||||||
static ClientOperationHeaderProto buildClientHeader(ExtendedBlock blk,
|
static ClientOperationHeaderProto buildClientHeader(ExtendedBlock blk,
|
||||||
String client, Token<BlockTokenIdentifier> blockToken) {
|
String client, Token<BlockTokenIdentifier> blockToken) {
|
||||||
ClientOperationHeaderProto header =
|
return ClientOperationHeaderProto.newBuilder()
|
||||||
ClientOperationHeaderProto.newBuilder()
|
.setBaseHeader(buildBaseHeader(blk, blockToken))
|
||||||
.setBaseHeader(buildBaseHeader(blk, blockToken))
|
.setClientName(client)
|
||||||
.setClientName(client)
|
.build();
|
||||||
.build();
|
|
||||||
return header;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static BaseHeaderProto buildBaseHeader(ExtendedBlock blk,
|
static BaseHeaderProto buildBaseHeader(ExtendedBlock blk,
|
||||||
Token<BlockTokenIdentifier> blockToken) {
|
Token<BlockTokenIdentifier> blockToken) {
|
||||||
BaseHeaderProto.Builder builder = BaseHeaderProto.newBuilder()
|
BaseHeaderProto.Builder builder = BaseHeaderProto.newBuilder()
|
||||||
.setBlock(PBHelperClient.convert(blk))
|
.setBlock(PBHelperClient.convert(blk))
|
||||||
.setToken(PBHelperClient.convert(blockToken));
|
.setToken(PBHelperClient.convert(blockToken));
|
||||||
SpanId spanId = Tracer.getCurrentSpanId();
|
SpanId spanId = Tracer.getCurrentSpanId();
|
||||||
if (spanId.isValid()) {
|
if (spanId.isValid()) {
|
||||||
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder()
|
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder()
|
||||||
|
|
|
@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public interface DataTransferProtocol {
|
public interface DataTransferProtocol {
|
||||||
public static final Logger LOG = LoggerFactory.getLogger(DataTransferProtocol.class);
|
Logger LOG = LoggerFactory.getLogger(DataTransferProtocol.class);
|
||||||
|
|
||||||
/** Version for data transfers between clients and datanodes
|
/** Version for data transfers between clients and datanodes
|
||||||
* This should change when serialization of DatanodeInfo, not just
|
* This should change when serialization of DatanodeInfo, not just
|
||||||
|
@ -49,7 +49,7 @@ public interface DataTransferProtocol {
|
||||||
* Version 28:
|
* Version 28:
|
||||||
* Declare methods in DataTransferProtocol interface.
|
* Declare methods in DataTransferProtocol interface.
|
||||||
*/
|
*/
|
||||||
public static final int DATA_TRANSFER_VERSION = 28;
|
int DATA_TRANSFER_VERSION = 28;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read a block.
|
* Read a block.
|
||||||
|
@ -63,7 +63,7 @@ public interface DataTransferProtocol {
|
||||||
* checksums
|
* checksums
|
||||||
* @param cachingStrategy The caching strategy to use.
|
* @param cachingStrategy The caching strategy to use.
|
||||||
*/
|
*/
|
||||||
public void readBlock(final ExtendedBlock blk,
|
void readBlock(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
final String clientName,
|
final String clientName,
|
||||||
final long blockOffset,
|
final long blockOffset,
|
||||||
|
@ -96,7 +96,7 @@ public interface DataTransferProtocol {
|
||||||
* @param pinning whether to pin the block, so Balancer won't move it.
|
* @param pinning whether to pin the block, so Balancer won't move it.
|
||||||
* @param targetPinnings whether to pin the block on target datanode
|
* @param targetPinnings whether to pin the block on target datanode
|
||||||
*/
|
*/
|
||||||
public void writeBlock(final ExtendedBlock blk,
|
void writeBlock(final ExtendedBlock blk,
|
||||||
final StorageType storageType,
|
final StorageType storageType,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
final String clientName,
|
final String clientName,
|
||||||
|
@ -124,7 +124,7 @@ public interface DataTransferProtocol {
|
||||||
* @param clientName client's name.
|
* @param clientName client's name.
|
||||||
* @param targets target datanodes.
|
* @param targets target datanodes.
|
||||||
*/
|
*/
|
||||||
public void transferBlock(final ExtendedBlock blk,
|
void transferBlock(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
final String clientName,
|
final String clientName,
|
||||||
final DatanodeInfo[] targets,
|
final DatanodeInfo[] targets,
|
||||||
|
@ -142,7 +142,7 @@ public interface DataTransferProtocol {
|
||||||
* @param supportsReceiptVerification True if the client supports
|
* @param supportsReceiptVerification True if the client supports
|
||||||
* receipt verification.
|
* receipt verification.
|
||||||
*/
|
*/
|
||||||
public void requestShortCircuitFds(final ExtendedBlock blk,
|
void requestShortCircuitFds(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
SlotId slotId, int maxVersion, boolean supportsReceiptVerification)
|
SlotId slotId, int maxVersion, boolean supportsReceiptVerification)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
|
@ -152,14 +152,14 @@ public interface DataTransferProtocol {
|
||||||
*
|
*
|
||||||
* @param slotId SlotID used by the earlier file descriptors.
|
* @param slotId SlotID used by the earlier file descriptors.
|
||||||
*/
|
*/
|
||||||
public void releaseShortCircuitFds(final SlotId slotId) throws IOException;
|
void releaseShortCircuitFds(final SlotId slotId) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Request a short circuit shared memory area from a DataNode.
|
* Request a short circuit shared memory area from a DataNode.
|
||||||
*
|
*
|
||||||
* @param clientName The name of the client.
|
* @param clientName The name of the client.
|
||||||
*/
|
*/
|
||||||
public void requestShortCircuitShm(String clientName) throws IOException;
|
void requestShortCircuitShm(String clientName) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Receive a block from a source datanode
|
* Receive a block from a source datanode
|
||||||
|
@ -174,7 +174,7 @@ public interface DataTransferProtocol {
|
||||||
* @param delHint the hint for deleting the block in the original datanode.
|
* @param delHint the hint for deleting the block in the original datanode.
|
||||||
* @param source the source datanode for receiving the block.
|
* @param source the source datanode for receiving the block.
|
||||||
*/
|
*/
|
||||||
public void replaceBlock(final ExtendedBlock blk,
|
void replaceBlock(final ExtendedBlock blk,
|
||||||
final StorageType storageType,
|
final StorageType storageType,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
final String delHint,
|
final String delHint,
|
||||||
|
@ -187,7 +187,7 @@ public interface DataTransferProtocol {
|
||||||
* @param blk the block being copied.
|
* @param blk the block being copied.
|
||||||
* @param blockToken security token for accessing the block.
|
* @param blockToken security token for accessing the block.
|
||||||
*/
|
*/
|
||||||
public void copyBlock(final ExtendedBlock blk,
|
void copyBlock(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken) throws IOException;
|
final Token<BlockTokenIdentifier> blockToken) throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -197,6 +197,6 @@ public interface DataTransferProtocol {
|
||||||
* @param blockToken security token for accessing the block.
|
* @param blockToken security token for accessing the block.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void blockChecksum(final ExtendedBlock blk,
|
void blockChecksum(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken) throws IOException;
|
final Token<BlockTokenIdentifier> blockToken) throws IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,7 +43,7 @@ public enum Op {
|
||||||
/** The code for this operation. */
|
/** The code for this operation. */
|
||||||
public final byte code;
|
public final byte code;
|
||||||
|
|
||||||
private Op(byte code) {
|
Op(byte code) {
|
||||||
this.code = code;
|
this.code = code;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,8 +51,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class PacketHeader {
|
public class PacketHeader {
|
||||||
private static final int MAX_PROTO_SIZE =
|
private static final int MAX_PROTO_SIZE = PacketHeaderProto.newBuilder()
|
||||||
PacketHeaderProto.newBuilder()
|
|
||||||
.setOffsetInBlock(0)
|
.setOffsetInBlock(0)
|
||||||
.setSeqno(0)
|
.setSeqno(0)
|
||||||
.setLastPacketInBlock(false)
|
.setLastPacketInBlock(false)
|
||||||
|
@ -78,17 +77,17 @@ public class PacketHeader {
|
||||||
packetLen);
|
packetLen);
|
||||||
|
|
||||||
PacketHeaderProto.Builder builder = PacketHeaderProto.newBuilder()
|
PacketHeaderProto.Builder builder = PacketHeaderProto.newBuilder()
|
||||||
.setOffsetInBlock(offsetInBlock)
|
.setOffsetInBlock(offsetInBlock)
|
||||||
.setSeqno(seqno)
|
.setSeqno(seqno)
|
||||||
.setLastPacketInBlock(lastPacketInBlock)
|
.setLastPacketInBlock(lastPacketInBlock)
|
||||||
.setDataLen(dataLen);
|
.setDataLen(dataLen);
|
||||||
|
|
||||||
if (syncBlock) {
|
if (syncBlock) {
|
||||||
// Only set syncBlock if it is specified.
|
// Only set syncBlock if it is specified.
|
||||||
// This is wire-incompatible with Hadoop 2.0.0-alpha due to HDFS-3721
|
// This is wire-incompatible with Hadoop 2.0.0-alpha due to HDFS-3721
|
||||||
// because it changes the length of the packet header, and BlockReceiver
|
// because it changes the length of the packet header, and BlockReceiver
|
||||||
// in that version did not support variable-length headers.
|
// in that version did not support variable-length headers.
|
||||||
builder.setSyncBlock(syncBlock);
|
builder.setSyncBlock(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
proto = builder.build();
|
proto = builder.build();
|
||||||
|
@ -187,8 +186,8 @@ public class PacketHeader {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform a sanity check on the packet, returning true if it is sane.
|
* Perform a sanity check on the packet, returning true if it is sane.
|
||||||
* @param lastSeqNo the previous sequence number received - we expect the current
|
* @param lastSeqNo the previous sequence number received - we expect the
|
||||||
* sequence number to be larger by 1.
|
* current sequence number to be larger by 1.
|
||||||
*/
|
*/
|
||||||
public boolean sanityCheck(long lastSeqNo) {
|
public boolean sanityCheck(long lastSeqNo) {
|
||||||
// We should only have a non-positive data length for the last packet
|
// We should only have a non-positive data length for the last packet
|
||||||
|
@ -196,8 +195,7 @@ public class PacketHeader {
|
||||||
// The last packet should not contain data
|
// The last packet should not contain data
|
||||||
if (proto.getLastPacketInBlock() && proto.getDataLen() != 0) return false;
|
if (proto.getLastPacketInBlock() && proto.getDataLen() != 0) return false;
|
||||||
// Seqnos should always increase by 1 with each packet received
|
// Seqnos should always increase by 1 with each packet received
|
||||||
if (proto.getSeqno() != lastSeqNo + 1) return false;
|
return proto.getSeqno() == lastSeqNo + 1;
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -43,7 +43,7 @@ public class ReplaceDatanodeOnFailure {
|
||||||
|
|
||||||
private final Condition condition;
|
private final Condition condition;
|
||||||
|
|
||||||
private Policy(Condition condition) {
|
Policy(Condition condition) {
|
||||||
this.condition = condition;
|
this.condition = condition;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,9 +53,9 @@ public class ReplaceDatanodeOnFailure {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Datanode replacement condition */
|
/** Datanode replacement condition */
|
||||||
private static interface Condition {
|
private interface Condition {
|
||||||
/** Return true unconditionally. */
|
/** Return true unconditionally. */
|
||||||
static final Condition TRUE = new Condition() {
|
Condition TRUE = new Condition() {
|
||||||
@Override
|
@Override
|
||||||
public boolean satisfy(short replication, DatanodeInfo[] existings,
|
public boolean satisfy(short replication, DatanodeInfo[] existings,
|
||||||
int nExistings, boolean isAppend, boolean isHflushed) {
|
int nExistings, boolean isAppend, boolean isHflushed) {
|
||||||
|
@ -64,7 +64,7 @@ public class ReplaceDatanodeOnFailure {
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Return false unconditionally. */
|
/** Return false unconditionally. */
|
||||||
static final Condition FALSE = new Condition() {
|
Condition FALSE = new Condition() {
|
||||||
@Override
|
@Override
|
||||||
public boolean satisfy(short replication, DatanodeInfo[] existings,
|
public boolean satisfy(short replication, DatanodeInfo[] existings,
|
||||||
int nExistings, boolean isAppend, boolean isHflushed) {
|
int nExistings, boolean isAppend, boolean isHflushed) {
|
||||||
|
@ -80,26 +80,19 @@ public class ReplaceDatanodeOnFailure {
|
||||||
* (1) floor(r/2) >= n; or
|
* (1) floor(r/2) >= n; or
|
||||||
* (2) r > n and the block is hflushed/appended.
|
* (2) r > n and the block is hflushed/appended.
|
||||||
*/
|
*/
|
||||||
static final Condition DEFAULT = new Condition() {
|
Condition DEFAULT = new Condition() {
|
||||||
@Override
|
@Override
|
||||||
public boolean satisfy(final short replication,
|
public boolean satisfy(final short replication,
|
||||||
final DatanodeInfo[] existings, final int n, final boolean isAppend,
|
final DatanodeInfo[] existings, final int n, final boolean isAppend,
|
||||||
final boolean isHflushed) {
|
final boolean isHflushed) {
|
||||||
if (replication < 3) {
|
return replication >= 3 &&
|
||||||
return false;
|
(n <= (replication / 2) || isAppend || isHflushed);
|
||||||
} else {
|
|
||||||
if (n <= (replication/2)) {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
return isAppend || isHflushed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Is the condition satisfied? */
|
/** Is the condition satisfied? */
|
||||||
public boolean satisfy(short replication, DatanodeInfo[] existings,
|
boolean satisfy(short replication, DatanodeInfo[] existings, int nExistings,
|
||||||
int nExistings, boolean isAppend, boolean isHflushed);
|
boolean isAppend, boolean isHflushed);
|
||||||
}
|
}
|
||||||
|
|
||||||
private final Policy policy;
|
private final Policy policy;
|
||||||
|
@ -137,14 +130,11 @@ public class ReplaceDatanodeOnFailure {
|
||||||
public boolean satisfy(
|
public boolean satisfy(
|
||||||
final short replication, final DatanodeInfo[] existings,
|
final short replication, final DatanodeInfo[] existings,
|
||||||
final boolean isAppend, final boolean isHflushed) {
|
final boolean isAppend, final boolean isHflushed) {
|
||||||
final int n = existings == null? 0: existings.length;
|
final int n = existings == null ? 0 : existings.length;
|
||||||
if (n == 0 || n >= replication) {
|
//don't need to add datanode for any policy.
|
||||||
//don't need to add datanode for any policy.
|
return !(n == 0 || n >= replication) &&
|
||||||
return false;
|
policy.getCondition().satisfy(replication, existings, n, isAppend,
|
||||||
} else {
|
isHflushed);
|
||||||
return policy.getCondition().satisfy(
|
|
||||||
replication, existings, n, isAppend, isHflushed);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -65,8 +65,7 @@ public class Sender implements DataTransferProtocol {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Initialize a operation. */
|
/** Initialize a operation. */
|
||||||
private static void op(final DataOutput out, final Op op
|
private static void op(final DataOutput out, final Op op) throws IOException {
|
||||||
) throws IOException {
|
|
||||||
out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
|
out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
|
||||||
op.write(out);
|
op.write(out);
|
||||||
}
|
}
|
||||||
|
@ -80,13 +79,14 @@ public class Sender implements DataTransferProtocol {
|
||||||
out.flush();
|
out.flush();
|
||||||
}
|
}
|
||||||
|
|
||||||
static private CachingStrategyProto getCachingStrategy(CachingStrategy cachingStrategy) {
|
static private CachingStrategyProto getCachingStrategy(
|
||||||
|
CachingStrategy cachingStrategy) {
|
||||||
CachingStrategyProto.Builder builder = CachingStrategyProto.newBuilder();
|
CachingStrategyProto.Builder builder = CachingStrategyProto.newBuilder();
|
||||||
if (cachingStrategy.getReadahead() != null) {
|
if (cachingStrategy.getReadahead() != null) {
|
||||||
builder.setReadahead(cachingStrategy.getReadahead().longValue());
|
builder.setReadahead(cachingStrategy.getReadahead());
|
||||||
}
|
}
|
||||||
if (cachingStrategy.getDropBehind() != null) {
|
if (cachingStrategy.getDropBehind() != null) {
|
||||||
builder.setDropBehind(cachingStrategy.getDropBehind().booleanValue());
|
builder.setDropBehind(cachingStrategy.getDropBehind());
|
||||||
}
|
}
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
@ -101,12 +101,13 @@ public class Sender implements DataTransferProtocol {
|
||||||
final CachingStrategy cachingStrategy) throws IOException {
|
final CachingStrategy cachingStrategy) throws IOException {
|
||||||
|
|
||||||
OpReadBlockProto proto = OpReadBlockProto.newBuilder()
|
OpReadBlockProto proto = OpReadBlockProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildClientHeader(blk, clientName, blockToken))
|
.setHeader(DataTransferProtoUtil.buildClientHeader(blk, clientName,
|
||||||
.setOffset(blockOffset)
|
blockToken))
|
||||||
.setLen(length)
|
.setOffset(blockOffset)
|
||||||
.setSendChecksums(sendChecksum)
|
.setLen(length)
|
||||||
.setCachingStrategy(getCachingStrategy(cachingStrategy))
|
.setSendChecksums(sendChecksum)
|
||||||
.build();
|
.setCachingStrategy(getCachingStrategy(cachingStrategy))
|
||||||
|
.build();
|
||||||
|
|
||||||
send(out, Op.READ_BLOCK, proto);
|
send(out, Op.READ_BLOCK, proto);
|
||||||
}
|
}
|
||||||
|
@ -134,23 +135,24 @@ public class Sender implements DataTransferProtocol {
|
||||||
blk, clientName, blockToken);
|
blk, clientName, blockToken);
|
||||||
|
|
||||||
ChecksumProto checksumProto =
|
ChecksumProto checksumProto =
|
||||||
DataTransferProtoUtil.toProto(requestedChecksum);
|
DataTransferProtoUtil.toProto(requestedChecksum);
|
||||||
|
|
||||||
OpWriteBlockProto.Builder proto = OpWriteBlockProto.newBuilder()
|
OpWriteBlockProto.Builder proto = OpWriteBlockProto.newBuilder()
|
||||||
.setHeader(header)
|
.setHeader(header)
|
||||||
.setStorageType(PBHelperClient.convertStorageType(storageType))
|
.setStorageType(PBHelperClient.convertStorageType(storageType))
|
||||||
.addAllTargets(PBHelperClient.convert(targets, 1))
|
.addAllTargets(PBHelperClient.convert(targets, 1))
|
||||||
.addAllTargetStorageTypes(PBHelperClient.convertStorageTypes(targetStorageTypes, 1))
|
.addAllTargetStorageTypes(
|
||||||
.setStage(toProto(stage))
|
PBHelperClient.convertStorageTypes(targetStorageTypes, 1))
|
||||||
.setPipelineSize(pipelineSize)
|
.setStage(toProto(stage))
|
||||||
.setMinBytesRcvd(minBytesRcvd)
|
.setPipelineSize(pipelineSize)
|
||||||
.setMaxBytesRcvd(maxBytesRcvd)
|
.setMinBytesRcvd(minBytesRcvd)
|
||||||
.setLatestGenerationStamp(latestGenerationStamp)
|
.setMaxBytesRcvd(maxBytesRcvd)
|
||||||
.setRequestedChecksum(checksumProto)
|
.setLatestGenerationStamp(latestGenerationStamp)
|
||||||
.setCachingStrategy(getCachingStrategy(cachingStrategy))
|
.setRequestedChecksum(checksumProto)
|
||||||
.setAllowLazyPersist(allowLazyPersist)
|
.setCachingStrategy(getCachingStrategy(cachingStrategy))
|
||||||
.setPinning(pinning)
|
.setAllowLazyPersist(allowLazyPersist)
|
||||||
.addAllTargetPinnings(PBHelperClient.convert(targetPinnings, 1));
|
.setPinning(pinning)
|
||||||
|
.addAllTargetPinnings(PBHelperClient.convert(targetPinnings, 1));
|
||||||
|
|
||||||
if (source != null) {
|
if (source != null) {
|
||||||
proto.setSource(PBHelperClient.convertDatanodeInfo(source));
|
proto.setSource(PBHelperClient.convertDatanodeInfo(source));
|
||||||
|
@ -167,11 +169,12 @@ public class Sender implements DataTransferProtocol {
|
||||||
final StorageType[] targetStorageTypes) throws IOException {
|
final StorageType[] targetStorageTypes) throws IOException {
|
||||||
|
|
||||||
OpTransferBlockProto proto = OpTransferBlockProto.newBuilder()
|
OpTransferBlockProto proto = OpTransferBlockProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildClientHeader(
|
.setHeader(DataTransferProtoUtil.buildClientHeader(
|
||||||
blk, clientName, blockToken))
|
blk, clientName, blockToken))
|
||||||
.addAllTargets(PBHelperClient.convert(targets))
|
.addAllTargets(PBHelperClient.convert(targets))
|
||||||
.addAllTargetStorageTypes(PBHelperClient.convertStorageTypes(targetStorageTypes))
|
.addAllTargetStorageTypes(
|
||||||
.build();
|
PBHelperClient.convertStorageTypes(targetStorageTypes))
|
||||||
|
.build();
|
||||||
|
|
||||||
send(out, Op.TRANSFER_BLOCK, proto);
|
send(out, Op.TRANSFER_BLOCK, proto);
|
||||||
}
|
}
|
||||||
|
@ -180,11 +183,11 @@ public class Sender implements DataTransferProtocol {
|
||||||
public void requestShortCircuitFds(final ExtendedBlock blk,
|
public void requestShortCircuitFds(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken,
|
final Token<BlockTokenIdentifier> blockToken,
|
||||||
SlotId slotId, int maxVersion, boolean supportsReceiptVerification)
|
SlotId slotId, int maxVersion, boolean supportsReceiptVerification)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
OpRequestShortCircuitAccessProto.Builder builder =
|
OpRequestShortCircuitAccessProto.Builder builder =
|
||||||
OpRequestShortCircuitAccessProto.newBuilder()
|
OpRequestShortCircuitAccessProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildBaseHeader(
|
.setHeader(DataTransferProtoUtil.buildBaseHeader(
|
||||||
blk, blockToken)).setMaxVersion(maxVersion);
|
blk, blockToken)).setMaxVersion(maxVersion);
|
||||||
if (slotId != null) {
|
if (slotId != null) {
|
||||||
builder.setSlotId(PBHelperClient.convert(slotId));
|
builder.setSlotId(PBHelperClient.convert(slotId));
|
||||||
}
|
}
|
||||||
|
@ -197,7 +200,7 @@ public class Sender implements DataTransferProtocol {
|
||||||
public void releaseShortCircuitFds(SlotId slotId) throws IOException {
|
public void releaseShortCircuitFds(SlotId slotId) throws IOException {
|
||||||
ReleaseShortCircuitAccessRequestProto.Builder builder =
|
ReleaseShortCircuitAccessRequestProto.Builder builder =
|
||||||
ReleaseShortCircuitAccessRequestProto.newBuilder().
|
ReleaseShortCircuitAccessRequestProto.newBuilder().
|
||||||
setSlotId(PBHelperClient.convert(slotId));
|
setSlotId(PBHelperClient.convert(slotId));
|
||||||
SpanId spanId = Tracer.getCurrentSpanId();
|
SpanId spanId = Tracer.getCurrentSpanId();
|
||||||
if (spanId.isValid()) {
|
if (spanId.isValid()) {
|
||||||
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder().
|
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder().
|
||||||
|
@ -212,7 +215,7 @@ public class Sender implements DataTransferProtocol {
|
||||||
public void requestShortCircuitShm(String clientName) throws IOException {
|
public void requestShortCircuitShm(String clientName) throws IOException {
|
||||||
ShortCircuitShmRequestProto.Builder builder =
|
ShortCircuitShmRequestProto.Builder builder =
|
||||||
ShortCircuitShmRequestProto.newBuilder().
|
ShortCircuitShmRequestProto.newBuilder().
|
||||||
setClientName(clientName);
|
setClientName(clientName);
|
||||||
SpanId spanId = Tracer.getCurrentSpanId();
|
SpanId spanId = Tracer.getCurrentSpanId();
|
||||||
if (spanId.isValid()) {
|
if (spanId.isValid()) {
|
||||||
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder().
|
builder.setTraceInfo(DataTransferTraceInfoProto.newBuilder().
|
||||||
|
@ -230,11 +233,11 @@ public class Sender implements DataTransferProtocol {
|
||||||
final String delHint,
|
final String delHint,
|
||||||
final DatanodeInfo source) throws IOException {
|
final DatanodeInfo source) throws IOException {
|
||||||
OpReplaceBlockProto proto = OpReplaceBlockProto.newBuilder()
|
OpReplaceBlockProto proto = OpReplaceBlockProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
||||||
.setStorageType(PBHelperClient.convertStorageType(storageType))
|
.setStorageType(PBHelperClient.convertStorageType(storageType))
|
||||||
.setDelHint(delHint)
|
.setDelHint(delHint)
|
||||||
.setSource(PBHelperClient.convertDatanodeInfo(source))
|
.setSource(PBHelperClient.convertDatanodeInfo(source))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
send(out, Op.REPLACE_BLOCK, proto);
|
send(out, Op.REPLACE_BLOCK, proto);
|
||||||
}
|
}
|
||||||
|
@ -243,8 +246,8 @@ public class Sender implements DataTransferProtocol {
|
||||||
public void copyBlock(final ExtendedBlock blk,
|
public void copyBlock(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken) throws IOException {
|
final Token<BlockTokenIdentifier> blockToken) throws IOException {
|
||||||
OpCopyBlockProto proto = OpCopyBlockProto.newBuilder()
|
OpCopyBlockProto proto = OpCopyBlockProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
send(out, Op.COPY_BLOCK, proto);
|
send(out, Op.COPY_BLOCK, proto);
|
||||||
}
|
}
|
||||||
|
@ -253,8 +256,8 @@ public class Sender implements DataTransferProtocol {
|
||||||
public void blockChecksum(final ExtendedBlock blk,
|
public void blockChecksum(final ExtendedBlock blk,
|
||||||
final Token<BlockTokenIdentifier> blockToken) throws IOException {
|
final Token<BlockTokenIdentifier> blockToken) throws IOException {
|
||||||
OpBlockChecksumProto proto = OpBlockChecksumProto.newBuilder()
|
OpBlockChecksumProto proto = OpBlockChecksumProto.newBuilder()
|
||||||
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
.setHeader(DataTransferProtoUtil.buildBaseHeader(blk, blockToken))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
send(out, Op.BLOCK_CHECKSUM, proto);
|
send(out, Op.BLOCK_CHECKSUM, proto);
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,14 +39,13 @@ public class TrustedChannelResolver implements Configurable {
|
||||||
/**
|
/**
|
||||||
* Returns an instance of TrustedChannelResolver.
|
* Returns an instance of TrustedChannelResolver.
|
||||||
* Looks up the configuration to see if there is custom class specified.
|
* Looks up the configuration to see if there is custom class specified.
|
||||||
* @param conf
|
|
||||||
* @return TrustedChannelResolver
|
* @return TrustedChannelResolver
|
||||||
*/
|
*/
|
||||||
public static TrustedChannelResolver getInstance(Configuration conf) {
|
public static TrustedChannelResolver getInstance(Configuration conf) {
|
||||||
Class<? extends TrustedChannelResolver> clazz =
|
Class<? extends TrustedChannelResolver> clazz =
|
||||||
conf.getClass(
|
conf.getClass(
|
||||||
HdfsClientConfigKeys.DFS_TRUSTEDCHANNEL_RESOLVER_CLASS,
|
HdfsClientConfigKeys.DFS_TRUSTEDCHANNEL_RESOLVER_CLASS,
|
||||||
TrustedChannelResolver.class, TrustedChannelResolver.class);
|
TrustedChannelResolver.class, TrustedChannelResolver.class);
|
||||||
return ReflectionUtils.newInstance(clazz, conf);
|
return ReflectionUtils.newInstance(clazz, conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,9 +6,9 @@
|
||||||
* to you under the Apache License, Version 2.0 (the
|
* to you under the Apache License, Version 2.0 (the
|
||||||
* "License"); you may not use this file except in compliance
|
* "License"); you may not use this file except in compliance
|
||||||
* with the License. You may obtain a copy of the License at
|
* with the License. You may obtain a copy of the License at
|
||||||
*
|
* <p/>
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*
|
* <p/>
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
@ -69,7 +69,7 @@ import com.google.protobuf.ByteString;
|
||||||
public final class DataTransferSaslUtil {
|
public final class DataTransferSaslUtil {
|
||||||
|
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
DataTransferSaslUtil.class);
|
DataTransferSaslUtil.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delimiter for the three-part SASL username string.
|
* Delimiter for the three-part SASL username string.
|
||||||
|
@ -97,14 +97,14 @@ public final class DataTransferSaslUtil {
|
||||||
throw new IOException("Failed to complete SASL handshake");
|
throw new IOException("Failed to complete SASL handshake");
|
||||||
}
|
}
|
||||||
Set<String> requestedQop = ImmutableSet.copyOf(Arrays.asList(
|
Set<String> requestedQop = ImmutableSet.copyOf(Arrays.asList(
|
||||||
saslProps.get(Sasl.QOP).split(",")));
|
saslProps.get(Sasl.QOP).split(",")));
|
||||||
String negotiatedQop = sasl.getNegotiatedQop();
|
String negotiatedQop = sasl.getNegotiatedQop();
|
||||||
LOG.debug("Verifying QOP, requested QOP = {}, negotiated QOP = {}",
|
LOG.debug("Verifying QOP, requested QOP = {}, negotiated QOP = {}",
|
||||||
requestedQop, negotiatedQop);
|
requestedQop, negotiatedQop);
|
||||||
if (!requestedQop.contains(negotiatedQop)) {
|
if (!requestedQop.contains(negotiatedQop)) {
|
||||||
throw new IOException(String.format("SASL handshake completed, but " +
|
throw new IOException(String.format("SASL handshake completed, but " +
|
||||||
"channel does not have acceptable quality of protection, " +
|
"channel does not have acceptable quality of protection, " +
|
||||||
"requested = %s, negotiated = %s", requestedQop, negotiatedQop));
|
"requested = %s, negotiated = %s", requestedQop, negotiatedQop));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -145,7 +145,7 @@ public final class DataTransferSaslUtil {
|
||||||
*/
|
*/
|
||||||
public static char[] encryptionKeyToPassword(byte[] encryptionKey) {
|
public static char[] encryptionKeyToPassword(byte[] encryptionKey) {
|
||||||
return new String(Base64.encodeBase64(encryptionKey, false), Charsets.UTF_8)
|
return new String(Base64.encodeBase64(encryptionKey, false), Charsets.UTF_8)
|
||||||
.toCharArray();
|
.toCharArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -153,7 +153,6 @@ public final class DataTransferSaslUtil {
|
||||||
* [host][/ip-address]:port. The host may be missing. The IP address (and
|
* [host][/ip-address]:port. The host may be missing. The IP address (and
|
||||||
* preceding '/') may be missing. The port preceded by ':' is always present.
|
* preceding '/') may be missing. The port preceded by ':' is always present.
|
||||||
*
|
*
|
||||||
* @param peer
|
|
||||||
* @return InetAddress from peer
|
* @return InetAddress from peer
|
||||||
*/
|
*/
|
||||||
public static InetAddress getPeerAddress(Peer peer) {
|
public static InetAddress getPeerAddress(Peer peer) {
|
||||||
|
@ -181,23 +180,26 @@ public final class DataTransferSaslUtil {
|
||||||
String qops = conf.get(DFS_DATA_TRANSFER_PROTECTION_KEY);
|
String qops = conf.get(DFS_DATA_TRANSFER_PROTECTION_KEY);
|
||||||
if (qops == null || qops.isEmpty()) {
|
if (qops == null || qops.isEmpty()) {
|
||||||
LOG.debug("DataTransferProtocol not using SaslPropertiesResolver, no " +
|
LOG.debug("DataTransferProtocol not using SaslPropertiesResolver, no " +
|
||||||
"QOP found in configuration for {}", DFS_DATA_TRANSFER_PROTECTION_KEY);
|
"QOP found in configuration for {}",
|
||||||
|
DFS_DATA_TRANSFER_PROTECTION_KEY);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
Configuration saslPropsResolverConf = new Configuration(conf);
|
Configuration saslPropsResolverConf = new Configuration(conf);
|
||||||
saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops);
|
saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops);
|
||||||
Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
|
Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass(
|
||||||
HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
||||||
SaslPropertiesResolver.class, SaslPropertiesResolver.class);
|
SaslPropertiesResolver.class, SaslPropertiesResolver.class);
|
||||||
resolverClass = conf.getClass(DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY,
|
resolverClass =
|
||||||
resolverClass, SaslPropertiesResolver.class);
|
conf.getClass(DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY,
|
||||||
|
resolverClass, SaslPropertiesResolver.class);
|
||||||
saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS,
|
||||||
resolverClass, SaslPropertiesResolver.class);
|
resolverClass, SaslPropertiesResolver.class);
|
||||||
SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance(
|
SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance(
|
||||||
saslPropsResolverConf);
|
saslPropsResolverConf);
|
||||||
LOG.debug("DataTransferProtocol using SaslPropertiesResolver, configured " +
|
LOG.debug("DataTransferProtocol using SaslPropertiesResolver, configured " +
|
||||||
"QOP {} = {}, configured class {} = {}", DFS_DATA_TRANSFER_PROTECTION_KEY, qops,
|
"QOP {} = {}, configured class {} = {}",
|
||||||
DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, resolverClass);
|
DFS_DATA_TRANSFER_PROTECTION_KEY, qops,
|
||||||
|
DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, resolverClass);
|
||||||
return resolver;
|
return resolver;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -279,6 +281,7 @@ public final class DataTransferSaslUtil {
|
||||||
byte[] inIv = new byte[suite.getAlgorithmBlockSize()];
|
byte[] inIv = new byte[suite.getAlgorithmBlockSize()];
|
||||||
byte[] outKey = new byte[keyLen];
|
byte[] outKey = new byte[keyLen];
|
||||||
byte[] outIv = new byte[suite.getAlgorithmBlockSize()];
|
byte[] outIv = new byte[suite.getAlgorithmBlockSize()];
|
||||||
|
assert codec != null;
|
||||||
codec.generateSecureRandom(inKey);
|
codec.generateSecureRandom(inKey);
|
||||||
codec.generateSecureRandom(inIv);
|
codec.generateSecureRandom(inIv);
|
||||||
codec.generateSecureRandom(outKey);
|
codec.generateSecureRandom(outKey);
|
||||||
|
@ -300,7 +303,7 @@ public final class DataTransferSaslUtil {
|
||||||
*/
|
*/
|
||||||
public static void sendSaslMessageAndNegotiatedCipherOption(
|
public static void sendSaslMessageAndNegotiatedCipherOption(
|
||||||
OutputStream out, byte[] payload, CipherOption option)
|
OutputStream out, byte[] payload, CipherOption option)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
DataTransferEncryptorMessageProto.Builder builder =
|
DataTransferEncryptorMessageProto.Builder builder =
|
||||||
DataTransferEncryptorMessageProto.newBuilder();
|
DataTransferEncryptorMessageProto.newBuilder();
|
||||||
|
|
||||||
|
@ -381,7 +384,7 @@ public final class DataTransferSaslUtil {
|
||||||
*/
|
*/
|
||||||
public static void sendSaslMessageAndNegotiationCipherOptions(
|
public static void sendSaslMessageAndNegotiationCipherOptions(
|
||||||
OutputStream out, byte[] payload, List<CipherOption> options)
|
OutputStream out, byte[] payload, List<CipherOption> options)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
DataTransferEncryptorMessageProto.Builder builder =
|
DataTransferEncryptorMessageProto.Builder builder =
|
||||||
DataTransferEncryptorMessageProto.newBuilder();
|
DataTransferEncryptorMessageProto.newBuilder();
|
||||||
|
|
||||||
|
@ -408,7 +411,7 @@ public final class DataTransferSaslUtil {
|
||||||
*/
|
*/
|
||||||
public static SaslResponseWithNegotiatedCipherOption
|
public static SaslResponseWithNegotiatedCipherOption
|
||||||
readSaslMessageAndNegotiatedCipherOption(InputStream in)
|
readSaslMessageAndNegotiatedCipherOption(InputStream in)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
DataTransferEncryptorMessageProto proto =
|
DataTransferEncryptorMessageProto proto =
|
||||||
DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
|
DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
|
||||||
if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
|
if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
|
||||||
|
@ -492,7 +495,7 @@ public final class DataTransferSaslUtil {
|
||||||
*/
|
*/
|
||||||
public static void sendSaslMessage(OutputStream out,
|
public static void sendSaslMessage(OutputStream out,
|
||||||
DataTransferEncryptorStatus status, byte[] payload, String message)
|
DataTransferEncryptorStatus status, byte[] payload, String message)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
DataTransferEncryptorMessageProto.Builder builder =
|
DataTransferEncryptorMessageProto.Builder builder =
|
||||||
DataTransferEncryptorMessageProto.newBuilder();
|
DataTransferEncryptorMessageProto.newBuilder();
|
||||||
|
|
||||||
|
|
|
@ -76,7 +76,7 @@ import com.google.common.collect.Lists;
|
||||||
public class SaslDataTransferClient {
|
public class SaslDataTransferClient {
|
||||||
|
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
SaslDataTransferClient.class);
|
SaslDataTransferClient.class);
|
||||||
|
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private final AtomicBoolean fallbackToSimpleAuth;
|
private final AtomicBoolean fallbackToSimpleAuth;
|
||||||
|
@ -138,9 +138,9 @@ public class SaslDataTransferClient {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// The encryption key factory only returns a key if encryption is enabled.
|
// The encryption key factory only returns a key if encryption is enabled.
|
||||||
DataEncryptionKey encryptionKey = !trustedChannelResolver.isTrusted() ?
|
DataEncryptionKey encryptionKey = !trustedChannelResolver.isTrusted() ?
|
||||||
encryptionKeyFactory.newDataEncryptionKey() : null;
|
encryptionKeyFactory.newDataEncryptionKey() : null;
|
||||||
IOStreamPair ios = send(socket.getInetAddress(), underlyingOut,
|
IOStreamPair ios = send(socket.getInetAddress(), underlyingOut,
|
||||||
underlyingIn, encryptionKey, accessToken, datanodeId);
|
underlyingIn, encryptionKey, accessToken, datanodeId);
|
||||||
return ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut);
|
return ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -158,8 +158,8 @@ public class SaslDataTransferClient {
|
||||||
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
IOStreamPair ios = checkTrustAndSend(getPeerAddress(peer),
|
IOStreamPair ios = checkTrustAndSend(getPeerAddress(peer),
|
||||||
peer.getOutputStream(), peer.getInputStream(), encryptionKeyFactory,
|
peer.getOutputStream(), peer.getInputStream(), encryptionKeyFactory,
|
||||||
accessToken, datanodeId);
|
accessToken, datanodeId);
|
||||||
// TODO: Consider renaming EncryptedPeer to SaslPeer.
|
// TODO: Consider renaming EncryptedPeer to SaslPeer.
|
||||||
return ios != null ? new EncryptedPeer(peer, ios) : peer;
|
return ios != null ? new EncryptedPeer(peer, ios) : peer;
|
||||||
}
|
}
|
||||||
|
@ -181,7 +181,7 @@ public class SaslDataTransferClient {
|
||||||
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
IOStreamPair ios = checkTrustAndSend(socket.getInetAddress(), underlyingOut,
|
IOStreamPair ios = checkTrustAndSend(socket.getInetAddress(), underlyingOut,
|
||||||
underlyingIn, encryptionKeyFactory, accessToken, datanodeId);
|
underlyingIn, encryptionKeyFactory, accessToken, datanodeId);
|
||||||
return ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut);
|
return ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -207,13 +207,13 @@ public class SaslDataTransferClient {
|
||||||
!trustedChannelResolver.isTrusted(addr)) {
|
!trustedChannelResolver.isTrusted(addr)) {
|
||||||
// The encryption key factory only returns a key if encryption is enabled.
|
// The encryption key factory only returns a key if encryption is enabled.
|
||||||
DataEncryptionKey encryptionKey =
|
DataEncryptionKey encryptionKey =
|
||||||
encryptionKeyFactory.newDataEncryptionKey();
|
encryptionKeyFactory.newDataEncryptionKey();
|
||||||
return send(addr, underlyingOut, underlyingIn, encryptionKey, accessToken,
|
return send(addr, underlyingOut, underlyingIn, encryptionKey, accessToken,
|
||||||
datanodeId);
|
datanodeId);
|
||||||
} else {
|
} else {
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
"SASL client skipping handshake on trusted connection for addr = {}, "
|
"SASL client skipping handshake on trusted connection for addr = {}, "
|
||||||
+ "datanodeId = {}", addr, datanodeId);
|
+ "datanodeId = {}", addr, datanodeId);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -236,40 +236,38 @@ public class SaslDataTransferClient {
|
||||||
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (encryptionKey != null) {
|
if (encryptionKey != null) {
|
||||||
LOG.debug(
|
LOG.debug("SASL client doing encrypted handshake for addr = {}, "
|
||||||
"SASL client doing encrypted handshake for addr = {}, datanodeId = {}",
|
+ "datanodeId = {}", addr, datanodeId);
|
||||||
addr, datanodeId);
|
|
||||||
return getEncryptedStreams(underlyingOut, underlyingIn,
|
return getEncryptedStreams(underlyingOut, underlyingIn,
|
||||||
encryptionKey);
|
encryptionKey);
|
||||||
} else if (!UserGroupInformation.isSecurityEnabled()) {
|
} else if (!UserGroupInformation.isSecurityEnabled()) {
|
||||||
LOG.debug(
|
LOG.debug("SASL client skipping handshake in unsecured configuration for "
|
||||||
"SASL client skipping handshake in unsecured configuration for "
|
+ "addr = {}, datanodeId = {}", addr, datanodeId);
|
||||||
+ "addr = {}, datanodeId = {}", addr, datanodeId);
|
|
||||||
return null;
|
return null;
|
||||||
} else if (SecurityUtil.isPrivilegedPort(datanodeId.getXferPort())) {
|
} else if (SecurityUtil.isPrivilegedPort(datanodeId.getXferPort())) {
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
"SASL client skipping handshake in secured configuration with "
|
"SASL client skipping handshake in secured configuration with "
|
||||||
+ "privileged port for addr = {}, datanodeId = {}", addr, datanodeId);
|
+ "privileged port for addr = {}, datanodeId = {}",
|
||||||
|
addr, datanodeId);
|
||||||
return null;
|
return null;
|
||||||
} else if (fallbackToSimpleAuth != null && fallbackToSimpleAuth.get()) {
|
} else if (fallbackToSimpleAuth != null && fallbackToSimpleAuth.get()) {
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
"SASL client skipping handshake in secured configuration with "
|
"SASL client skipping handshake in secured configuration with "
|
||||||
+ "unsecured cluster for addr = {}, datanodeId = {}", addr, datanodeId);
|
+ "unsecured cluster for addr = {}, datanodeId = {}",
|
||||||
|
addr, datanodeId);
|
||||||
return null;
|
return null;
|
||||||
} else if (saslPropsResolver != null) {
|
} else if (saslPropsResolver != null) {
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
"SASL client doing general handshake for addr = {}, datanodeId = {}",
|
"SASL client doing general handshake for addr = {}, datanodeId = {}",
|
||||||
addr, datanodeId);
|
addr, datanodeId);
|
||||||
return getSaslStreams(addr, underlyingOut, underlyingIn, accessToken,
|
return getSaslStreams(addr, underlyingOut, underlyingIn, accessToken);
|
||||||
datanodeId);
|
|
||||||
} else {
|
} else {
|
||||||
// It's a secured cluster using non-privileged ports, but no SASL. The
|
// It's a secured cluster using non-privileged ports, but no SASL. The
|
||||||
// only way this can happen is if the DataNode has
|
// only way this can happen is if the DataNode has
|
||||||
// ignore.secure.ports.for.testing configured, so this is a rare edge case.
|
// ignore.secure.ports.for.testing configured so this is a rare edge case.
|
||||||
LOG.debug(
|
LOG.debug("SASL client skipping handshake in secured configuration with "
|
||||||
"SASL client skipping handshake in secured configuration with no SASL "
|
+ "no SASL protection configured for addr = {}, datanodeId = {}",
|
||||||
+ "protection configured for addr = {}, datanodeId = {}",
|
addr, datanodeId);
|
||||||
addr, datanodeId);
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -287,17 +285,17 @@ public class SaslDataTransferClient {
|
||||||
InputStream underlyingIn, DataEncryptionKey encryptionKey)
|
InputStream underlyingIn, DataEncryptionKey encryptionKey)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Map<String, String> saslProps = createSaslPropertiesForEncryption(
|
Map<String, String> saslProps = createSaslPropertiesForEncryption(
|
||||||
encryptionKey.encryptionAlgorithm);
|
encryptionKey.encryptionAlgorithm);
|
||||||
|
|
||||||
LOG.debug("Client using encryption algorithm {}",
|
LOG.debug("Client using encryption algorithm {}",
|
||||||
encryptionKey.encryptionAlgorithm);
|
encryptionKey.encryptionAlgorithm);
|
||||||
|
|
||||||
String userName = getUserNameFromEncryptionKey(encryptionKey);
|
String userName = getUserNameFromEncryptionKey(encryptionKey);
|
||||||
char[] password = encryptionKeyToPassword(encryptionKey.encryptionKey);
|
char[] password = encryptionKeyToPassword(encryptionKey.encryptionKey);
|
||||||
CallbackHandler callbackHandler = new SaslClientCallbackHandler(userName,
|
CallbackHandler callbackHandler = new SaslClientCallbackHandler(userName,
|
||||||
password);
|
password);
|
||||||
return doSaslHandshake(underlyingOut, underlyingIn, userName, saslProps,
|
return doSaslHandshake(underlyingOut, underlyingIn, userName, saslProps,
|
||||||
callbackHandler);
|
callbackHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -312,7 +310,8 @@ public class SaslDataTransferClient {
|
||||||
DataEncryptionKey encryptionKey) {
|
DataEncryptionKey encryptionKey) {
|
||||||
return encryptionKey.keyId + NAME_DELIMITER +
|
return encryptionKey.keyId + NAME_DELIMITER +
|
||||||
encryptionKey.blockPoolId + NAME_DELIMITER +
|
encryptionKey.blockPoolId + NAME_DELIMITER +
|
||||||
new String(Base64.encodeBase64(encryptionKey.nonce, false), Charsets.UTF_8);
|
new String(Base64.encodeBase64(encryptionKey.nonce, false),
|
||||||
|
Charsets.UTF_8);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -328,7 +327,7 @@ public class SaslDataTransferClient {
|
||||||
* Creates a new SaslClientCallbackHandler.
|
* Creates a new SaslClientCallbackHandler.
|
||||||
*
|
*
|
||||||
* @param userName SASL user name
|
* @param userName SASL user name
|
||||||
* @Param password SASL password
|
* @param password SASL password
|
||||||
*/
|
*/
|
||||||
public SaslClientCallbackHandler(String userName, char[] password) {
|
public SaslClientCallbackHandler(String userName, char[] password) {
|
||||||
this.password = password;
|
this.password = password;
|
||||||
|
@ -342,15 +341,13 @@ public class SaslDataTransferClient {
|
||||||
PasswordCallback pc = null;
|
PasswordCallback pc = null;
|
||||||
RealmCallback rc = null;
|
RealmCallback rc = null;
|
||||||
for (Callback callback : callbacks) {
|
for (Callback callback : callbacks) {
|
||||||
if (callback instanceof RealmChoiceCallback) {
|
if (callback instanceof NameCallback) {
|
||||||
continue;
|
|
||||||
} else if (callback instanceof NameCallback) {
|
|
||||||
nc = (NameCallback) callback;
|
nc = (NameCallback) callback;
|
||||||
} else if (callback instanceof PasswordCallback) {
|
} else if (callback instanceof PasswordCallback) {
|
||||||
pc = (PasswordCallback) callback;
|
pc = (PasswordCallback) callback;
|
||||||
} else if (callback instanceof RealmCallback) {
|
} else if (callback instanceof RealmCallback) {
|
||||||
rc = (RealmCallback) callback;
|
rc = (RealmCallback) callback;
|
||||||
} else {
|
} else if (!(callback instanceof RealmChoiceCallback)) {
|
||||||
throw new UnsupportedCallbackException(callback,
|
throw new UnsupportedCallbackException(callback,
|
||||||
"Unrecognized SASL client callback");
|
"Unrecognized SASL client callback");
|
||||||
}
|
}
|
||||||
|
@ -374,22 +371,21 @@ public class SaslDataTransferClient {
|
||||||
* @param underlyingOut connection output stream
|
* @param underlyingOut connection output stream
|
||||||
* @param underlyingIn connection input stream
|
* @param underlyingIn connection input stream
|
||||||
* @param accessToken connection block access token
|
* @param accessToken connection block access token
|
||||||
* @param datanodeId ID of destination DataNode
|
|
||||||
* @return new pair of streams, wrapped after SASL negotiation
|
* @return new pair of streams, wrapped after SASL negotiation
|
||||||
* @throws IOException for any error
|
* @throws IOException for any error
|
||||||
*/
|
*/
|
||||||
private IOStreamPair getSaslStreams(InetAddress addr,
|
private IOStreamPair getSaslStreams(InetAddress addr,
|
||||||
OutputStream underlyingOut, InputStream underlyingIn,
|
OutputStream underlyingOut, InputStream underlyingIn,
|
||||||
Token<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
|
Token<BlockTokenIdentifier> accessToken)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Map<String, String> saslProps = saslPropsResolver.getClientProperties(addr);
|
Map<String, String> saslProps = saslPropsResolver.getClientProperties(addr);
|
||||||
|
|
||||||
String userName = buildUserName(accessToken);
|
String userName = buildUserName(accessToken);
|
||||||
char[] password = buildClientPassword(accessToken);
|
char[] password = buildClientPassword(accessToken);
|
||||||
CallbackHandler callbackHandler = new SaslClientCallbackHandler(userName,
|
CallbackHandler callbackHandler = new SaslClientCallbackHandler(userName,
|
||||||
password);
|
password);
|
||||||
return doSaslHandshake(underlyingOut, underlyingIn, userName, saslProps,
|
return doSaslHandshake(underlyingOut, underlyingIn, userName, saslProps,
|
||||||
callbackHandler);
|
callbackHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -404,7 +400,7 @@ public class SaslDataTransferClient {
|
||||||
*/
|
*/
|
||||||
private static String buildUserName(Token<BlockTokenIdentifier> blockToken) {
|
private static String buildUserName(Token<BlockTokenIdentifier> blockToken) {
|
||||||
return new String(Base64.encodeBase64(blockToken.getIdentifier(), false),
|
return new String(Base64.encodeBase64(blockToken.getIdentifier(), false),
|
||||||
Charsets.UTF_8);
|
Charsets.UTF_8);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -416,7 +412,7 @@ public class SaslDataTransferClient {
|
||||||
*/
|
*/
|
||||||
private char[] buildClientPassword(Token<BlockTokenIdentifier> blockToken) {
|
private char[] buildClientPassword(Token<BlockTokenIdentifier> blockToken) {
|
||||||
return new String(Base64.encodeBase64(blockToken.getPassword(), false),
|
return new String(Base64.encodeBase64(blockToken.getPassword(), false),
|
||||||
Charsets.UTF_8).toCharArray();
|
Charsets.UTF_8).toCharArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -438,7 +434,7 @@ public class SaslDataTransferClient {
|
||||||
DataInputStream in = new DataInputStream(underlyingIn);
|
DataInputStream in = new DataInputStream(underlyingIn);
|
||||||
|
|
||||||
SaslParticipant sasl= SaslParticipant.createClientSaslParticipant(userName,
|
SaslParticipant sasl= SaslParticipant.createClientSaslParticipant(userName,
|
||||||
saslProps, callbackHandler);
|
saslProps, callbackHandler);
|
||||||
|
|
||||||
out.writeInt(SASL_TRANSFER_MAGIC_NUMBER);
|
out.writeInt(SASL_TRANSFER_MAGIC_NUMBER);
|
||||||
out.flush();
|
out.flush();
|
||||||
|
@ -489,7 +485,7 @@ public class SaslDataTransferClient {
|
||||||
// stream pair.
|
// stream pair.
|
||||||
return cipherOption != null ? createStreamPair(
|
return cipherOption != null ? createStreamPair(
|
||||||
conf, cipherOption, underlyingOut, underlyingIn, false) :
|
conf, cipherOption, underlyingOut, underlyingIn, false) :
|
||||||
sasl.createStreamPair(out, in);
|
sasl.createStreamPair(out, in);
|
||||||
} catch (IOException ioe) {
|
} catch (IOException ioe) {
|
||||||
sendGenericSaslErrorMessage(out, ioe.getMessage());
|
sendGenericSaslErrorMessage(out, ioe.getMessage());
|
||||||
throw ioe;
|
throw ioe;
|
||||||
|
|
|
@ -115,7 +115,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
Configuration conf, int socketTimeout, boolean connectToDnViaHostname,
|
Configuration conf, int socketTimeout, boolean connectToDnViaHostname,
|
||||||
LocatedBlock locatedBlock) throws IOException {
|
LocatedBlock locatedBlock) throws IOException {
|
||||||
rpcProxy = createClientDatanodeProtocolProxy( datanodeid, conf,
|
rpcProxy = createClientDatanodeProtocolProxy( datanodeid, conf,
|
||||||
socketTimeout, connectToDnViaHostname, locatedBlock);
|
socketTimeout, connectToDnViaHostname, locatedBlock);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ClientDatanodeProtocolTranslatorPB(InetSocketAddress addr,
|
public ClientDatanodeProtocolTranslatorPB(InetSocketAddress addr,
|
||||||
|
@ -145,7 +145,8 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
|
static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
|
||||||
DatanodeID datanodeid, Configuration conf, int socketTimeout,
|
DatanodeID datanodeid, Configuration conf, int socketTimeout,
|
||||||
boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
|
boolean connectToDnViaHostname, LocatedBlock locatedBlock)
|
||||||
|
throws IOException {
|
||||||
final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
|
final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
|
||||||
InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
|
InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
|
||||||
LOG.debug("Connecting to datanode {} addr={}", dnAddr, addr);
|
LOG.debug("Connecting to datanode {} addr={}", dnAddr, addr);
|
||||||
|
@ -185,8 +186,9 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getReplicaVisibleLength(ExtendedBlock b) throws IOException {
|
public long getReplicaVisibleLength(ExtendedBlock b) throws IOException {
|
||||||
GetReplicaVisibleLengthRequestProto req = GetReplicaVisibleLengthRequestProto
|
GetReplicaVisibleLengthRequestProto req =
|
||||||
.newBuilder().setBlock(PBHelperClient.convert(b)).build();
|
GetReplicaVisibleLengthRequestProto.newBuilder()
|
||||||
|
.setBlock(PBHelperClient.convert(b)).build();
|
||||||
try {
|
try {
|
||||||
return rpcProxy.getReplicaVisibleLength(NULL_CONTROLLER, req).getLength();
|
return rpcProxy.getReplicaVisibleLength(NULL_CONTROLLER, req).getLength();
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -219,8 +221,8 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
Token<BlockTokenIdentifier> token) throws IOException {
|
Token<BlockTokenIdentifier> token) throws IOException {
|
||||||
GetBlockLocalPathInfoRequestProto req =
|
GetBlockLocalPathInfoRequestProto req =
|
||||||
GetBlockLocalPathInfoRequestProto.newBuilder()
|
GetBlockLocalPathInfoRequestProto.newBuilder()
|
||||||
.setBlock(PBHelperClient.convert(block))
|
.setBlock(PBHelperClient.convert(block))
|
||||||
.setToken(PBHelperClient.convert(token)).build();
|
.setToken(PBHelperClient.convert(token)).build();
|
||||||
GetBlockLocalPathInfoResponseProto resp;
|
GetBlockLocalPathInfoResponseProto resp;
|
||||||
try {
|
try {
|
||||||
resp = rpcProxy.getBlockLocalPathInfo(NULL_CONTROLLER, req);
|
resp = rpcProxy.getBlockLocalPathInfo(NULL_CONTROLLER, req);
|
||||||
|
@ -294,7 +296,8 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
public DatanodeLocalInfo getDatanodeInfo() throws IOException {
|
public DatanodeLocalInfo getDatanodeInfo() throws IOException {
|
||||||
GetDatanodeInfoResponseProto response;
|
GetDatanodeInfoResponseProto response;
|
||||||
try {
|
try {
|
||||||
response = rpcProxy.getDatanodeInfo(NULL_CONTROLLER, VOID_GET_DATANODE_INFO);
|
response = rpcProxy.getDatanodeInfo(NULL_CONTROLLER,
|
||||||
|
VOID_GET_DATANODE_INFO);
|
||||||
return PBHelperClient.convert(response.getLocalInfo());
|
return PBHelperClient.convert(response.getLocalInfo());
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
@ -311,7 +314,8 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ReconfigurationTaskStatus getReconfigurationStatus() throws IOException {
|
public ReconfigurationTaskStatus getReconfigurationStatus()
|
||||||
|
throws IOException {
|
||||||
GetReconfigurationStatusResponseProto response;
|
GetReconfigurationStatusResponseProto response;
|
||||||
Map<PropertyChange, Optional<String>> statusMap = null;
|
Map<PropertyChange, Optional<String>> statusMap = null;
|
||||||
long startTime;
|
long startTime;
|
||||||
|
@ -361,8 +365,8 @@ public class ClientDatanodeProtocolTranslatorPB implements
|
||||||
try {
|
try {
|
||||||
rpcProxy.triggerBlockReport(NULL_CONTROLLER,
|
rpcProxy.triggerBlockReport(NULL_CONTROLLER,
|
||||||
TriggerBlockReportRequestProto.newBuilder().
|
TriggerBlockReportRequestProto.newBuilder().
|
||||||
setIncremental(options.isIncremental()).
|
setIncremental(options.isIncremental()).
|
||||||
build());
|
build());
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,5 +42,5 @@ import org.apache.hadoop.security.token.TokenInfo;
|
||||||
* add annotations required for security.
|
* add annotations required for security.
|
||||||
*/
|
*/
|
||||||
public interface ClientNamenodeProtocolPB extends
|
public interface ClientNamenodeProtocolPB extends
|
||||||
ClientNamenodeProtocol.BlockingInterface {
|
ClientNamenodeProtocol.BlockingInterface {
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.hdfs.protocolPB;
|
package org.apache.hadoop.hdfs.protocolPB;
|
||||||
|
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
@ -33,12 +32,9 @@ import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
|
||||||
import org.apache.hadoop.fs.CacheFlag;
|
import org.apache.hadoop.fs.CacheFlag;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
import org.apache.hadoop.fs.CreateFlag;
|
import org.apache.hadoop.fs.CreateFlag;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
|
||||||
import org.apache.hadoop.fs.FsServerDefaults;
|
import org.apache.hadoop.fs.FsServerDefaults;
|
||||||
import org.apache.hadoop.fs.Options.Rename;
|
import org.apache.hadoop.fs.Options.Rename;
|
||||||
import org.apache.hadoop.fs.ParentNotDirectoryException;
|
|
||||||
import org.apache.hadoop.fs.StorageType;
|
import org.apache.hadoop.fs.StorageType;
|
||||||
import org.apache.hadoop.fs.UnresolvedLinkException;
|
|
||||||
import org.apache.hadoop.fs.XAttr;
|
import org.apache.hadoop.fs.XAttr;
|
||||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||||
import org.apache.hadoop.fs.permission.AclEntry;
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
|
@ -46,7 +42,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.inotify.EventBatchList;
|
import org.apache.hadoop.hdfs.inotify.EventBatchList;
|
||||||
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
|
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
|
||||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
||||||
|
@ -54,7 +49,6 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||||
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
|
@ -67,7 +61,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
|
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
|
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||||
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
|
||||||
|
@ -171,8 +164,6 @@ import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto
|
||||||
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto;
|
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto;
|
||||||
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
|
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
|
||||||
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
|
|
||||||
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
|
|
||||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
|
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
|
||||||
import org.apache.hadoop.io.EnumSetWritable;
|
import org.apache.hadoop.io.EnumSetWritable;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
|
@ -181,7 +172,6 @@ import org.apache.hadoop.ipc.ProtocolMetaInterface;
|
||||||
import org.apache.hadoop.ipc.ProtocolTranslator;
|
import org.apache.hadoop.ipc.ProtocolTranslator;
|
||||||
import org.apache.hadoop.ipc.RPC;
|
import org.apache.hadoop.ipc.RPC;
|
||||||
import org.apache.hadoop.ipc.RpcClientUtil;
|
import org.apache.hadoop.ipc.RpcClientUtil;
|
||||||
import org.apache.hadoop.security.AccessControlException;
|
|
||||||
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
|
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
|
||||||
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
|
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
|
||||||
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
|
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
|
||||||
|
@ -207,10 +197,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
final private ClientNamenodeProtocolPB rpcProxy;
|
final private ClientNamenodeProtocolPB rpcProxy;
|
||||||
|
|
||||||
static final GetServerDefaultsRequestProto VOID_GET_SERVER_DEFAULT_REQUEST =
|
static final GetServerDefaultsRequestProto VOID_GET_SERVER_DEFAULT_REQUEST =
|
||||||
GetServerDefaultsRequestProto.newBuilder().build();
|
GetServerDefaultsRequestProto.newBuilder().build();
|
||||||
|
|
||||||
private final static GetFsStatusRequestProto VOID_GET_FSSTATUS_REQUEST =
|
private final static GetFsStatusRequestProto VOID_GET_FSSTATUS_REQUEST =
|
||||||
GetFsStatusRequestProto.newBuilder().build();
|
GetFsStatusRequestProto.newBuilder().build();
|
||||||
|
|
||||||
private final static SaveNamespaceRequestProto VOID_SAVE_NAMESPACE_REQUEST =
|
private final static SaveNamespaceRequestProto VOID_SAVE_NAMESPACE_REQUEST =
|
||||||
SaveNamespaceRequestProto.newBuilder().build();
|
SaveNamespaceRequestProto.newBuilder().build();
|
||||||
|
@ -219,18 +209,18 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
RollEditsRequestProto.getDefaultInstance();
|
RollEditsRequestProto.getDefaultInstance();
|
||||||
|
|
||||||
private final static RefreshNodesRequestProto VOID_REFRESH_NODES_REQUEST =
|
private final static RefreshNodesRequestProto VOID_REFRESH_NODES_REQUEST =
|
||||||
RefreshNodesRequestProto.newBuilder().build();
|
RefreshNodesRequestProto.newBuilder().build();
|
||||||
|
|
||||||
private final static FinalizeUpgradeRequestProto
|
private final static FinalizeUpgradeRequestProto
|
||||||
VOID_FINALIZE_UPGRADE_REQUEST =
|
VOID_FINALIZE_UPGRADE_REQUEST =
|
||||||
FinalizeUpgradeRequestProto.newBuilder().build();
|
FinalizeUpgradeRequestProto.newBuilder().build();
|
||||||
|
|
||||||
private final static GetDataEncryptionKeyRequestProto
|
private final static GetDataEncryptionKeyRequestProto
|
||||||
VOID_GET_DATA_ENCRYPTIONKEY_REQUEST =
|
VOID_GET_DATA_ENCRYPTIONKEY_REQUEST =
|
||||||
GetDataEncryptionKeyRequestProto.newBuilder().build();
|
GetDataEncryptionKeyRequestProto.newBuilder().build();
|
||||||
|
|
||||||
private final static GetStoragePoliciesRequestProto
|
private final static GetStoragePoliciesRequestProto
|
||||||
VOID_GET_STORAGE_POLICIES_REQUEST =
|
VOID_GET_STORAGE_POLICIES_REQUEST =
|
||||||
GetStoragePoliciesRequestProto.newBuilder().build();
|
GetStoragePoliciesRequestProto.newBuilder().build();
|
||||||
|
|
||||||
public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) {
|
public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) {
|
||||||
|
@ -244,8 +234,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public LocatedBlocks getBlockLocations(String src, long offset, long length)
|
public LocatedBlocks getBlockLocations(String src, long offset, long length)
|
||||||
throws AccessControlException, FileNotFoundException,
|
throws IOException {
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
GetBlockLocationsRequestProto req = GetBlockLocationsRequestProto
|
GetBlockLocationsRequestProto req = GetBlockLocationsRequestProto
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
|
@ -256,7 +245,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
GetBlockLocationsResponseProto resp = rpcProxy.getBlockLocations(null,
|
GetBlockLocationsResponseProto resp = rpcProxy.getBlockLocations(null,
|
||||||
req);
|
req);
|
||||||
return resp.hasLocations() ?
|
return resp.hasLocations() ?
|
||||||
PBHelperClient.convert(resp.getLocations()) : null;
|
PBHelperClient.convert(resp.getLocations()) : null;
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
@ -278,11 +267,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
String clientName, EnumSetWritable<CreateFlag> flag,
|
String clientName, EnumSetWritable<CreateFlag> flag,
|
||||||
boolean createParent, short replication, long blockSize,
|
boolean createParent, short replication, long blockSize,
|
||||||
CryptoProtocolVersion[] supportedVersions)
|
CryptoProtocolVersion[] supportedVersions)
|
||||||
throws AccessControlException, AlreadyBeingCreatedException,
|
throws IOException {
|
||||||
DSQuotaExceededException, FileAlreadyExistsException,
|
|
||||||
FileNotFoundException, NSQuotaExceededException,
|
|
||||||
ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
CreateRequestProto.Builder builder = CreateRequestProto.newBuilder()
|
CreateRequestProto.Builder builder = CreateRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setMasked(PBHelperClient.convert(masked))
|
.setMasked(PBHelperClient.convert(masked))
|
||||||
|
@ -291,7 +276,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
.setCreateParent(createParent)
|
.setCreateParent(createParent)
|
||||||
.setReplication(replication)
|
.setReplication(replication)
|
||||||
.setBlockSize(blockSize);
|
.setBlockSize(blockSize);
|
||||||
builder.addAllCryptoProtocolVersion(PBHelperClient.convert(supportedVersions));
|
builder.addAllCryptoProtocolVersion(
|
||||||
|
PBHelperClient.convert(supportedVersions));
|
||||||
CreateRequestProto req = builder.build();
|
CreateRequestProto req = builder.build();
|
||||||
try {
|
try {
|
||||||
CreateResponseProto res = rpcProxy.create(null, req);
|
CreateResponseProto res = rpcProxy.create(null, req);
|
||||||
|
@ -304,7 +290,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean truncate(String src, long newLength, String clientName)
|
public boolean truncate(String src, long newLength, String clientName)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException {
|
||||||
TruncateRequestProto req = TruncateRequestProto.newBuilder()
|
TruncateRequestProto req = TruncateRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setNewLength(newLength)
|
.setNewLength(newLength)
|
||||||
|
@ -319,18 +305,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public LastBlockWithStatus append(String src, String clientName,
|
public LastBlockWithStatus append(String src, String clientName,
|
||||||
EnumSetWritable<CreateFlag> flag) throws AccessControlException,
|
EnumSetWritable<CreateFlag> flag) throws IOException {
|
||||||
DSQuotaExceededException, FileNotFoundException, SafeModeException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
AppendRequestProto req = AppendRequestProto.newBuilder().setSrc(src)
|
AppendRequestProto req = AppendRequestProto.newBuilder().setSrc(src)
|
||||||
.setClientName(clientName).setFlag(PBHelperClient.convertCreateFlag(flag))
|
.setClientName(clientName).setFlag(
|
||||||
|
PBHelperClient.convertCreateFlag(flag))
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
AppendResponseProto res = rpcProxy.append(null, req);
|
AppendResponseProto res = rpcProxy.append(null, req);
|
||||||
LocatedBlock lastBlock = res.hasBlock() ? PBHelperClient
|
LocatedBlock lastBlock = res.hasBlock() ? PBHelperClient
|
||||||
.convert(res.getBlock()) : null;
|
.convert(res.getBlock()) : null;
|
||||||
HdfsFileStatus stat = (res.hasStat()) ? PBHelperClient.convert(res.getStat())
|
HdfsFileStatus stat = (res.hasStat()) ?
|
||||||
: null;
|
PBHelperClient.convert(res.getStat()) : null;
|
||||||
return new LastBlockWithStatus(lastBlock, stat);
|
return new LastBlockWithStatus(lastBlock, stat);
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
@ -339,9 +324,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean setReplication(String src, short replication)
|
public boolean setReplication(String src, short replication)
|
||||||
throws AccessControlException, DSQuotaExceededException,
|
throws IOException {
|
||||||
FileNotFoundException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
SetReplicationRequestProto req = SetReplicationRequestProto.newBuilder()
|
SetReplicationRequestProto req = SetReplicationRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setReplication(replication)
|
.setReplication(replication)
|
||||||
|
@ -355,8 +338,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setPermission(String src, FsPermission permission)
|
public void setPermission(String src, FsPermission permission)
|
||||||
throws AccessControlException, FileNotFoundException, SafeModeException,
|
throws IOException {
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
SetPermissionRequestProto req = SetPermissionRequestProto.newBuilder()
|
SetPermissionRequestProto req = SetPermissionRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setPermission(PBHelperClient.convert(permission))
|
.setPermission(PBHelperClient.convert(permission))
|
||||||
|
@ -370,14 +352,13 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setOwner(String src, String username, String groupname)
|
public void setOwner(String src, String username, String groupname)
|
||||||
throws AccessControlException, FileNotFoundException, SafeModeException,
|
throws IOException {
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
SetOwnerRequestProto.Builder req = SetOwnerRequestProto.newBuilder()
|
SetOwnerRequestProto.Builder req = SetOwnerRequestProto.newBuilder()
|
||||||
.setSrc(src);
|
.setSrc(src);
|
||||||
if (username != null)
|
if (username != null)
|
||||||
req.setUsername(username);
|
req.setUsername(username);
|
||||||
if (groupname != null)
|
if (groupname != null)
|
||||||
req.setGroupname(groupname);
|
req.setGroupname(groupname);
|
||||||
try {
|
try {
|
||||||
rpcProxy.setOwner(null, req.build());
|
rpcProxy.setOwner(null, req.build());
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -387,11 +368,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void abandonBlock(ExtendedBlock b, long fileId, String src,
|
public void abandonBlock(ExtendedBlock b, long fileId, String src,
|
||||||
String holder) throws AccessControlException, FileNotFoundException,
|
String holder) throws IOException {
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
AbandonBlockRequestProto req = AbandonBlockRequestProto.newBuilder()
|
AbandonBlockRequestProto req = AbandonBlockRequestProto.newBuilder()
|
||||||
.setB(PBHelperClient.convert(b)).setSrc(src).setHolder(holder)
|
.setB(PBHelperClient.convert(b)).setSrc(src).setHolder(holder)
|
||||||
.setFileId(fileId).build();
|
.setFileId(fileId).build();
|
||||||
try {
|
try {
|
||||||
rpcProxy.abandonBlock(null, req);
|
rpcProxy.abandonBlock(null, req);
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -402,10 +382,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
@Override
|
@Override
|
||||||
public LocatedBlock addBlock(String src, String clientName,
|
public LocatedBlock addBlock(String src, String clientName,
|
||||||
ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
|
ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
|
||||||
String[] favoredNodes)
|
String[] favoredNodes) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
NotReplicatedYetException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
AddBlockRequestProto.Builder req = AddBlockRequestProto.newBuilder()
|
AddBlockRequestProto.Builder req = AddBlockRequestProto.newBuilder()
|
||||||
.setSrc(src).setClientName(clientName).setFileId(fileId);
|
.setSrc(src).setClientName(clientName).setFileId(fileId);
|
||||||
if (previous != null)
|
if (previous != null)
|
||||||
|
@ -425,10 +402,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
@Override
|
@Override
|
||||||
public LocatedBlock getAdditionalDatanode(String src, long fileId,
|
public LocatedBlock getAdditionalDatanode(String src, long fileId,
|
||||||
ExtendedBlock blk, DatanodeInfo[] existings, String[] existingStorageIDs,
|
ExtendedBlock blk, DatanodeInfo[] existings, String[] existingStorageIDs,
|
||||||
DatanodeInfo[] excludes,
|
DatanodeInfo[] excludes, int numAdditionalNodes, String clientName)
|
||||||
int numAdditionalNodes, String clientName) throws AccessControlException,
|
throws IOException {
|
||||||
FileNotFoundException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
GetAdditionalDatanodeRequestProto req = GetAdditionalDatanodeRequestProto
|
GetAdditionalDatanodeRequestProto req = GetAdditionalDatanodeRequestProto
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
|
@ -450,9 +425,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean complete(String src, String clientName,
|
public boolean complete(String src, String clientName,
|
||||||
ExtendedBlock last, long fileId)
|
ExtendedBlock last, long fileId) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException, SafeModeException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
CompleteRequestProto.Builder req = CompleteRequestProto.newBuilder()
|
CompleteRequestProto.Builder req = CompleteRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setClientName(clientName)
|
.setClientName(clientName)
|
||||||
|
@ -469,7 +442,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
@Override
|
@Override
|
||||||
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
|
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
|
||||||
ReportBadBlocksRequestProto req = ReportBadBlocksRequestProto.newBuilder()
|
ReportBadBlocksRequestProto req = ReportBadBlocksRequestProto.newBuilder()
|
||||||
.addAllBlocks(Arrays.asList(PBHelperClient.convertLocatedBlock(blocks)))
|
.addAllBlocks(Arrays.asList(
|
||||||
|
PBHelperClient.convertLocatedBlock(blocks)))
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
rpcProxy.reportBadBlocks(null, req);
|
rpcProxy.reportBadBlocks(null, req);
|
||||||
|
@ -479,8 +453,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean rename(String src, String dst) throws UnresolvedLinkException,
|
public boolean rename(String src, String dst) throws IOException {
|
||||||
IOException {
|
|
||||||
RenameRequestProto req = RenameRequestProto.newBuilder()
|
RenameRequestProto req = RenameRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setDst(dst).build();
|
.setDst(dst).build();
|
||||||
|
@ -494,10 +467,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rename2(String src, String dst, Rename... options)
|
public void rename2(String src, String dst, Rename... options)
|
||||||
throws AccessControlException, DSQuotaExceededException,
|
throws IOException {
|
||||||
FileAlreadyExistsException, FileNotFoundException,
|
|
||||||
NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
boolean overwrite = false;
|
boolean overwrite = false;
|
||||||
if (options != null) {
|
if (options != null) {
|
||||||
for (Rename option : options) {
|
for (Rename option : options) {
|
||||||
|
@ -519,8 +489,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void concat(String trg, String[] srcs) throws IOException,
|
public void concat(String trg, String[] srcs) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
ConcatRequestProto req = ConcatRequestProto.newBuilder().
|
ConcatRequestProto req = ConcatRequestProto.newBuilder().
|
||||||
setTrg(trg).
|
setTrg(trg).
|
||||||
addAllSrcs(Arrays.asList(srcs)).build();
|
addAllSrcs(Arrays.asList(srcs)).build();
|
||||||
|
@ -533,10 +502,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean delete(String src, boolean recursive)
|
public boolean delete(String src, boolean recursive) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException, SafeModeException,
|
DeleteRequestProto req = DeleteRequestProto.newBuilder().setSrc(src)
|
||||||
UnresolvedLinkException, IOException {
|
.setRecursive(recursive).build();
|
||||||
DeleteRequestProto req = DeleteRequestProto.newBuilder().setSrc(src).setRecursive(recursive).build();
|
|
||||||
try {
|
try {
|
||||||
return rpcProxy.delete(null, req).getResult();
|
return rpcProxy.delete(null, req).getResult();
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -546,10 +514,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
|
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
|
||||||
throws AccessControlException, FileAlreadyExistsException,
|
throws IOException {
|
||||||
FileNotFoundException, NSQuotaExceededException,
|
|
||||||
ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
|
MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setMasked(PBHelperClient.convert(masked))
|
.setMasked(PBHelperClient.convert(masked))
|
||||||
|
@ -564,8 +529,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DirectoryListing getListing(String src, byte[] startAfter,
|
public DirectoryListing getListing(String src, byte[] startAfter,
|
||||||
boolean needLocation) throws AccessControlException,
|
boolean needLocation) throws IOException {
|
||||||
FileNotFoundException, UnresolvedLinkException, IOException {
|
|
||||||
GetListingRequestProto req = GetListingRequestProto.newBuilder()
|
GetListingRequestProto req = GetListingRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setStartAfter(ByteString.copyFrom(startAfter))
|
.setStartAfter(ByteString.copyFrom(startAfter))
|
||||||
|
@ -583,8 +547,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void renewLease(String clientName) throws AccessControlException,
|
public void renewLease(String clientName) throws IOException {
|
||||||
IOException {
|
|
||||||
RenewLeaseRequestProto req = RenewLeaseRequestProto.newBuilder()
|
RenewLeaseRequestProto req = RenewLeaseRequestProto.newBuilder()
|
||||||
.setClientName(clientName).build();
|
.setClientName(clientName).build();
|
||||||
try {
|
try {
|
||||||
|
@ -632,22 +595,22 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DatanodeStorageReport[] getDatanodeStorageReport(DatanodeReportType type)
|
public DatanodeStorageReport[] getDatanodeStorageReport(
|
||||||
throws IOException {
|
DatanodeReportType type) throws IOException {
|
||||||
final GetDatanodeStorageReportRequestProto req
|
final GetDatanodeStorageReportRequestProto req
|
||||||
= GetDatanodeStorageReportRequestProto.newBuilder()
|
= GetDatanodeStorageReportRequestProto.newBuilder()
|
||||||
.setType(PBHelperClient.convert(type)).build();
|
.setType(PBHelperClient.convert(type)).build();
|
||||||
try {
|
try {
|
||||||
return PBHelperClient.convertDatanodeStorageReports(
|
return PBHelperClient.convertDatanodeStorageReports(
|
||||||
rpcProxy.getDatanodeStorageReport(null, req).getDatanodeStorageReportsList());
|
rpcProxy.getDatanodeStorageReport(null, req)
|
||||||
|
.getDatanodeStorageReportsList());
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getPreferredBlockSize(String filename) throws IOException,
|
public long getPreferredBlockSize(String filename) throws IOException {
|
||||||
UnresolvedLinkException {
|
|
||||||
GetPreferredBlockSizeRequestProto req = GetPreferredBlockSizeRequestProto
|
GetPreferredBlockSizeRequestProto req = GetPreferredBlockSizeRequestProto
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.setFilename(filename)
|
.setFilename(filename)
|
||||||
|
@ -660,9 +623,11 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException {
|
public boolean setSafeMode(SafeModeAction action, boolean isChecked)
|
||||||
|
throws IOException {
|
||||||
SetSafeModeRequestProto req = SetSafeModeRequestProto.newBuilder()
|
SetSafeModeRequestProto req = SetSafeModeRequestProto.newBuilder()
|
||||||
.setAction(PBHelperClient.convert(action)).setChecked(isChecked).build();
|
.setAction(PBHelperClient.convert(action))
|
||||||
|
.setChecked(isChecked).build();
|
||||||
try {
|
try {
|
||||||
return rpcProxy.setSafeMode(null, req).getResult();
|
return rpcProxy.setSafeMode(null, req).getResult();
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -671,7 +636,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void saveNamespace() throws AccessControlException, IOException {
|
public void saveNamespace() throws IOException {
|
||||||
try {
|
try {
|
||||||
rpcProxy.saveNamespace(null, VOID_SAVE_NAMESPACE_REQUEST);
|
rpcProxy.saveNamespace(null, VOID_SAVE_NAMESPACE_REQUEST);
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -680,7 +645,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long rollEdits() throws AccessControlException, IOException {
|
public long rollEdits() throws IOException {
|
||||||
try {
|
try {
|
||||||
RollEditsResponseProto resp = rpcProxy.rollEdits(null,
|
RollEditsResponseProto resp = rpcProxy.rollEdits(null,
|
||||||
VOID_ROLLEDITS_REQUEST);
|
VOID_ROLLEDITS_REQUEST);
|
||||||
|
@ -691,8 +656,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean restoreFailedStorage(String arg)
|
public boolean restoreFailedStorage(String arg) throws IOException{
|
||||||
throws AccessControlException, IOException{
|
|
||||||
RestoreFailedStorageRequestProto req = RestoreFailedStorageRequestProto
|
RestoreFailedStorageRequestProto req = RestoreFailedStorageRequestProto
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.setArg(arg).build();
|
.setArg(arg).build();
|
||||||
|
@ -722,11 +686,13 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException {
|
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
|
||||||
|
throws IOException {
|
||||||
final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder()
|
final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder()
|
||||||
.setAction(PBHelperClient.convert(action)).build();
|
.setAction(PBHelperClient.convert(action)).build();
|
||||||
try {
|
try {
|
||||||
final RollingUpgradeResponseProto proto = rpcProxy.rollingUpgrade(null, r);
|
final RollingUpgradeResponseProto proto =
|
||||||
|
rpcProxy.rollingUpgrade(null, r);
|
||||||
if (proto.hasRollingUpgradeInfo()) {
|
if (proto.hasRollingUpgradeInfo()) {
|
||||||
return PBHelperClient.convert(proto.getRollingUpgradeInfo());
|
return PBHelperClient.convert(proto.getRollingUpgradeInfo());
|
||||||
}
|
}
|
||||||
|
@ -764,8 +730,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public HdfsFileStatus getFileInfo(String src) throws AccessControlException,
|
public HdfsFileStatus getFileInfo(String src) throws IOException {
|
||||||
FileNotFoundException, UnresolvedLinkException, IOException {
|
|
||||||
GetFileInfoRequestProto req = GetFileInfoRequestProto.newBuilder()
|
GetFileInfoRequestProto req = GetFileInfoRequestProto.newBuilder()
|
||||||
.setSrc(src).build();
|
.setSrc(src).build();
|
||||||
try {
|
try {
|
||||||
|
@ -777,23 +742,21 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public HdfsFileStatus getFileLinkInfo(String src)
|
public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
|
||||||
throws AccessControlException, UnresolvedLinkException, IOException {
|
|
||||||
GetFileLinkInfoRequestProto req = GetFileLinkInfoRequestProto.newBuilder()
|
GetFileLinkInfoRequestProto req = GetFileLinkInfoRequestProto.newBuilder()
|
||||||
.setSrc(src).build();
|
.setSrc(src).build();
|
||||||
try {
|
try {
|
||||||
GetFileLinkInfoResponseProto result = rpcProxy.getFileLinkInfo(null, req);
|
GetFileLinkInfoResponseProto result = rpcProxy.getFileLinkInfo(null, req);
|
||||||
return result.hasFs() ?
|
return result.hasFs() ?
|
||||||
PBHelperClient.convert(rpcProxy.getFileLinkInfo(null, req).getFs()) : null;
|
PBHelperClient.convert(rpcProxy.getFileLinkInfo(null, req).getFs()) :
|
||||||
|
null;
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ContentSummary getContentSummary(String path)
|
public ContentSummary getContentSummary(String path) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
GetContentSummaryRequestProto req = GetContentSummaryRequestProto
|
GetContentSummaryRequestProto req = GetContentSummaryRequestProto
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
.setPath(path)
|
.setPath(path)
|
||||||
|
@ -808,9 +771,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
|
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
|
||||||
StorageType type)
|
StorageType type) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
final SetQuotaRequestProto.Builder builder
|
final SetQuotaRequestProto.Builder builder
|
||||||
= SetQuotaRequestProto.newBuilder()
|
= SetQuotaRequestProto.newBuilder()
|
||||||
.setPath(path)
|
.setPath(path)
|
||||||
|
@ -829,12 +790,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void fsync(String src, long fileId, String client,
|
public void fsync(String src, long fileId, String client,
|
||||||
long lastBlockLength)
|
long lastBlockLength) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
FsyncRequestProto req = FsyncRequestProto.newBuilder().setSrc(src)
|
FsyncRequestProto req = FsyncRequestProto.newBuilder().setSrc(src)
|
||||||
.setClient(client).setLastBlockLength(lastBlockLength)
|
.setClient(client).setLastBlockLength(lastBlockLength)
|
||||||
.setFileId(fileId).build();
|
.setFileId(fileId).build();
|
||||||
try {
|
try {
|
||||||
rpcProxy.fsync(null, req);
|
rpcProxy.fsync(null, req);
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -843,9 +802,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setTimes(String src, long mtime, long atime)
|
public void setTimes(String src, long mtime, long atime) throws IOException {
|
||||||
throws AccessControlException, FileNotFoundException,
|
|
||||||
UnresolvedLinkException, IOException {
|
|
||||||
SetTimesRequestProto req = SetTimesRequestProto.newBuilder()
|
SetTimesRequestProto req = SetTimesRequestProto.newBuilder()
|
||||||
.setSrc(src)
|
.setSrc(src)
|
||||||
.setMtime(mtime)
|
.setMtime(mtime)
|
||||||
|
@ -860,10 +817,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void createSymlink(String target, String link, FsPermission dirPerm,
|
public void createSymlink(String target, String link, FsPermission dirPerm,
|
||||||
boolean createParent) throws AccessControlException,
|
boolean createParent) throws IOException {
|
||||||
FileAlreadyExistsException, FileNotFoundException,
|
|
||||||
ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
|
|
||||||
IOException {
|
|
||||||
CreateSymlinkRequestProto req = CreateSymlinkRequestProto.newBuilder()
|
CreateSymlinkRequestProto req = CreateSymlinkRequestProto.newBuilder()
|
||||||
.setTarget(target)
|
.setTarget(target)
|
||||||
.setLink(link)
|
.setLink(link)
|
||||||
|
@ -878,8 +832,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getLinkTarget(String path) throws AccessControlException,
|
public String getLinkTarget(String path) throws IOException {
|
||||||
FileNotFoundException, IOException {
|
|
||||||
GetLinkTargetRequestProto req = GetLinkTargetRequestProto.newBuilder()
|
GetLinkTargetRequestProto req = GetLinkTargetRequestProto.newBuilder()
|
||||||
.setPath(path).build();
|
.setPath(path).build();
|
||||||
try {
|
try {
|
||||||
|
@ -908,7 +861,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void updatePipeline(String clientName, ExtendedBlock oldBlock,
|
public void updatePipeline(String clientName, ExtendedBlock oldBlock,
|
||||||
ExtendedBlock newBlock, DatanodeID[] newNodes, String[] storageIDs) throws IOException {
|
ExtendedBlock newBlock, DatanodeID[] newNodes, String[] storageIDs)
|
||||||
|
throws IOException {
|
||||||
UpdatePipelineRequestProto req = UpdatePipelineRequestProto.newBuilder()
|
UpdatePipelineRequestProto req = UpdatePipelineRequestProto.newBuilder()
|
||||||
.setClientName(clientName)
|
.setClientName(clientName)
|
||||||
.setOldBlock(PBHelperClient.convert(oldBlock))
|
.setOldBlock(PBHelperClient.convert(oldBlock))
|
||||||
|
@ -931,9 +885,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
.setRenewer(renewer == null ? "" : renewer.toString())
|
.setRenewer(renewer == null ? "" : renewer.toString())
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
GetDelegationTokenResponseProto resp = rpcProxy.getDelegationToken(null, req);
|
GetDelegationTokenResponseProto resp =
|
||||||
return resp.hasToken() ? PBHelperClient.convertDelegationToken(resp.getToken())
|
rpcProxy.getDelegationToken(null, req);
|
||||||
: null;
|
return resp.hasToken() ?
|
||||||
|
PBHelperClient.convertDelegationToken(resp.getToken()) : null;
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
@ -942,9 +897,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
@Override
|
@Override
|
||||||
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
|
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
RenewDelegationTokenRequestProto req = RenewDelegationTokenRequestProto.newBuilder().
|
RenewDelegationTokenRequestProto req =
|
||||||
setToken(PBHelperClient.convert(token)).
|
RenewDelegationTokenRequestProto.newBuilder().
|
||||||
build();
|
setToken(PBHelperClient.convert(token)).
|
||||||
|
build();
|
||||||
try {
|
try {
|
||||||
return rpcProxy.renewDelegationToken(null, req).getNewExpiryTime();
|
return rpcProxy.renewDelegationToken(null, req).getNewExpiryTime();
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -968,9 +924,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setBalancerBandwidth(long bandwidth) throws IOException {
|
public void setBalancerBandwidth(long bandwidth) throws IOException {
|
||||||
SetBalancerBandwidthRequestProto req = SetBalancerBandwidthRequestProto.newBuilder()
|
SetBalancerBandwidthRequestProto req =
|
||||||
.setBandwidth(bandwidth)
|
SetBalancerBandwidthRequestProto.newBuilder()
|
||||||
.build();
|
.setBandwidth(bandwidth)
|
||||||
|
.build();
|
||||||
try {
|
try {
|
||||||
rpcProxy.setBalancerBandwidth(null, req);
|
rpcProxy.setBalancerBandwidth(null, req);
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
|
@ -990,7 +947,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
try {
|
try {
|
||||||
GetDataEncryptionKeyResponseProto rsp = rpcProxy.getDataEncryptionKey(
|
GetDataEncryptionKeyResponseProto rsp = rpcProxy.getDataEncryptionKey(
|
||||||
null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST);
|
null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST);
|
||||||
return rsp.hasDataEncryptionKey() ?
|
return rsp.hasDataEncryptionKey() ?
|
||||||
PBHelperClient.convert(rsp.getDataEncryptionKey()) : null;
|
PBHelperClient.convert(rsp.getDataEncryptionKey()) : null;
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
|
@ -999,8 +956,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isFileClosed(String src) throws AccessControlException,
|
public boolean isFileClosed(String src) throws IOException {
|
||||||
FileNotFoundException, UnresolvedLinkException, IOException {
|
|
||||||
IsFileClosedRequestProto req = IsFileClosedRequestProto.newBuilder()
|
IsFileClosedRequestProto req = IsFileClosedRequestProto.newBuilder()
|
||||||
.setSrc(src).build();
|
.setSrc(src).build();
|
||||||
try {
|
try {
|
||||||
|
@ -1182,19 +1138,18 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public BatchedEntries<CacheDirectiveEntry>
|
public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(long prevId,
|
||||||
listCacheDirectives(long prevId,
|
CacheDirectiveInfo filter) throws IOException {
|
||||||
CacheDirectiveInfo filter) throws IOException {
|
|
||||||
if (filter == null) {
|
if (filter == null) {
|
||||||
filter = new CacheDirectiveInfo.Builder().build();
|
filter = new CacheDirectiveInfo.Builder().build();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return new BatchedCacheEntries(
|
return new BatchedCacheEntries(
|
||||||
rpcProxy.listCacheDirectives(null,
|
rpcProxy.listCacheDirectives(null,
|
||||||
ListCacheDirectivesRequestProto.newBuilder().
|
ListCacheDirectivesRequestProto.newBuilder().
|
||||||
setPrevId(prevId).
|
setPrevId(prevId).
|
||||||
setFilter(PBHelperClient.convert(filter)).
|
setFilter(PBHelperClient.convert(filter)).
|
||||||
build()));
|
build()));
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
@ -1229,15 +1184,15 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
try {
|
try {
|
||||||
rpcProxy.removeCachePool(null,
|
rpcProxy.removeCachePool(null,
|
||||||
RemoveCachePoolRequestProto.newBuilder().
|
RemoveCachePoolRequestProto.newBuilder().
|
||||||
setPoolName(cachePoolName).build());
|
setPoolName(cachePoolName).build());
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class BatchedCachePoolEntries
|
private static class BatchedCachePoolEntries
|
||||||
implements BatchedEntries<CachePoolEntry> {
|
implements BatchedEntries<CachePoolEntry> {
|
||||||
private final ListCachePoolsResponseProto proto;
|
private final ListCachePoolsResponseProto proto;
|
||||||
|
|
||||||
public BatchedCachePoolEntries(ListCachePoolsResponseProto proto) {
|
public BatchedCachePoolEntries(ListCachePoolsResponseProto proto) {
|
||||||
this.proto = proto;
|
this.proto = proto;
|
||||||
|
@ -1265,9 +1220,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
throws IOException {
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
return new BatchedCachePoolEntries(
|
return new BatchedCachePoolEntries(
|
||||||
rpcProxy.listCachePools(null,
|
rpcProxy.listCachePools(null,
|
||||||
ListCachePoolsRequestProto.newBuilder().
|
ListCachePoolsRequestProto.newBuilder().
|
||||||
setPrevPoolName(prevKey).build()));
|
setPrevPoolName(prevKey).build()));
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
@ -1347,9 +1302,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void createEncryptionZone(String src, String keyName)
|
public void createEncryptionZone(String src, String keyName)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final CreateEncryptionZoneRequestProto.Builder builder =
|
final CreateEncryptionZoneRequestProto.Builder builder =
|
||||||
CreateEncryptionZoneRequestProto.newBuilder();
|
CreateEncryptionZoneRequestProto.newBuilder();
|
||||||
builder.setSrc(src);
|
builder.setSrc(src);
|
||||||
if (keyName != null && !keyName.isEmpty()) {
|
if (keyName != null && !keyName.isEmpty()) {
|
||||||
builder.setKeyName(keyName);
|
builder.setKeyName(keyName);
|
||||||
|
@ -1363,8 +1318,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EncryptionZone getEZForPath(String src)
|
public EncryptionZone getEZForPath(String src) throws IOException {
|
||||||
throws IOException {
|
|
||||||
final GetEZForPathRequestProto.Builder builder =
|
final GetEZForPathRequestProto.Builder builder =
|
||||||
GetEZForPathRequestProto.newBuilder();
|
GetEZForPathRequestProto.newBuilder();
|
||||||
builder.setSrc(src);
|
builder.setSrc(src);
|
||||||
|
@ -1386,9 +1340,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
public BatchedEntries<EncryptionZone> listEncryptionZones(long id)
|
public BatchedEntries<EncryptionZone> listEncryptionZones(long id)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final ListEncryptionZonesRequestProto req =
|
final ListEncryptionZonesRequestProto req =
|
||||||
ListEncryptionZonesRequestProto.newBuilder()
|
ListEncryptionZonesRequestProto.newBuilder()
|
||||||
.setId(id)
|
.setId(id)
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
EncryptionZonesProtos.ListEncryptionZonesResponseProto response =
|
EncryptionZonesProtos.ListEncryptionZonesResponseProto response =
|
||||||
rpcProxy.listEncryptionZones(null, req);
|
rpcProxy.listEncryptionZones(null, req);
|
||||||
|
@ -1397,8 +1351,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
for (EncryptionZoneProto p : response.getZonesList()) {
|
for (EncryptionZoneProto p : response.getZonesList()) {
|
||||||
elements.add(PBHelperClient.convert(p));
|
elements.add(PBHelperClient.convert(p));
|
||||||
}
|
}
|
||||||
return new BatchedListEntries<EncryptionZone>(elements,
|
return new BatchedListEntries<>(elements, response.getHasMore());
|
||||||
response.getHasMore());
|
|
||||||
} catch (ServiceException e) {
|
} catch (ServiceException e) {
|
||||||
throw ProtobufHelper.getRemoteException(e);
|
throw ProtobufHelper.getRemoteException(e);
|
||||||
}
|
}
|
||||||
|
@ -1436,9 +1389,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<XAttr> listXAttrs(String src)
|
public List<XAttr> listXAttrs(String src) throws IOException {
|
||||||
throws IOException {
|
ListXAttrsRequestProto.Builder builder =
|
||||||
ListXAttrsRequestProto.Builder builder = ListXAttrsRequestProto.newBuilder();
|
ListXAttrsRequestProto.newBuilder();
|
||||||
builder.setSrc(src);
|
builder.setSrc(src);
|
||||||
ListXAttrsRequestProto req = builder.build();
|
ListXAttrsRequestProto req = builder.build();
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -207,34 +207,34 @@ public class PBHelperClient {
|
||||||
public static ExtendedBlockProto convert(final ExtendedBlock b) {
|
public static ExtendedBlockProto convert(final ExtendedBlock b) {
|
||||||
if (b == null) return null;
|
if (b == null) return null;
|
||||||
return ExtendedBlockProto.newBuilder().
|
return ExtendedBlockProto.newBuilder().
|
||||||
setPoolId(b.getBlockPoolId()).
|
setPoolId(b.getBlockPoolId()).
|
||||||
setBlockId(b.getBlockId()).
|
setBlockId(b.getBlockId()).
|
||||||
setNumBytes(b.getNumBytes()).
|
setNumBytes(b.getNumBytes()).
|
||||||
setGenerationStamp(b.getGenerationStamp()).
|
setGenerationStamp(b.getGenerationStamp()).
|
||||||
build();
|
build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static TokenProto convert(Token<?> tok) {
|
public static TokenProto convert(Token<?> tok) {
|
||||||
return TokenProto.newBuilder().
|
return TokenProto.newBuilder().
|
||||||
setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
|
setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
|
||||||
setPassword(ByteString.copyFrom(tok.getPassword())).
|
setPassword(ByteString.copyFrom(tok.getPassword())).
|
||||||
setKind(tok.getKind().toString()).
|
setKind(tok.getKind().toString()).
|
||||||
setService(tok.getService().toString()).build();
|
setService(tok.getService().toString()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ShortCircuitShmIdProto convert(ShmId shmId) {
|
public static ShortCircuitShmIdProto convert(ShmId shmId) {
|
||||||
return ShortCircuitShmIdProto.newBuilder().
|
return ShortCircuitShmIdProto.newBuilder().
|
||||||
setHi(shmId.getHi()).
|
setHi(shmId.getHi()).
|
||||||
setLo(shmId.getLo()).
|
setLo(shmId.getLo()).
|
||||||
build();
|
build();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
|
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
|
||||||
return ShortCircuitShmSlotProto.newBuilder().
|
return ShortCircuitShmSlotProto.newBuilder().
|
||||||
setShmId(convert(slotId.getShmId())).
|
setShmId(convert(slotId.getShmId())).
|
||||||
setSlotIdx(slotId.getSlotIdx()).
|
setSlotIdx(slotId.getSlotIdx()).
|
||||||
build();
|
build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DatanodeIDProto convert(DatanodeID dn) {
|
public static DatanodeIDProto convert(DatanodeID dn) {
|
||||||
|
@ -242,23 +242,24 @@ public class PBHelperClient {
|
||||||
// which is the same as the DatanodeUuid. Since StorageID is a required
|
// which is the same as the DatanodeUuid. Since StorageID is a required
|
||||||
// field we pass the empty string if the DatanodeUuid is not yet known.
|
// field we pass the empty string if the DatanodeUuid is not yet known.
|
||||||
return DatanodeIDProto.newBuilder()
|
return DatanodeIDProto.newBuilder()
|
||||||
.setIpAddr(dn.getIpAddr())
|
.setIpAddr(dn.getIpAddr())
|
||||||
.setHostName(dn.getHostName())
|
.setHostName(dn.getHostName())
|
||||||
.setXferPort(dn.getXferPort())
|
.setXferPort(dn.getXferPort())
|
||||||
.setDatanodeUuid(dn.getDatanodeUuid() != null ? dn.getDatanodeUuid() : "")
|
.setDatanodeUuid(dn.getDatanodeUuid() != null ?
|
||||||
.setInfoPort(dn.getInfoPort())
|
dn.getDatanodeUuid() : "")
|
||||||
.setInfoSecurePort(dn.getInfoSecurePort())
|
.setInfoPort(dn.getInfoPort())
|
||||||
.setIpcPort(dn.getIpcPort()).build();
|
.setInfoSecurePort(dn.getInfoSecurePort())
|
||||||
|
.setIpcPort(dn.getIpcPort()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DatanodeInfoProto.AdminState convert(
|
public static DatanodeInfoProto.AdminState convert(
|
||||||
final DatanodeInfo.AdminStates inAs) {
|
final DatanodeInfo.AdminStates inAs) {
|
||||||
switch (inAs) {
|
switch (inAs) {
|
||||||
case NORMAL: return DatanodeInfoProto.AdminState.NORMAL;
|
case NORMAL: return DatanodeInfoProto.AdminState.NORMAL;
|
||||||
case DECOMMISSION_INPROGRESS:
|
case DECOMMISSION_INPROGRESS:
|
||||||
return DatanodeInfoProto.AdminState.DECOMMISSION_INPROGRESS;
|
return DatanodeInfoProto.AdminState.DECOMMISSION_INPROGRESS;
|
||||||
case DECOMMISSIONED: return DatanodeInfoProto.AdminState.DECOMMISSIONED;
|
case DECOMMISSIONED: return DatanodeInfoProto.AdminState.DECOMMISSIONED;
|
||||||
default: return DatanodeInfoProto.AdminState.NORMAL;
|
default: return DatanodeInfoProto.AdminState.NORMAL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -271,23 +272,23 @@ public class PBHelperClient {
|
||||||
builder.setUpgradeDomain(info.getUpgradeDomain());
|
builder.setUpgradeDomain(info.getUpgradeDomain());
|
||||||
}
|
}
|
||||||
builder
|
builder
|
||||||
.setId(convert((DatanodeID) info))
|
.setId(convert((DatanodeID) info))
|
||||||
.setCapacity(info.getCapacity())
|
.setCapacity(info.getCapacity())
|
||||||
.setDfsUsed(info.getDfsUsed())
|
.setDfsUsed(info.getDfsUsed())
|
||||||
.setRemaining(info.getRemaining())
|
.setRemaining(info.getRemaining())
|
||||||
.setBlockPoolUsed(info.getBlockPoolUsed())
|
.setBlockPoolUsed(info.getBlockPoolUsed())
|
||||||
.setCacheCapacity(info.getCacheCapacity())
|
.setCacheCapacity(info.getCacheCapacity())
|
||||||
.setCacheUsed(info.getCacheUsed())
|
.setCacheUsed(info.getCacheUsed())
|
||||||
.setLastUpdate(info.getLastUpdate())
|
.setLastUpdate(info.getLastUpdate())
|
||||||
.setLastUpdateMonotonic(info.getLastUpdateMonotonic())
|
.setLastUpdateMonotonic(info.getLastUpdateMonotonic())
|
||||||
.setXceiverCount(info.getXceiverCount())
|
.setXceiverCount(info.getXceiverCount())
|
||||||
.setAdminState(convert(info.getAdminState()))
|
.setAdminState(convert(info.getAdminState()))
|
||||||
.build();
|
.build();
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
|
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
|
||||||
DatanodeInfo[] dnInfos) {
|
DatanodeInfo[] dnInfos) {
|
||||||
return convert(dnInfos, 0);
|
return convert(dnInfos, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -296,11 +297,11 @@ public class PBHelperClient {
|
||||||
* {@code startIdx}.
|
* {@code startIdx}.
|
||||||
*/
|
*/
|
||||||
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
|
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
|
||||||
DatanodeInfo[] dnInfos, int startIdx) {
|
DatanodeInfo[] dnInfos, int startIdx) {
|
||||||
if (dnInfos == null)
|
if (dnInfos == null)
|
||||||
return null;
|
return null;
|
||||||
ArrayList<HdfsProtos.DatanodeInfoProto> protos = Lists
|
ArrayList<HdfsProtos.DatanodeInfoProto> protos = Lists
|
||||||
.newArrayListWithCapacity(dnInfos.length);
|
.newArrayListWithCapacity(dnInfos.length);
|
||||||
for (int i = startIdx; i < dnInfos.length; i++) {
|
for (int i = startIdx; i < dnInfos.length; i++) {
|
||||||
protos.add(convert(dnInfos[i]));
|
protos.add(convert(dnInfos[i]));
|
||||||
}
|
}
|
||||||
|
@ -337,48 +338,48 @@ public class PBHelperClient {
|
||||||
|
|
||||||
public static StorageTypeProto convertStorageType(StorageType type) {
|
public static StorageTypeProto convertStorageType(StorageType type) {
|
||||||
switch(type) {
|
switch(type) {
|
||||||
case DISK:
|
case DISK:
|
||||||
return StorageTypeProto.DISK;
|
return StorageTypeProto.DISK;
|
||||||
case SSD:
|
case SSD:
|
||||||
return StorageTypeProto.SSD;
|
return StorageTypeProto.SSD;
|
||||||
case ARCHIVE:
|
case ARCHIVE:
|
||||||
return StorageTypeProto.ARCHIVE;
|
return StorageTypeProto.ARCHIVE;
|
||||||
case RAM_DISK:
|
case RAM_DISK:
|
||||||
return StorageTypeProto.RAM_DISK;
|
return StorageTypeProto.RAM_DISK;
|
||||||
default:
|
default:
|
||||||
throw new IllegalStateException(
|
throw new IllegalStateException(
|
||||||
"BUG: StorageType not found, type=" + type);
|
"BUG: StorageType not found, type=" + type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static StorageType convertStorageType(StorageTypeProto type) {
|
public static StorageType convertStorageType(StorageTypeProto type) {
|
||||||
switch(type) {
|
switch(type) {
|
||||||
case DISK:
|
case DISK:
|
||||||
return StorageType.DISK;
|
return StorageType.DISK;
|
||||||
case SSD:
|
case SSD:
|
||||||
return StorageType.SSD;
|
return StorageType.SSD;
|
||||||
case ARCHIVE:
|
case ARCHIVE:
|
||||||
return StorageType.ARCHIVE;
|
return StorageType.ARCHIVE;
|
||||||
case RAM_DISK:
|
case RAM_DISK:
|
||||||
return StorageType.RAM_DISK;
|
return StorageType.RAM_DISK;
|
||||||
default:
|
default:
|
||||||
throw new IllegalStateException(
|
throw new IllegalStateException(
|
||||||
"BUG: StorageTypeProto not found, type=" + type);
|
"BUG: StorageTypeProto not found, type=" + type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<StorageTypeProto> convertStorageTypes(
|
public static List<StorageTypeProto> convertStorageTypes(
|
||||||
StorageType[] types) {
|
StorageType[] types) {
|
||||||
return convertStorageTypes(types, 0);
|
return convertStorageTypes(types, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<StorageTypeProto> convertStorageTypes(
|
public static List<StorageTypeProto> convertStorageTypes(
|
||||||
StorageType[] types, int startIdx) {
|
StorageType[] types, int startIdx) {
|
||||||
if (types == null) {
|
if (types == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
final List<StorageTypeProto> protos = new ArrayList<>(
|
final List<StorageTypeProto> protos = new ArrayList<>(
|
||||||
types.length);
|
types.length);
|
||||||
for (int i = startIdx; i < types.length; ++i) {
|
for (int i = startIdx; i < types.length; ++i) {
|
||||||
protos.add(convertStorageType(types[i]));
|
protos.add(convertStorageType(types[i]));
|
||||||
}
|
}
|
||||||
|
@ -386,7 +387,7 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static InputStream vintPrefixed(final InputStream input)
|
public static InputStream vintPrefixed(final InputStream input)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final int firstByte = input.read();
|
final int firstByte = input.read();
|
||||||
if (firstByte == -1) {
|
if (firstByte == -1) {
|
||||||
throw new EOFException("Premature EOF: no length prefix available");
|
throw new EOFException("Premature EOF: no length prefix available");
|
||||||
|
@ -438,8 +439,8 @@ public class PBHelperClient {
|
||||||
|
|
||||||
public static HdfsProtos.CipherOptionProto convert(CipherOption option) {
|
public static HdfsProtos.CipherOptionProto convert(CipherOption option) {
|
||||||
if (option != null) {
|
if (option != null) {
|
||||||
HdfsProtos.CipherOptionProto.Builder builder = HdfsProtos.CipherOptionProto.
|
HdfsProtos.CipherOptionProto.Builder builder =
|
||||||
newBuilder();
|
HdfsProtos.CipherOptionProto.newBuilder();
|
||||||
if (option.getCipherSuite() != null) {
|
if (option.getCipherSuite() != null) {
|
||||||
builder.setSuite(convert(option.getCipherSuite()));
|
builder.setSuite(convert(option.getCipherSuite()));
|
||||||
}
|
}
|
||||||
|
@ -514,7 +515,8 @@ public class PBHelperClient {
|
||||||
storageIDs = null;
|
storageIDs = null;
|
||||||
} else {
|
} else {
|
||||||
Preconditions.checkState(storageIDsCount == locs.size());
|
Preconditions.checkState(storageIDsCount == locs.size());
|
||||||
storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
|
storageIDs = proto.getStorageIDsList()
|
||||||
|
.toArray(new String[storageIDsCount]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set values from the isCached list, re-using references from loc
|
// Set values from the isCached list, re-using references from loc
|
||||||
|
@ -550,7 +552,7 @@ public class PBHelperClient {
|
||||||
List<StorageTypeProto> storageTypesList, int expectedSize) {
|
List<StorageTypeProto> storageTypesList, int expectedSize) {
|
||||||
final StorageType[] storageTypes = new StorageType[expectedSize];
|
final StorageType[] storageTypes = new StorageType[expectedSize];
|
||||||
if (storageTypesList.size() != expectedSize) {
|
if (storageTypesList.size() != expectedSize) {
|
||||||
// missing storage types
|
// missing storage types
|
||||||
Preconditions.checkState(storageTypesList.isEmpty());
|
Preconditions.checkState(storageTypesList.isEmpty());
|
||||||
Arrays.fill(storageTypes, StorageType.DEFAULT);
|
Arrays.fill(storageTypes, StorageType.DEFAULT);
|
||||||
} else {
|
} else {
|
||||||
|
@ -570,9 +572,9 @@ public class PBHelperClient {
|
||||||
|
|
||||||
// DatanodeId
|
// DatanodeId
|
||||||
public static DatanodeID convert(DatanodeIDProto dn) {
|
public static DatanodeID convert(DatanodeIDProto dn) {
|
||||||
return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
|
return new DatanodeID(dn.getIpAddr(), dn.getHostName(),
|
||||||
dn.getXferPort(), dn.getInfoPort(), dn.hasInfoSecurePort() ? dn
|
dn.getDatanodeUuid(), dn.getXferPort(), dn.getInfoPort(),
|
||||||
.getInfoSecurePort() : 0, dn.getIpcPort());
|
dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0, dn.getIpcPort());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static AdminStates convert(AdminState adminState) {
|
public static AdminStates convert(AdminState adminState) {
|
||||||
|
@ -611,8 +613,8 @@ public class PBHelperClient {
|
||||||
return policies;
|
return policies;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
|
public static EventBatchList convert(GetEditsFromTxidResponseProto resp)
|
||||||
IOException {
|
throws IOException {
|
||||||
final InotifyProtos.EventsListProto list = resp.getEventsList();
|
final InotifyProtos.EventsListProto list = resp.getEventsList();
|
||||||
final long firstTxid = list.getFirstTxid();
|
final long firstTxid = list.getFirstTxid();
|
||||||
final long lastTxid = list.getLastTxid();
|
final long lastTxid = list.getLastTxid();
|
||||||
|
@ -631,82 +633,82 @@ public class PBHelperClient {
|
||||||
List<Event> events = Lists.newArrayList();
|
List<Event> events = Lists.newArrayList();
|
||||||
for (InotifyProtos.EventProto p : bp.getEventsList()) {
|
for (InotifyProtos.EventProto p : bp.getEventsList()) {
|
||||||
switch (p.getType()) {
|
switch (p.getType()) {
|
||||||
case EVENT_CLOSE:
|
case EVENT_CLOSE:
|
||||||
InotifyProtos.CloseEventProto close =
|
InotifyProtos.CloseEventProto close =
|
||||||
InotifyProtos.CloseEventProto.parseFrom(p.getContents());
|
InotifyProtos.CloseEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.CloseEvent(close.getPath(),
|
events.add(new Event.CloseEvent(close.getPath(),
|
||||||
close.getFileSize(), close.getTimestamp()));
|
close.getFileSize(), close.getTimestamp()));
|
||||||
break;
|
break;
|
||||||
case EVENT_CREATE:
|
case EVENT_CREATE:
|
||||||
InotifyProtos.CreateEventProto create =
|
InotifyProtos.CreateEventProto create =
|
||||||
InotifyProtos.CreateEventProto.parseFrom(p.getContents());
|
InotifyProtos.CreateEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.CreateEvent.Builder()
|
events.add(new Event.CreateEvent.Builder()
|
||||||
.iNodeType(createTypeConvert(create.getType()))
|
.iNodeType(createTypeConvert(create.getType()))
|
||||||
.path(create.getPath())
|
.path(create.getPath())
|
||||||
.ctime(create.getCtime())
|
.ctime(create.getCtime())
|
||||||
.ownerName(create.getOwnerName())
|
.ownerName(create.getOwnerName())
|
||||||
.groupName(create.getGroupName())
|
.groupName(create.getGroupName())
|
||||||
.perms(convert(create.getPerms()))
|
.perms(convert(create.getPerms()))
|
||||||
.replication(create.getReplication())
|
.replication(create.getReplication())
|
||||||
.symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
|
.symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
|
||||||
create.getSymlinkTarget())
|
create.getSymlinkTarget())
|
||||||
.defaultBlockSize(create.getDefaultBlockSize())
|
.defaultBlockSize(create.getDefaultBlockSize())
|
||||||
.overwrite(create.getOverwrite()).build());
|
.overwrite(create.getOverwrite()).build());
|
||||||
break;
|
break;
|
||||||
case EVENT_METADATA:
|
case EVENT_METADATA:
|
||||||
InotifyProtos.MetadataUpdateEventProto meta =
|
InotifyProtos.MetadataUpdateEventProto meta =
|
||||||
InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
|
InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.MetadataUpdateEvent.Builder()
|
events.add(new Event.MetadataUpdateEvent.Builder()
|
||||||
.path(meta.getPath())
|
.path(meta.getPath())
|
||||||
.metadataType(metadataUpdateTypeConvert(meta.getType()))
|
.metadataType(metadataUpdateTypeConvert(meta.getType()))
|
||||||
.mtime(meta.getMtime())
|
.mtime(meta.getMtime())
|
||||||
.atime(meta.getAtime())
|
.atime(meta.getAtime())
|
||||||
.replication(meta.getReplication())
|
.replication(meta.getReplication())
|
||||||
.ownerName(
|
.ownerName(
|
||||||
meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
|
meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
|
||||||
.groupName(
|
.groupName(
|
||||||
meta.getGroupName().isEmpty() ? null : meta.getGroupName())
|
meta.getGroupName().isEmpty() ? null : meta.getGroupName())
|
||||||
.perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
|
.perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
|
||||||
.acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
|
.acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
|
||||||
meta.getAclsList()))
|
meta.getAclsList()))
|
||||||
.xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
|
.xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
|
||||||
meta.getXAttrsList()))
|
meta.getXAttrsList()))
|
||||||
.xAttrsRemoved(meta.getXAttrsRemoved())
|
.xAttrsRemoved(meta.getXAttrsRemoved())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case EVENT_RENAME:
|
case EVENT_RENAME:
|
||||||
InotifyProtos.RenameEventProto rename =
|
InotifyProtos.RenameEventProto rename =
|
||||||
InotifyProtos.RenameEventProto.parseFrom(p.getContents());
|
InotifyProtos.RenameEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.RenameEvent.Builder()
|
events.add(new Event.RenameEvent.Builder()
|
||||||
.srcPath(rename.getSrcPath())
|
.srcPath(rename.getSrcPath())
|
||||||
.dstPath(rename.getDestPath())
|
.dstPath(rename.getDestPath())
|
||||||
.timestamp(rename.getTimestamp())
|
.timestamp(rename.getTimestamp())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case EVENT_APPEND:
|
case EVENT_APPEND:
|
||||||
InotifyProtos.AppendEventProto append =
|
InotifyProtos.AppendEventProto append =
|
||||||
InotifyProtos.AppendEventProto.parseFrom(p.getContents());
|
InotifyProtos.AppendEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.AppendEvent.Builder().path(append.getPath())
|
events.add(new Event.AppendEvent.Builder().path(append.getPath())
|
||||||
.newBlock(append.hasNewBlock() && append.getNewBlock())
|
.newBlock(append.hasNewBlock() && append.getNewBlock())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case EVENT_UNLINK:
|
case EVENT_UNLINK:
|
||||||
InotifyProtos.UnlinkEventProto unlink =
|
InotifyProtos.UnlinkEventProto unlink =
|
||||||
InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
|
InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.UnlinkEvent.Builder()
|
events.add(new Event.UnlinkEvent.Builder()
|
||||||
.path(unlink.getPath())
|
.path(unlink.getPath())
|
||||||
.timestamp(unlink.getTimestamp())
|
.timestamp(unlink.getTimestamp())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case EVENT_TRUNCATE:
|
case EVENT_TRUNCATE:
|
||||||
InotifyProtos.TruncateEventProto truncate =
|
InotifyProtos.TruncateEventProto truncate =
|
||||||
InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
|
InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
|
||||||
events.add(new Event.TruncateEvent(truncate.getPath(),
|
events.add(new Event.TruncateEvent(truncate.getPath(),
|
||||||
truncate.getFileSize(), truncate.getTimestamp()));
|
truncate.getFileSize(), truncate.getTimestamp()));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new RuntimeException("Unexpected inotify event type: " +
|
throw new RuntimeException("Unexpected inotify event type: " +
|
||||||
p.getType());
|
p.getType());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
batches.add(new EventBatch(txid, events.toArray(new Event[0])));
|
batches.add(new EventBatch(txid, events.toArray(new Event[0])));
|
||||||
|
@ -878,7 +880,7 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
|
static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
|
||||||
type) {
|
type) {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case DIRECTORY:
|
case DIRECTORY:
|
||||||
return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
|
return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
|
||||||
|
@ -1090,7 +1092,7 @@ public class PBHelperClient {
|
||||||
String poolName = checkNotNull(proto.getPoolName());
|
String poolName = checkNotNull(proto.getPoolName());
|
||||||
CachePoolInfo info = new CachePoolInfo(poolName);
|
CachePoolInfo info = new CachePoolInfo(poolName);
|
||||||
if (proto.hasOwnerName()) {
|
if (proto.hasOwnerName()) {
|
||||||
info.setOwnerName(proto.getOwnerName());
|
info.setOwnerName(proto.getOwnerName());
|
||||||
}
|
}
|
||||||
if (proto.hasGroupName()) {
|
if (proto.hasGroupName()) {
|
||||||
info.setGroupName(proto.getGroupName());
|
info.setGroupName(proto.getGroupName());
|
||||||
|
@ -1138,8 +1140,7 @@ public class PBHelperClient {
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static CacheDirectiveInfoProto convert
|
public static CacheDirectiveInfoProto convert(CacheDirectiveInfo info) {
|
||||||
(CacheDirectiveInfo info) {
|
|
||||||
CacheDirectiveInfoProto.Builder builder =
|
CacheDirectiveInfoProto.Builder builder =
|
||||||
CacheDirectiveInfoProto.newBuilder();
|
CacheDirectiveInfoProto.newBuilder();
|
||||||
if (info.getId() != null) {
|
if (info.getId() != null) {
|
||||||
|
@ -1184,10 +1185,8 @@ public class PBHelperClient {
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static CacheDirectiveInfo convert
|
public static CacheDirectiveInfo convert(CacheDirectiveInfoProto proto) {
|
||||||
(CacheDirectiveInfoProto proto) {
|
CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder();
|
||||||
CacheDirectiveInfo.Builder builder =
|
|
||||||
new CacheDirectiveInfo.Builder();
|
|
||||||
if (proto.hasId()) {
|
if (proto.hasId()) {
|
||||||
builder.setId(proto.getId());
|
builder.setId(proto.getId());
|
||||||
}
|
}
|
||||||
|
@ -1223,7 +1222,8 @@ public class PBHelperClient {
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
|
public static SnapshotDiffReport convert(
|
||||||
|
SnapshotDiffReportProto reportProto) {
|
||||||
if (reportProto == null) {
|
if (reportProto == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -1442,8 +1442,7 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static SafeModeActionProto convert(
|
public static SafeModeActionProto convert(SafeModeAction a) {
|
||||||
SafeModeAction a) {
|
|
||||||
switch (a) {
|
switch (a) {
|
||||||
case SAFEMODE_LEAVE:
|
case SAFEMODE_LEAVE:
|
||||||
return SafeModeActionProto.SAFEMODE_LEAVE;
|
return SafeModeActionProto.SAFEMODE_LEAVE;
|
||||||
|
@ -1469,16 +1468,18 @@ public class PBHelperClient {
|
||||||
result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity();
|
result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity();
|
||||||
result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed();
|
result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed();
|
||||||
result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining();
|
result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining();
|
||||||
result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = res.getUnderReplicated();
|
result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] =
|
||||||
result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = res.getCorruptBlocks();
|
res.getUnderReplicated();
|
||||||
result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = res.getMissingBlocks();
|
result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] =
|
||||||
|
res.getCorruptBlocks();
|
||||||
|
result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] =
|
||||||
|
res.getMissingBlocks();
|
||||||
result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] =
|
result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] =
|
||||||
res.getMissingReplOneBlocks();
|
res.getMissingReplOneBlocks();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DatanodeReportTypeProto
|
public static DatanodeReportTypeProto convert(DatanodeReportType t) {
|
||||||
convert(DatanodeReportType t) {
|
|
||||||
switch (t) {
|
switch (t) {
|
||||||
case ALL: return DatanodeReportTypeProto.ALL;
|
case ALL: return DatanodeReportTypeProto.ALL;
|
||||||
case LIVE: return DatanodeReportTypeProto.LIVE;
|
case LIVE: return DatanodeReportTypeProto.LIVE;
|
||||||
|
@ -1636,8 +1637,8 @@ public class PBHelperClient {
|
||||||
DatanodeStorageReport[] reports) {
|
DatanodeStorageReport[] reports) {
|
||||||
final List<DatanodeStorageReportProto> protos
|
final List<DatanodeStorageReportProto> protos
|
||||||
= new ArrayList<>(reports.length);
|
= new ArrayList<>(reports.length);
|
||||||
for(int i = 0; i < reports.length; i++) {
|
for (DatanodeStorageReport report : reports) {
|
||||||
protos.add(convertDatanodeStorageReport(reports[i]));
|
protos.add(convertDatanodeStorageReport(report));
|
||||||
}
|
}
|
||||||
return protos;
|
return protos;
|
||||||
}
|
}
|
||||||
|
@ -1682,20 +1683,20 @@ public class PBHelperClient {
|
||||||
public static FsServerDefaultsProto convert(FsServerDefaults fs) {
|
public static FsServerDefaultsProto convert(FsServerDefaults fs) {
|
||||||
if (fs == null) return null;
|
if (fs == null) return null;
|
||||||
return FsServerDefaultsProto.newBuilder().
|
return FsServerDefaultsProto.newBuilder().
|
||||||
setBlockSize(fs.getBlockSize()).
|
setBlockSize(fs.getBlockSize()).
|
||||||
setBytesPerChecksum(fs.getBytesPerChecksum()).
|
setBytesPerChecksum(fs.getBytesPerChecksum()).
|
||||||
setWritePacketSize(fs.getWritePacketSize())
|
setWritePacketSize(fs.getWritePacketSize())
|
||||||
.setReplication(fs.getReplication())
|
.setReplication(fs.getReplication())
|
||||||
.setFileBufferSize(fs.getFileBufferSize())
|
.setFileBufferSize(fs.getFileBufferSize())
|
||||||
.setEncryptDataTransfer(fs.getEncryptDataTransfer())
|
.setEncryptDataTransfer(fs.getEncryptDataTransfer())
|
||||||
.setTrashInterval(fs.getTrashInterval())
|
.setTrashInterval(fs.getTrashInterval())
|
||||||
.setChecksumType(convert(fs.getChecksumType()))
|
.setChecksumType(convert(fs.getChecksumType()))
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
|
public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
|
||||||
EnumSet<CreateFlag> result =
|
EnumSet<CreateFlag> result =
|
||||||
EnumSet.noneOf(CreateFlag.class);
|
EnumSet.noneOf(CreateFlag.class);
|
||||||
if ((flag & CreateFlagProto.APPEND_VALUE) == CreateFlagProto.APPEND_VALUE) {
|
if ((flag & CreateFlagProto.APPEND_VALUE) == CreateFlagProto.APPEND_VALUE) {
|
||||||
result.add(CreateFlag.APPEND);
|
result.add(CreateFlag.APPEND);
|
||||||
}
|
}
|
||||||
|
@ -1714,7 +1715,7 @@ public class PBHelperClient {
|
||||||
== CreateFlagProto.NEW_BLOCK_VALUE) {
|
== CreateFlagProto.NEW_BLOCK_VALUE) {
|
||||||
result.add(CreateFlag.NEW_BLOCK);
|
result.add(CreateFlag.NEW_BLOCK);
|
||||||
}
|
}
|
||||||
return new EnumSetWritable<CreateFlag>(result, CreateFlag.class);
|
return new EnumSetWritable<>(result, CreateFlag.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
|
public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
|
||||||
|
@ -1736,20 +1737,20 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
HdfsFileStatusProto.Builder builder =
|
HdfsFileStatusProto.Builder builder =
|
||||||
HdfsFileStatusProto.newBuilder().
|
HdfsFileStatusProto.newBuilder().
|
||||||
setLength(fs.getLen()).
|
setLength(fs.getLen()).
|
||||||
setFileType(fType).
|
setFileType(fType).
|
||||||
setBlockReplication(fs.getReplication()).
|
setBlockReplication(fs.getReplication()).
|
||||||
setBlocksize(fs.getBlockSize()).
|
setBlocksize(fs.getBlockSize()).
|
||||||
setModificationTime(fs.getModificationTime()).
|
setModificationTime(fs.getModificationTime()).
|
||||||
setAccessTime(fs.getAccessTime()).
|
setAccessTime(fs.getAccessTime()).
|
||||||
setPermission(convert(fs.getPermission())).
|
setPermission(convert(fs.getPermission())).
|
||||||
setOwner(fs.getOwner()).
|
setOwner(fs.getOwner()).
|
||||||
setGroup(fs.getGroup()).
|
setGroup(fs.getGroup()).
|
||||||
setFileId(fs.getFileId()).
|
setFileId(fs.getFileId()).
|
||||||
setChildrenNum(fs.getChildrenNum()).
|
setChildrenNum(fs.getChildrenNum()).
|
||||||
setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
|
setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
|
||||||
setStoragePolicy(fs.getStoragePolicy());
|
setStoragePolicy(fs.getStoragePolicy());
|
||||||
if (fs.isSymlink()) {
|
if (fs.isSymlink()) {
|
||||||
builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
|
builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
|
||||||
}
|
}
|
||||||
|
@ -1779,9 +1780,11 @@ public class PBHelperClient {
|
||||||
HdfsFileStatusProto fs = convert(status.getDirStatus());
|
HdfsFileStatusProto fs = convert(status.getDirStatus());
|
||||||
SnapshottableDirectoryStatusProto.Builder builder =
|
SnapshottableDirectoryStatusProto.Builder builder =
|
||||||
SnapshottableDirectoryStatusProto
|
SnapshottableDirectoryStatusProto
|
||||||
.newBuilder().setSnapshotNumber(snapshotNumber)
|
.newBuilder()
|
||||||
.setSnapshotQuota(snapshotQuota).setParentFullpath(parentFullPathBytes)
|
.setSnapshotNumber(snapshotNumber)
|
||||||
.setDirStatus(fs);
|
.setSnapshotQuota(snapshotQuota)
|
||||||
|
.setParentFullpath(parentFullPathBytes)
|
||||||
|
.setDirStatus(fs);
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1816,14 +1819,15 @@ public class PBHelperClient {
|
||||||
result.setRemaining(fsStats[ClientProtocol.GET_STATS_REMAINING_IDX]);
|
result.setRemaining(fsStats[ClientProtocol.GET_STATS_REMAINING_IDX]);
|
||||||
if (fsStats.length >= ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX + 1)
|
if (fsStats.length >= ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX + 1)
|
||||||
result.setUnderReplicated(
|
result.setUnderReplicated(
|
||||||
fsStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
|
fsStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
|
||||||
if (fsStats.length >= ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX + 1)
|
if (fsStats.length >= ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX + 1)
|
||||||
result.setCorruptBlocks(
|
result.setCorruptBlocks(
|
||||||
fsStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]);
|
fsStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]);
|
||||||
if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1)
|
if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1)
|
||||||
result.setMissingBlocks(
|
result.setMissingBlocks(
|
||||||
fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]);
|
fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]);
|
||||||
if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1)
|
if (fsStats.length >=
|
||||||
|
ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1)
|
||||||
result.setMissingReplOneBlocks(
|
result.setMissingReplOneBlocks(
|
||||||
fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]);
|
fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]);
|
||||||
return result.build();
|
return result.build();
|
||||||
|
@ -1901,7 +1905,7 @@ public class PBHelperClient {
|
||||||
public static ContentSummaryProto convert(ContentSummary cs) {
|
public static ContentSummaryProto convert(ContentSummary cs) {
|
||||||
if (cs == null) return null;
|
if (cs == null) return null;
|
||||||
ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
|
ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
|
||||||
builder.setLength(cs.getLength()).
|
builder.setLength(cs.getLength()).
|
||||||
setFileCount(cs.getFileCount()).
|
setFileCount(cs.getFileCount()).
|
||||||
setDirectoryCount(cs.getDirectoryCount()).
|
setDirectoryCount(cs.getDirectoryCount()).
|
||||||
setQuota(cs.getQuota()).
|
setQuota(cs.getQuota()).
|
||||||
|
@ -1951,11 +1955,11 @@ public class PBHelperClient {
|
||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<StorageReportProto> convertStorageReports(StorageReport[] storages) {
|
public static List<StorageReportProto> convertStorageReports(
|
||||||
final List<StorageReportProto> protos = new ArrayList<StorageReportProto>(
|
StorageReport[] storages) {
|
||||||
storages.length);
|
final List<StorageReportProto> protos = new ArrayList<>(storages.length);
|
||||||
for(int i = 0; i < storages.length; i++) {
|
for (StorageReport storage : storages) {
|
||||||
protos.add(convert(storages[i]));
|
protos.add(convert(storage));
|
||||||
}
|
}
|
||||||
return protos;
|
return protos;
|
||||||
}
|
}
|
||||||
|
@ -1978,17 +1982,16 @@ public class PBHelperClient {
|
||||||
if (entry == null) {
|
if (entry == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
ByteString sourcePath = ByteString
|
ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ?
|
||||||
.copyFrom(entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : entry
|
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
|
||||||
.getSourcePath());
|
|
||||||
String modification = entry.getType().getLabel();
|
String modification = entry.getType().getLabel();
|
||||||
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
|
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
|
||||||
.newBuilder().setFullpath(sourcePath)
|
.newBuilder().setFullpath(sourcePath)
|
||||||
.setModificationLabel(modification);
|
.setModificationLabel(modification);
|
||||||
if (entry.getType() == DiffType.RENAME) {
|
if (entry.getType() == DiffType.RENAME) {
|
||||||
ByteString targetPath = ByteString
|
ByteString targetPath =
|
||||||
.copyFrom(entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : entry
|
ByteString.copyFrom(entry.getTargetPath() == null ?
|
||||||
.getTargetPath());
|
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
|
||||||
builder.setTargetPath(targetPath);
|
builder.setTargetPath(targetPath);
|
||||||
}
|
}
|
||||||
return builder.build();
|
return builder.build();
|
||||||
|
@ -2006,12 +2009,11 @@ public class PBHelperClient {
|
||||||
entryProtos.add(entryProto);
|
entryProtos.add(entryProto);
|
||||||
}
|
}
|
||||||
|
|
||||||
SnapshotDiffReportProto reportProto = SnapshotDiffReportProto.newBuilder()
|
return SnapshotDiffReportProto.newBuilder()
|
||||||
.setSnapshotRoot(report.getSnapshotRoot())
|
.setSnapshotRoot(report.getSnapshotRoot())
|
||||||
.setFromSnapshot(report.getFromSnapshot())
|
.setFromSnapshot(report.getFromSnapshot())
|
||||||
.setToSnapshot(report.getLaterSnapshotName())
|
.setToSnapshot(report.getLaterSnapshotName())
|
||||||
.addAllDiffReportEntries(entryProtos).build();
|
.addAllDiffReportEntries(entryProtos).build();
|
||||||
return reportProto;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) {
|
public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) {
|
||||||
|
@ -2034,7 +2036,7 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean[] convertBooleanList(
|
public static boolean[] convertBooleanList(
|
||||||
List<Boolean> targetPinningsList) {
|
List<Boolean> targetPinningsList) {
|
||||||
final boolean[] targetPinnings = new boolean[targetPinningsList.size()];
|
final boolean[] targetPinnings = new boolean[targetPinningsList.size()];
|
||||||
for (int i = 0; i < targetPinningsList.size(); i++) {
|
for (int i = 0; i < targetPinningsList.size(); i++) {
|
||||||
targetPinnings[i] = targetPinningsList.get(i);
|
targetPinnings[i] = targetPinningsList.get(i);
|
||||||
|
@ -2060,7 +2062,8 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DatanodeLocalInfoProto convert(DatanodeLocalInfo info) {
|
public static DatanodeLocalInfoProto convert(DatanodeLocalInfo info) {
|
||||||
DatanodeLocalInfoProto.Builder builder = DatanodeLocalInfoProto.newBuilder();
|
DatanodeLocalInfoProto.Builder builder =
|
||||||
|
DatanodeLocalInfoProto.newBuilder();
|
||||||
builder.setSoftwareVersion(info.getSoftwareVersion());
|
builder.setSoftwareVersion(info.getSoftwareVersion());
|
||||||
builder.setConfigVersion(info.getConfigVersion());
|
builder.setConfigVersion(info.getConfigVersion());
|
||||||
builder.setUptime(info.getUptime());
|
builder.setUptime(info.getUptime());
|
||||||
|
@ -2116,9 +2119,9 @@ public class PBHelperClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ListXAttrsResponseProto convertListXAttrsResponse(
|
public static ListXAttrsResponseProto convertListXAttrsResponse(
|
||||||
List<XAttr> names) {
|
List<XAttr> names) {
|
||||||
ListXAttrsResponseProto.Builder builder =
|
ListXAttrsResponseProto.Builder builder =
|
||||||
ListXAttrsResponseProto.newBuilder();
|
ListXAttrsResponseProto.newBuilder();
|
||||||
if (names != null) {
|
if (names != null) {
|
||||||
builder.addAllXAttrs(convertXAttrProto(names));
|
builder.addAllXAttrs(convertXAttrProto(names));
|
||||||
}
|
}
|
||||||
|
@ -2140,114 +2143,115 @@ public class PBHelperClient {
|
||||||
slotId.getSlotIdx());
|
slotId.getSlotIdx());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
|
public static GetEditsFromTxidResponseProto convertEditsResponse(
|
||||||
|
EventBatchList el) {
|
||||||
InotifyProtos.EventsListProto.Builder builder =
|
InotifyProtos.EventsListProto.Builder builder =
|
||||||
InotifyProtos.EventsListProto.newBuilder();
|
InotifyProtos.EventsListProto.newBuilder();
|
||||||
for (EventBatch b : el.getBatches()) {
|
for (EventBatch b : el.getBatches()) {
|
||||||
List<InotifyProtos.EventProto> events = Lists.newArrayList();
|
List<InotifyProtos.EventProto> events = Lists.newArrayList();
|
||||||
for (Event e : b.getEvents()) {
|
for (Event e : b.getEvents()) {
|
||||||
switch (e.getEventType()) {
|
switch (e.getEventType()) {
|
||||||
case CLOSE:
|
case CLOSE:
|
||||||
Event.CloseEvent ce = (Event.CloseEvent) e;
|
Event.CloseEvent ce = (Event.CloseEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_CLOSE)
|
.setType(InotifyProtos.EventType.EVENT_CLOSE)
|
||||||
.setContents(
|
.setContents(
|
||||||
InotifyProtos.CloseEventProto.newBuilder()
|
InotifyProtos.CloseEventProto.newBuilder()
|
||||||
.setPath(ce.getPath())
|
.setPath(ce.getPath())
|
||||||
.setFileSize(ce.getFileSize())
|
.setFileSize(ce.getFileSize())
|
||||||
.setTimestamp(ce.getTimestamp()).build().toByteString()
|
.setTimestamp(ce.getTimestamp()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
case CREATE:
|
case CREATE:
|
||||||
Event.CreateEvent ce2 = (Event.CreateEvent) e;
|
Event.CreateEvent ce2 = (Event.CreateEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_CREATE)
|
.setType(InotifyProtos.EventType.EVENT_CREATE)
|
||||||
.setContents(
|
.setContents(
|
||||||
InotifyProtos.CreateEventProto.newBuilder()
|
InotifyProtos.CreateEventProto.newBuilder()
|
||||||
.setType(createTypeConvert(ce2.getiNodeType()))
|
.setType(createTypeConvert(ce2.getiNodeType()))
|
||||||
.setPath(ce2.getPath())
|
.setPath(ce2.getPath())
|
||||||
.setCtime(ce2.getCtime())
|
.setCtime(ce2.getCtime())
|
||||||
.setOwnerName(ce2.getOwnerName())
|
.setOwnerName(ce2.getOwnerName())
|
||||||
.setGroupName(ce2.getGroupName())
|
.setGroupName(ce2.getGroupName())
|
||||||
.setPerms(convert(ce2.getPerms()))
|
.setPerms(convert(ce2.getPerms()))
|
||||||
.setReplication(ce2.getReplication())
|
.setReplication(ce2.getReplication())
|
||||||
.setSymlinkTarget(ce2.getSymlinkTarget() == null ?
|
.setSymlinkTarget(ce2.getSymlinkTarget() == null ?
|
||||||
"" : ce2.getSymlinkTarget())
|
"" : ce2.getSymlinkTarget())
|
||||||
.setDefaultBlockSize(ce2.getDefaultBlockSize())
|
.setDefaultBlockSize(ce2.getDefaultBlockSize())
|
||||||
.setOverwrite(ce2.getOverwrite()).build().toByteString()
|
.setOverwrite(ce2.getOverwrite()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
case METADATA:
|
case METADATA:
|
||||||
Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
|
Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
|
||||||
InotifyProtos.MetadataUpdateEventProto.Builder metaB =
|
InotifyProtos.MetadataUpdateEventProto.Builder metaB =
|
||||||
InotifyProtos.MetadataUpdateEventProto.newBuilder()
|
InotifyProtos.MetadataUpdateEventProto.newBuilder()
|
||||||
.setPath(me.getPath())
|
.setPath(me.getPath())
|
||||||
.setType(metadataUpdateTypeConvert(me.getMetadataType()))
|
.setType(metadataUpdateTypeConvert(me.getMetadataType()))
|
||||||
.setMtime(me.getMtime())
|
.setMtime(me.getMtime())
|
||||||
.setAtime(me.getAtime())
|
.setAtime(me.getAtime())
|
||||||
.setReplication(me.getReplication())
|
.setReplication(me.getReplication())
|
||||||
.setOwnerName(me.getOwnerName() == null ? "" :
|
.setOwnerName(me.getOwnerName() == null ? "" :
|
||||||
me.getOwnerName())
|
me.getOwnerName())
|
||||||
.setGroupName(me.getGroupName() == null ? "" :
|
.setGroupName(me.getGroupName() == null ? "" :
|
||||||
me.getGroupName())
|
me.getGroupName())
|
||||||
.addAllAcls(me.getAcls() == null ?
|
.addAllAcls(me.getAcls() == null ?
|
||||||
Lists.<AclEntryProto>newArrayList() :
|
Lists.<AclEntryProto>newArrayList() :
|
||||||
convertAclEntryProto(me.getAcls()))
|
convertAclEntryProto(me.getAcls()))
|
||||||
.addAllXAttrs(me.getxAttrs() == null ?
|
.addAllXAttrs(me.getxAttrs() == null ?
|
||||||
Lists.<XAttrProto>newArrayList() :
|
Lists.<XAttrProto>newArrayList() :
|
||||||
convertXAttrProto(me.getxAttrs()))
|
convertXAttrProto(me.getxAttrs()))
|
||||||
.setXAttrsRemoved(me.isxAttrsRemoved());
|
.setXAttrsRemoved(me.isxAttrsRemoved());
|
||||||
if (me.getPerms() != null) {
|
if (me.getPerms() != null) {
|
||||||
metaB.setPerms(convert(me.getPerms()));
|
metaB.setPerms(convert(me.getPerms()));
|
||||||
}
|
}
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_METADATA)
|
.setType(InotifyProtos.EventType.EVENT_METADATA)
|
||||||
.setContents(metaB.build().toByteString())
|
.setContents(metaB.build().toByteString())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case RENAME:
|
case RENAME:
|
||||||
Event.RenameEvent re = (Event.RenameEvent) e;
|
Event.RenameEvent re = (Event.RenameEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_RENAME)
|
.setType(InotifyProtos.EventType.EVENT_RENAME)
|
||||||
.setContents(
|
.setContents(
|
||||||
InotifyProtos.RenameEventProto.newBuilder()
|
InotifyProtos.RenameEventProto.newBuilder()
|
||||||
.setSrcPath(re.getSrcPath())
|
.setSrcPath(re.getSrcPath())
|
||||||
.setDestPath(re.getDstPath())
|
.setDestPath(re.getDstPath())
|
||||||
.setTimestamp(re.getTimestamp()).build().toByteString()
|
.setTimestamp(re.getTimestamp()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
case APPEND:
|
case APPEND:
|
||||||
Event.AppendEvent re2 = (Event.AppendEvent) e;
|
Event.AppendEvent re2 = (Event.AppendEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_APPEND)
|
.setType(InotifyProtos.EventType.EVENT_APPEND)
|
||||||
.setContents(InotifyProtos.AppendEventProto.newBuilder()
|
.setContents(InotifyProtos.AppendEventProto.newBuilder()
|
||||||
.setPath(re2.getPath())
|
.setPath(re2.getPath())
|
||||||
.setNewBlock(re2.toNewBlock()).build().toByteString())
|
.setNewBlock(re2.toNewBlock()).build().toByteString())
|
||||||
.build());
|
.build());
|
||||||
break;
|
break;
|
||||||
case UNLINK:
|
case UNLINK:
|
||||||
Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
|
Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_UNLINK)
|
.setType(InotifyProtos.EventType.EVENT_UNLINK)
|
||||||
.setContents(
|
.setContents(
|
||||||
InotifyProtos.UnlinkEventProto.newBuilder()
|
InotifyProtos.UnlinkEventProto.newBuilder()
|
||||||
.setPath(ue.getPath())
|
.setPath(ue.getPath())
|
||||||
.setTimestamp(ue.getTimestamp()).build().toByteString()
|
.setTimestamp(ue.getTimestamp()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
case TRUNCATE:
|
case TRUNCATE:
|
||||||
Event.TruncateEvent te = (Event.TruncateEvent) e;
|
Event.TruncateEvent te = (Event.TruncateEvent) e;
|
||||||
events.add(InotifyProtos.EventProto.newBuilder()
|
events.add(InotifyProtos.EventProto.newBuilder()
|
||||||
.setType(InotifyProtos.EventType.EVENT_TRUNCATE)
|
.setType(InotifyProtos.EventType.EVENT_TRUNCATE)
|
||||||
.setContents(
|
.setContents(
|
||||||
InotifyProtos.TruncateEventProto.newBuilder()
|
InotifyProtos.TruncateEventProto.newBuilder()
|
||||||
.setPath(te.getPath())
|
.setPath(te.getPath())
|
||||||
.setFileSize(te.getFileSize())
|
.setFileSize(te.getFileSize())
|
||||||
.setTimestamp(te.getTimestamp()).build().toByteString()
|
.setTimestamp(te.getTimestamp()).build().toByteString()
|
||||||
).build());
|
).build());
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new RuntimeException("Unexpected inotify event: " + e);
|
throw new RuntimeException("Unexpected inotify event: " + e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
|
builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
|
||||||
|
|
|
@ -34,7 +34,8 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdenti
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class DelegationTokenIdentifier
|
public class DelegationTokenIdentifier
|
||||||
extends AbstractDelegationTokenIdentifier {
|
extends AbstractDelegationTokenIdentifier {
|
||||||
public static final Text HDFS_DELEGATION_KIND = new Text("HDFS_DELEGATION_TOKEN");
|
public static final Text HDFS_DELEGATION_KIND =
|
||||||
|
new Text("HDFS_DELEGATION_TOKEN");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create an empty delegation token identifier for reading into.
|
* Create an empty delegation token identifier for reading into.
|
||||||
|
|
|
@ -137,7 +137,8 @@ public class BlockMetadataHeader {
|
||||||
* @return Metadata Header
|
* @return Metadata Header
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static BlockMetadataHeader readHeader(DataInputStream in) throws IOException {
|
public static BlockMetadataHeader readHeader(DataInputStream in)
|
||||||
|
throws IOException {
|
||||||
return readHeader(in.readShort(), in);
|
return readHeader(in.readShort(), in);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -163,7 +164,8 @@ public class BlockMetadataHeader {
|
||||||
* The current file position will be altered by this method.
|
* The current file position will be altered by this method.
|
||||||
* If an error occurs, the file is <em>not</em> closed.
|
* If an error occurs, the file is <em>not</em> closed.
|
||||||
*/
|
*/
|
||||||
public static BlockMetadataHeader readHeader(RandomAccessFile raf) throws IOException {
|
public static BlockMetadataHeader readHeader(RandomAccessFile raf)
|
||||||
|
throws IOException {
|
||||||
byte[] buf = new byte[getHeaderSize()];
|
byte[] buf = new byte[getHeaderSize()];
|
||||||
raf.seek(0);
|
raf.seek(0);
|
||||||
raf.readFully(buf, 0, buf.length);
|
raf.readFully(buf, 0, buf.length);
|
||||||
|
@ -171,8 +173,8 @@ public class BlockMetadataHeader {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Version is already read.
|
// Version is already read.
|
||||||
private static BlockMetadataHeader readHeader(short version, DataInputStream in)
|
private static BlockMetadataHeader readHeader(short version,
|
||||||
throws IOException {
|
DataInputStream in) throws IOException {
|
||||||
DataChecksum checksum = DataChecksum.newDataChecksum(in);
|
DataChecksum checksum = DataChecksum.newDataChecksum(in);
|
||||||
return new BlockMetadataHeader(version, checksum);
|
return new BlockMetadataHeader(version, checksum);
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,15 +29,17 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
*/
|
*/
|
||||||
public class ReplicaNotFoundException extends IOException {
|
public class ReplicaNotFoundException extends IOException {
|
||||||
private static final long serialVersionUID = 1L;
|
private static final long serialVersionUID = 1L;
|
||||||
public final static String NON_RBW_REPLICA = "Cannot recover a non-RBW replica ";
|
public final static String NON_RBW_REPLICA =
|
||||||
|
"Cannot recover a non-RBW replica ";
|
||||||
public final static String UNFINALIZED_REPLICA =
|
public final static String UNFINALIZED_REPLICA =
|
||||||
"Cannot append to an unfinalized replica ";
|
"Cannot append to an unfinalized replica ";
|
||||||
public final static String UNFINALIZED_AND_NONRBW_REPLICA =
|
public final static String UNFINALIZED_AND_NONRBW_REPLICA =
|
||||||
"Cannot recover append/close to a replica that's not FINALIZED and not RBW ";
|
"Cannot recover append/close to a replica that's not FINALIZED and not RBW"
|
||||||
|
+ " ";
|
||||||
public final static String NON_EXISTENT_REPLICA =
|
public final static String NON_EXISTENT_REPLICA =
|
||||||
"Cannot append to a non-existent replica ";
|
"Cannot append to a non-existent replica ";
|
||||||
public final static String UNEXPECTED_GS_REPLICA =
|
public final static String UNEXPECTED_GS_REPLICA =
|
||||||
"Cannot append to a replica with unexpected generation stamp ";
|
"Cannot append to a replica with unexpected generation stamp ";
|
||||||
|
|
||||||
public ReplicaNotFoundException() {
|
public ReplicaNotFoundException() {
|
||||||
super();
|
super();
|
||||||
|
|
|
@ -20,11 +20,10 @@ package org.apache.hadoop.hdfs.server.namenode.ha;
|
||||||
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.io.retry.FailoverProxyProvider;
|
import org.apache.hadoop.io.retry.FailoverProxyProvider;
|
||||||
|
|
||||||
public abstract class AbstractNNFailoverProxyProvider<T> implements
|
public abstract class AbstractNNFailoverProxyProvider<T> implements
|
||||||
FailoverProxyProvider <T> {
|
FailoverProxyProvider <T> {
|
||||||
|
|
||||||
private AtomicBoolean fallbackToSimpleAuth;
|
private AtomicBoolean fallbackToSimpleAuth;
|
||||||
|
|
||||||
|
|
|
@ -30,18 +30,20 @@ public class DatanodeStorage {
|
||||||
NORMAL,
|
NORMAL,
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A storage that represents a read-only path to replicas stored on a shared storage device.
|
* A storage that represents a read-only path to replicas stored on a shared
|
||||||
* Replicas on {@link #READ_ONLY_SHARED} storage are not counted towards live replicas.
|
* storage device. Replicas on {@link #READ_ONLY_SHARED} storage are not
|
||||||
|
* counted towards live replicas.
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* In certain implementations, a {@link #READ_ONLY_SHARED} storage may be correlated to
|
* In certain implementations, a {@link #READ_ONLY_SHARED} storage may be
|
||||||
* its {@link #NORMAL} counterpart using the {@link DatanodeStorage#storageID}. This
|
* correlated to its {@link #NORMAL} counterpart using the
|
||||||
* property should be used for debugging purposes only.
|
* {@link DatanodeStorage#storageID}. This property should be used for
|
||||||
|
* debugging purposes only.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
READ_ONLY_SHARED,
|
READ_ONLY_SHARED,
|
||||||
|
|
||||||
FAILED;
|
FAILED
|
||||||
}
|
}
|
||||||
|
|
||||||
private final String storageID;
|
private final String storageID;
|
||||||
|
@ -91,10 +93,9 @@ public class DatanodeStorage {
|
||||||
try {
|
try {
|
||||||
// Attempt to parse the UUID.
|
// Attempt to parse the UUID.
|
||||||
if (storageID != null && storageID.indexOf(STORAGE_ID_PREFIX) == 0) {
|
if (storageID != null && storageID.indexOf(STORAGE_ID_PREFIX) == 0) {
|
||||||
UUID.fromString(storageID.substring(STORAGE_ID_PREFIX.length()));
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} catch (IllegalArgumentException iae) {
|
} catch (IllegalArgumentException ignored) {
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
|
|
|
@ -52,13 +52,14 @@ import org.slf4j.LoggerFactory;
|
||||||
/**
|
/**
|
||||||
* Manages short-circuit memory segments for an HDFS client.
|
* Manages short-circuit memory segments for an HDFS client.
|
||||||
*
|
*
|
||||||
* Clients are responsible for requesting and releasing shared memory segments used
|
* Clients are responsible for requesting and releasing shared memory segments
|
||||||
* for communicating with the DataNode. The client will try to allocate new slots
|
* used for communicating with the DataNode. The client will try to allocate new
|
||||||
* in the set of existing segments, falling back to getting a new segment from the
|
* slots in the set of existing segments, falling back to getting a new segment
|
||||||
* DataNode via {@link DataTransferProtocol#requestShortCircuitFds}.
|
* from the DataNode via {@link DataTransferProtocol#requestShortCircuitFds}.
|
||||||
*
|
*
|
||||||
* The counterpart to this class on the DataNode is {@link ShortCircuitRegistry}.
|
* The counterpart to this class on the DataNode is
|
||||||
* See {@link ShortCircuitRegistry} for more information on the communication protocol.
|
* {@link ShortCircuitRegistry}. See {@link ShortCircuitRegistry} for more
|
||||||
|
* information on the communication protocol.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class DfsClientShmManager implements Closeable {
|
public class DfsClientShmManager implements Closeable {
|
||||||
|
@ -79,16 +80,14 @@ public class DfsClientShmManager implements Closeable {
|
||||||
*
|
*
|
||||||
* Protected by the manager lock.
|
* Protected by the manager lock.
|
||||||
*/
|
*/
|
||||||
private final TreeMap<ShmId, DfsClientShm> full =
|
private final TreeMap<ShmId, DfsClientShm> full = new TreeMap<>();
|
||||||
new TreeMap<ShmId, DfsClientShm>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shared memory segments which have at least one empty slot.
|
* Shared memory segments which have at least one empty slot.
|
||||||
*
|
*
|
||||||
* Protected by the manager lock.
|
* Protected by the manager lock.
|
||||||
*/
|
*/
|
||||||
private final TreeMap<ShmId, DfsClientShm> notFull =
|
private final TreeMap<ShmId, DfsClientShm> notFull = new TreeMap<>();
|
||||||
new TreeMap<ShmId, DfsClientShm>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* True if this datanode doesn't support short-circuit shared memory
|
* True if this datanode doesn't support short-circuit shared memory
|
||||||
|
@ -383,7 +382,7 @@ public class DfsClientShmManager implements Closeable {
|
||||||
* Information about each Datanode.
|
* Information about each Datanode.
|
||||||
*/
|
*/
|
||||||
private final HashMap<DatanodeInfo, EndpointShmManager> datanodes =
|
private final HashMap<DatanodeInfo, EndpointShmManager> datanodes =
|
||||||
new HashMap<DatanodeInfo, EndpointShmManager>(1);
|
new HashMap<>(1);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The DomainSocketWatcher which keeps track of the UNIX domain socket
|
* The DomainSocketWatcher which keeps track of the UNIX domain socket
|
||||||
|
@ -456,8 +455,7 @@ public class DfsClientShmManager implements Closeable {
|
||||||
public void visit(Visitor visitor) throws IOException {
|
public void visit(Visitor visitor) throws IOException {
|
||||||
lock.lock();
|
lock.lock();
|
||||||
try {
|
try {
|
||||||
HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info =
|
HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info = new HashMap<>();
|
||||||
new HashMap<DatanodeInfo, PerDatanodeVisitorInfo>();
|
|
||||||
for (Entry<DatanodeInfo, EndpointShmManager> entry :
|
for (Entry<DatanodeInfo, EndpointShmManager> entry :
|
||||||
datanodes.entrySet()) {
|
datanodes.entrySet()) {
|
||||||
info.put(entry.getKey(), entry.getValue().getVisitorInfo());
|
info.put(entry.getKey(), entry.getValue().getVisitorInfo());
|
||||||
|
|
|
@ -85,8 +85,7 @@ public class DomainSocketFactory {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return new StringBuilder().append("PathInfo{path=").append(path).
|
return "PathInfo{path=" + path + ", state=" + state + "}";
|
||||||
append(", state=").append(state).append("}").toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -107,7 +107,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
|
|
||||||
int numDemoted = demoteOldEvictableMmaped(curMs);
|
int numDemoted = demoteOldEvictableMmaped(curMs);
|
||||||
int numPurged = 0;
|
int numPurged = 0;
|
||||||
Long evictionTimeNs = Long.valueOf(0);
|
Long evictionTimeNs = (long) 0;
|
||||||
while (true) {
|
while (true) {
|
||||||
Entry<Long, ShortCircuitReplica> entry =
|
Entry<Long, ShortCircuitReplica> entry =
|
||||||
evictable.ceilingEntry(evictionTimeNs);
|
evictable.ceilingEntry(evictionTimeNs);
|
||||||
|
@ -119,14 +119,14 @@ public class ShortCircuitCache implements Closeable {
|
||||||
ShortCircuitReplica replica = entry.getValue();
|
ShortCircuitReplica replica = entry.getValue();
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
LOG.trace("CacheCleaner: purging " + replica + ": " +
|
LOG.trace("CacheCleaner: purging " + replica + ": " +
|
||||||
StringUtils.getStackTrace(Thread.currentThread()));
|
StringUtils.getStackTrace(Thread.currentThread()));
|
||||||
}
|
}
|
||||||
purge(replica);
|
purge(replica);
|
||||||
numPurged++;
|
numPurged++;
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG.debug("{}: finishing cache cleaner run started at {}. Demoted {} "
|
LOG.debug("{}: finishing cache cleaner run started at {}. Demoted {} "
|
||||||
+ "mmapped replicas; purged {} replicas.",
|
+ "mmapped replicas; purged {} replicas.",
|
||||||
this, curMs, numDemoted, numPurged);
|
this, curMs, numDemoted, numPurged);
|
||||||
} finally {
|
} finally {
|
||||||
ShortCircuitCache.this.lock.unlock();
|
ShortCircuitCache.this.lock.unlock();
|
||||||
|
@ -236,17 +236,17 @@ public class ShortCircuitCache implements Closeable {
|
||||||
* The executor service that runs the cacheCleaner.
|
* The executor service that runs the cacheCleaner.
|
||||||
*/
|
*/
|
||||||
private final ScheduledThreadPoolExecutor cleanerExecutor
|
private final ScheduledThreadPoolExecutor cleanerExecutor
|
||||||
= new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().
|
= new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().
|
||||||
setDaemon(true).setNameFormat("ShortCircuitCache_Cleaner").
|
setDaemon(true).setNameFormat("ShortCircuitCache_Cleaner").
|
||||||
build());
|
build());
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The executor service that runs the cacheCleaner.
|
* The executor service that runs the cacheCleaner.
|
||||||
*/
|
*/
|
||||||
private final ScheduledThreadPoolExecutor releaserExecutor
|
private final ScheduledThreadPoolExecutor releaserExecutor
|
||||||
= new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().
|
= new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().
|
||||||
setDaemon(true).setNameFormat("ShortCircuitCache_SlotReleaser").
|
setDaemon(true).setNameFormat("ShortCircuitCache_SlotReleaser").
|
||||||
build());
|
build());
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A map containing all ShortCircuitReplicaInfo objects, organized by Key.
|
* A map containing all ShortCircuitReplicaInfo objects, organized by Key.
|
||||||
|
@ -254,8 +254,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
* exception.
|
* exception.
|
||||||
*/
|
*/
|
||||||
private final HashMap<ExtendedBlockId, Waitable<ShortCircuitReplicaInfo>>
|
private final HashMap<ExtendedBlockId, Waitable<ShortCircuitReplicaInfo>>
|
||||||
replicaInfoMap = new HashMap<ExtendedBlockId,
|
replicaInfoMap = new HashMap<>();
|
||||||
Waitable<ShortCircuitReplicaInfo>>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The CacheCleaner. We don't create this and schedule it until it becomes
|
* The CacheCleaner. We don't create this and schedule it until it becomes
|
||||||
|
@ -268,8 +267,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
*
|
*
|
||||||
* Maps (unique) insertion time in nanoseconds to the element.
|
* Maps (unique) insertion time in nanoseconds to the element.
|
||||||
*/
|
*/
|
||||||
private final TreeMap<Long, ShortCircuitReplica> evictable =
|
private final TreeMap<Long, ShortCircuitReplica> evictable = new TreeMap<>();
|
||||||
new TreeMap<Long, ShortCircuitReplica>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum total size of the cache, including both mmapped and
|
* Maximum total size of the cache, including both mmapped and
|
||||||
|
@ -288,7 +286,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
* Maps (unique) insertion time in nanoseconds to the element.
|
* Maps (unique) insertion time in nanoseconds to the element.
|
||||||
*/
|
*/
|
||||||
private final TreeMap<Long, ShortCircuitReplica> evictableMmapped =
|
private final TreeMap<Long, ShortCircuitReplica> evictableMmapped =
|
||||||
new TreeMap<Long, ShortCircuitReplica>();
|
new TreeMap<>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum number of mmaped evictable elements.
|
* Maximum number of mmaped evictable elements.
|
||||||
|
@ -435,13 +433,13 @@ public class ShortCircuitCache implements Closeable {
|
||||||
if (newRefCount == 0) {
|
if (newRefCount == 0) {
|
||||||
// Close replica, since there are no remaining references to it.
|
// Close replica, since there are no remaining references to it.
|
||||||
Preconditions.checkArgument(replica.purged,
|
Preconditions.checkArgument(replica.purged,
|
||||||
"Replica %s reached a refCount of 0 without being purged", replica);
|
"Replica %s reached a refCount of 0 without being purged", replica);
|
||||||
replica.close();
|
replica.close();
|
||||||
} else if (newRefCount == 1) {
|
} else if (newRefCount == 1) {
|
||||||
Preconditions.checkState(null == replica.getEvictableTimeNs(),
|
Preconditions.checkState(null == replica.getEvictableTimeNs(),
|
||||||
"Replica %s had a refCount higher than 1, " +
|
"Replica %s had a refCount higher than 1, " +
|
||||||
"but was still evictable (evictableTimeNs = %d)",
|
"but was still evictable (evictableTimeNs = %d)",
|
||||||
replica, replica.getEvictableTimeNs());
|
replica, replica.getEvictableTimeNs());
|
||||||
if (!replica.purged) {
|
if (!replica.purged) {
|
||||||
// Add the replica to the end of an eviction list.
|
// Add the replica to the end of an eviction list.
|
||||||
// Eviction lists are sorted by time.
|
// Eviction lists are sorted by time.
|
||||||
|
@ -457,7 +455,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
} else {
|
} else {
|
||||||
Preconditions.checkArgument(replica.refCount >= 0,
|
Preconditions.checkArgument(replica.refCount >= 0,
|
||||||
"replica's refCount went negative (refCount = %d" +
|
"replica's refCount went negative (refCount = %d" +
|
||||||
" for %s)", replica.refCount, replica);
|
" for %s)", replica.refCount, replica);
|
||||||
}
|
}
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
LOG.trace(this + ": unref replica " + replica +
|
LOG.trace(this + ": unref replica " + replica +
|
||||||
|
@ -484,7 +482,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
private int demoteOldEvictableMmaped(long now) {
|
private int demoteOldEvictableMmaped(long now) {
|
||||||
int numDemoted = 0;
|
int numDemoted = 0;
|
||||||
boolean needMoreSpace = false;
|
boolean needMoreSpace = false;
|
||||||
Long evictionTimeNs = Long.valueOf(0);
|
Long evictionTimeNs = (long) 0;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
Entry<Long, ShortCircuitReplica> entry =
|
Entry<Long, ShortCircuitReplica> entry =
|
||||||
|
@ -530,13 +528,13 @@ public class ShortCircuitCache implements Closeable {
|
||||||
}
|
}
|
||||||
ShortCircuitReplica replica;
|
ShortCircuitReplica replica;
|
||||||
if (evictableSize == 0) {
|
if (evictableSize == 0) {
|
||||||
replica = evictableMmapped.firstEntry().getValue();
|
replica = evictableMmapped.firstEntry().getValue();
|
||||||
} else {
|
} else {
|
||||||
replica = evictable.firstEntry().getValue();
|
replica = evictable.firstEntry().getValue();
|
||||||
}
|
}
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
LOG.trace(this + ": trimEvictionMaps is purging " + replica +
|
LOG.trace(this + ": trimEvictionMaps is purging " + replica +
|
||||||
StringUtils.getStackTrace(Thread.currentThread()));
|
StringUtils.getStackTrace(Thread.currentThread()));
|
||||||
}
|
}
|
||||||
purge(replica);
|
purge(replica);
|
||||||
}
|
}
|
||||||
|
@ -677,13 +675,12 @@ public class ShortCircuitCache implements Closeable {
|
||||||
info = fetch(key, waitable);
|
info = fetch(key, waitable);
|
||||||
} catch (RetriableException e) {
|
} catch (RetriableException e) {
|
||||||
LOG.debug("{}: retrying {}", this, e.getMessage());
|
LOG.debug("{}: retrying {}", this, e.getMessage());
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} while (false);
|
} while (false);
|
||||||
if (info != null) return info;
|
if (info != null) return info;
|
||||||
// We need to load the replica ourselves.
|
// We need to load the replica ourselves.
|
||||||
newWaitable = new Waitable<ShortCircuitReplicaInfo>(lock.newCondition());
|
newWaitable = new Waitable<>(lock.newCondition());
|
||||||
replicaInfoMap.put(key, newWaitable);
|
replicaInfoMap.put(key, newWaitable);
|
||||||
} finally {
|
} finally {
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
|
@ -716,7 +713,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
}
|
}
|
||||||
if (info.getInvalidTokenException() != null) {
|
if (info.getInvalidTokenException() != null) {
|
||||||
LOG.info(this + ": could not get " + key + " due to InvalidToken " +
|
LOG.info(this + ": could not get " + key + " due to InvalidToken " +
|
||||||
"exception.", info.getInvalidTokenException());
|
"exception.", info.getInvalidTokenException());
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
ShortCircuitReplica replica = info.getReplica();
|
ShortCircuitReplica replica = info.getReplica();
|
||||||
|
@ -833,7 +830,7 @@ public class ShortCircuitCache implements Closeable {
|
||||||
lock.lock();
|
lock.lock();
|
||||||
try {
|
try {
|
||||||
if (map == null) {
|
if (map == null) {
|
||||||
replica.mmapData = Long.valueOf(Time.monotonicNow());
|
replica.mmapData = Time.monotonicNow();
|
||||||
newCond.signalAll();
|
newCond.signalAll();
|
||||||
return null;
|
return null;
|
||||||
} else {
|
} else {
|
||||||
|
@ -920,12 +917,10 @@ public class ShortCircuitCache implements Closeable {
|
||||||
public void accept(CacheVisitor visitor) {
|
public void accept(CacheVisitor visitor) {
|
||||||
lock.lock();
|
lock.lock();
|
||||||
try {
|
try {
|
||||||
Map<ExtendedBlockId, ShortCircuitReplica> replicas =
|
Map<ExtendedBlockId, ShortCircuitReplica> replicas = new HashMap<>();
|
||||||
new HashMap<ExtendedBlockId, ShortCircuitReplica>();
|
Map<ExtendedBlockId, InvalidToken> failedLoads = new HashMap<>();
|
||||||
Map<ExtendedBlockId, InvalidToken> failedLoads =
|
|
||||||
new HashMap<ExtendedBlockId, InvalidToken>();
|
|
||||||
for (Entry<ExtendedBlockId, Waitable<ShortCircuitReplicaInfo>> entry :
|
for (Entry<ExtendedBlockId, Waitable<ShortCircuitReplicaInfo>> entry :
|
||||||
replicaInfoMap.entrySet()) {
|
replicaInfoMap.entrySet()) {
|
||||||
Waitable<ShortCircuitReplicaInfo> waitable = entry.getValue();
|
Waitable<ShortCircuitReplicaInfo> waitable = entry.getValue();
|
||||||
if (waitable.hasVal()) {
|
if (waitable.hasVal()) {
|
||||||
if (waitable.getVal().getReplica() != null) {
|
if (waitable.getVal().getReplica() != null) {
|
||||||
|
@ -939,11 +934,11 @@ public class ShortCircuitCache implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
LOG.debug("visiting {} with outstandingMmapCount={}, replicas={}, "
|
LOG.debug("visiting {} with outstandingMmapCount={}, replicas={}, "
|
||||||
+ "failedLoads={}, evictable={}, evictableMmapped={}",
|
+ "failedLoads={}, evictable={}, evictableMmapped={}",
|
||||||
visitor.getClass().getName(), outstandingMmapCount, replicas,
|
visitor.getClass().getName(), outstandingMmapCount, replicas,
|
||||||
failedLoads, evictable, evictableMmapped);
|
failedLoads, evictable, evictableMmapped);
|
||||||
visitor.visit(outstandingMmapCount, replicas, failedLoads,
|
visitor.visit(outstandingMmapCount, replicas, failedLoads,
|
||||||
evictable, evictableMmapped);
|
evictable, evictableMmapped);
|
||||||
} finally {
|
} finally {
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
}
|
}
|
||||||
|
@ -971,8 +966,8 @@ public class ShortCircuitCache implements Closeable {
|
||||||
* the datanode.
|
* the datanode.
|
||||||
*/
|
*/
|
||||||
public Slot allocShmSlot(DatanodeInfo datanode,
|
public Slot allocShmSlot(DatanodeInfo datanode,
|
||||||
DomainPeer peer, MutableBoolean usedPeer,
|
DomainPeer peer, MutableBoolean usedPeer,
|
||||||
ExtendedBlockId blockId, String clientName) throws IOException {
|
ExtendedBlockId blockId, String clientName) throws IOException {
|
||||||
if (shmManager != null) {
|
if (shmManager != null) {
|
||||||
return shmManager.allocSlot(datanode, peer, usedPeer,
|
return shmManager.allocSlot(datanode, peer, usedPeer,
|
||||||
blockId, clientName);
|
blockId, clientName);
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.hdfs.ExtendedBlockId;
|
||||||
import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
|
import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
|
||||||
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
|
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
|
||||||
import org.apache.hadoop.hdfs.util.IOUtilsClient;
|
import org.apache.hadoop.hdfs.util.IOUtilsClient;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
|
||||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||||
import org.apache.hadoop.util.Time;
|
import org.apache.hadoop.util.Time;
|
||||||
|
|
||||||
|
@ -325,13 +324,10 @@ public class ShortCircuitReplica {
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return new StringBuilder().append("ShortCircuitReplica{").
|
return "ShortCircuitReplica{" + "key=" + key
|
||||||
append("key=").append(key).
|
+ ", metaHeader.version=" + metaHeader.getVersion()
|
||||||
append(", metaHeader.version=").append(metaHeader.getVersion()).
|
+ ", metaHeader.checksum=" + metaHeader.getChecksum()
|
||||||
append(", metaHeader.checksum=").append(metaHeader.getChecksum()).
|
+ ", ident=" + "0x" + Integer.toHexString(System.identityHashCode(this))
|
||||||
append(", ident=").append("0x").
|
+ ", creationTimeMs=" + creationTimeMs + "}";
|
||||||
append(Integer.toHexString(System.identityHashCode(this))).
|
|
||||||
append(", creationTimeMs=").append(creationTimeMs).
|
|
||||||
append("}").toString();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,6 @@ public final class ShortCircuitReplicaInfo {
|
||||||
}
|
}
|
||||||
if (exc != null) {
|
if (exc != null) {
|
||||||
builder.append(prefix).append(exc);
|
builder.append(prefix).append(exc);
|
||||||
prefix = ", ";
|
|
||||||
}
|
}
|
||||||
builder.append("}");
|
builder.append("}");
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
|
|
|
@ -43,11 +43,14 @@ import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.ComparisonChain;
|
import com.google.common.collect.ComparisonChain;
|
||||||
import com.google.common.primitives.Ints;
|
import com.google.common.primitives.Ints;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A shared memory segment used to implement short-circuit reads.
|
* A shared memory segment used to implement short-circuit reads.
|
||||||
*/
|
*/
|
||||||
public class ShortCircuitShm {
|
public class ShortCircuitShm {
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(ShortCircuitShm.class);
|
private static final Logger LOG = LoggerFactory.getLogger(
|
||||||
|
ShortCircuitShm.class);
|
||||||
|
|
||||||
protected static final int BYTES_PER_SLOT = 64;
|
protected static final int BYTES_PER_SLOT = 64;
|
||||||
|
|
||||||
|
@ -140,13 +143,13 @@ public class ShortCircuitShm {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int compareTo(ShmId other) {
|
public int compareTo(@Nonnull ShmId other) {
|
||||||
return ComparisonChain.start().
|
return ComparisonChain.start().
|
||||||
compare(hi, other.hi).
|
compare(hi, other.hi).
|
||||||
compare(lo, other.lo).
|
compare(lo, other.lo).
|
||||||
result();
|
result();
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Uniquely identifies a slot.
|
* Uniquely identifies a slot.
|
||||||
|
@ -367,11 +370,8 @@ public class ShortCircuitShm {
|
||||||
|
|
||||||
public boolean isAnchored() {
|
public boolean isAnchored() {
|
||||||
long prev = unsafe.getLongVolatile(null, this.slotAddress);
|
long prev = unsafe.getLongVolatile(null, this.slotAddress);
|
||||||
if ((prev & VALID_FLAG) == 0) {
|
// Slot is no longer valid.
|
||||||
// Slot is no longer valid.
|
return (prev & VALID_FLAG) != 0 && ((prev & 0x7fffffff) != 0);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return ((prev & 0x7fffffff) != 0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -517,7 +517,7 @@ public class ShortCircuitShm {
|
||||||
* @param slotIdx Index of the slot.
|
* @param slotIdx Index of the slot.
|
||||||
* @return The base address of the slot.
|
* @return The base address of the slot.
|
||||||
*/
|
*/
|
||||||
private final long calculateSlotAddress(int slotIdx) {
|
private long calculateSlotAddress(int slotIdx) {
|
||||||
long offset = slotIdx;
|
long offset = slotIdx;
|
||||||
offset *= BYTES_PER_SLOT;
|
offset *= BYTES_PER_SLOT;
|
||||||
return this.baseAddress + offset;
|
return this.baseAddress + offset;
|
||||||
|
|
|
@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An InputStream implementations which reads from some other InputStream
|
* An InputStream implementations which reads from some other InputStream
|
||||||
* but expects an exact number of bytes. Any attempts to read past the
|
* but expects an exact number of bytes. Any attempts to read past the
|
||||||
|
@ -80,7 +82,7 @@ public class ExactSizeInputStream extends FilterInputStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int read(final byte[] b, final int off, int len)
|
public int read(@Nonnull final byte[] b, final int off, int len)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (remaining <= 0) {
|
if (remaining <= 0) {
|
||||||
return -1;
|
return -1;
|
||||||
|
|
|
@ -38,7 +38,8 @@ public class LongBitFormat implements Serializable {
|
||||||
/** Bit mask */
|
/** Bit mask */
|
||||||
private final long MASK;
|
private final long MASK;
|
||||||
|
|
||||||
public LongBitFormat(String name, LongBitFormat previous, int length, long min) {
|
public LongBitFormat(String name, LongBitFormat previous, int length,
|
||||||
|
long min) {
|
||||||
NAME = name;
|
NAME = name;
|
||||||
OFFSET = previous == null? 0: previous.OFFSET + previous.LENGTH;
|
OFFSET = previous == null? 0: previous.OFFSET + previous.LENGTH;
|
||||||
LENGTH = length;
|
LENGTH = length;
|
||||||
|
|
|
@ -34,6 +34,8 @@ import org.apache.http.HttpStatus;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.net.HttpHeaders;
|
import com.google.common.net.HttpHeaders;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* To support HTTP byte streams, a new connection to an HTTP server needs to be
|
* To support HTTP byte streams, a new connection to an HTTP server needs to be
|
||||||
* created each time. This class hides the complexity of those multiple
|
* created each time. This class hides the complexity of those multiple
|
||||||
|
@ -102,24 +104,24 @@ public abstract class ByteRangeInputStream extends FSInputStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract URL getResolvedUrl(final HttpURLConnection connection
|
protected abstract URL getResolvedUrl(final HttpURLConnection connection
|
||||||
) throws IOException;
|
) throws IOException;
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected InputStream getInputStream() throws IOException {
|
protected InputStream getInputStream() throws IOException {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case NORMAL:
|
case NORMAL:
|
||||||
break;
|
break;
|
||||||
case SEEK:
|
case SEEK:
|
||||||
if (in != null) {
|
if (in != null) {
|
||||||
in.close();
|
in.close();
|
||||||
}
|
}
|
||||||
InputStreamAndFileLength fin = openInputStream(startPos);
|
InputStreamAndFileLength fin = openInputStream(startPos);
|
||||||
in = fin.in;
|
in = fin.in;
|
||||||
fileLength = fin.length;
|
fileLength = fin.length;
|
||||||
status = StreamStatus.NORMAL;
|
status = StreamStatus.NORMAL;
|
||||||
break;
|
break;
|
||||||
case CLOSED:
|
case CLOSED:
|
||||||
throw new IOException("Stream closed");
|
throw new IOException("Stream closed");
|
||||||
}
|
}
|
||||||
return in;
|
return in;
|
||||||
}
|
}
|
||||||
|
@ -225,7 +227,7 @@ public abstract class ByteRangeInputStream extends FSInputStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int read(byte b[], int off, int len) throws IOException {
|
public int read(@Nonnull byte b[], int off, int len) throws IOException {
|
||||||
return update(getInputStream().read(b, off, len));
|
return update(getInputStream().read(b, off, len));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -60,7 +60,8 @@ class JsonUtilClient {
|
||||||
|
|
||||||
/** Convert a Json map to a RemoteException. */
|
/** Convert a Json map to a RemoteException. */
|
||||||
static RemoteException toRemoteException(final Map<?, ?> json) {
|
static RemoteException toRemoteException(final Map<?, ?> json) {
|
||||||
final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
|
final Map<?, ?> m = (Map<?, ?>)json.get(
|
||||||
|
RemoteException.class.getSimpleName());
|
||||||
final String message = (String)m.get("message");
|
final String message = (String)m.get("message");
|
||||||
final String javaClassName = (String)m.get("javaClassName");
|
final String javaClassName = (String)m.get("javaClassName");
|
||||||
return new RemoteException(javaClassName, message);
|
return new RemoteException(javaClassName, message);
|
||||||
|
@ -100,7 +101,8 @@ class JsonUtilClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Convert a Json map to a HdfsFileStatus object. */
|
/** Convert a Json map to a HdfsFileStatus object. */
|
||||||
static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
|
static HdfsFileStatus toFileStatus(final Map<?, ?> json,
|
||||||
|
boolean includesType) {
|
||||||
if (json == null) {
|
if (json == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -108,7 +110,8 @@ class JsonUtilClient {
|
||||||
final Map<?, ?> m = includesType ?
|
final Map<?, ?> m = includesType ?
|
||||||
(Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
|
(Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
|
||||||
final String localName = (String) m.get("pathSuffix");
|
final String localName = (String) m.get("pathSuffix");
|
||||||
final WebHdfsConstants.PathType type = WebHdfsConstants.PathType.valueOf((String) m.get("type"));
|
final WebHdfsConstants.PathType type =
|
||||||
|
WebHdfsConstants.PathType.valueOf((String) m.get("type"));
|
||||||
final byte[] symlink = type != WebHdfsConstants.PathType.SYMLINK? null
|
final byte[] symlink = type != WebHdfsConstants.PathType.SYMLINK? null
|
||||||
: DFSUtilClient.string2Bytes((String) m.get("symlink"));
|
: DFSUtilClient.string2Bytes((String) m.get("symlink"));
|
||||||
|
|
||||||
|
@ -116,20 +119,21 @@ class JsonUtilClient {
|
||||||
final String owner = (String) m.get("owner");
|
final String owner = (String) m.get("owner");
|
||||||
final String group = (String) m.get("group");
|
final String group = (String) m.get("group");
|
||||||
final FsPermission permission = toFsPermission((String) m.get("permission"),
|
final FsPermission permission = toFsPermission((String) m.get("permission"),
|
||||||
(Boolean) m.get("aclBit"),
|
(Boolean) m.get("aclBit"),
|
||||||
(Boolean) m.get("encBit"));
|
(Boolean) m.get("encBit"));
|
||||||
final long aTime = ((Number) m.get("accessTime")).longValue();
|
final long aTime = ((Number) m.get("accessTime")).longValue();
|
||||||
final long mTime = ((Number) m.get("modificationTime")).longValue();
|
final long mTime = ((Number) m.get("modificationTime")).longValue();
|
||||||
final long blockSize = ((Number) m.get("blockSize")).longValue();
|
final long blockSize = ((Number) m.get("blockSize")).longValue();
|
||||||
final short replication = ((Number) m.get("replication")).shortValue();
|
final short replication = ((Number) m.get("replication")).shortValue();
|
||||||
final long fileId = m.containsKey("fileId") ?
|
final long fileId = m.containsKey("fileId") ?
|
||||||
((Number) m.get("fileId")).longValue() : HdfsConstants.GRANDFATHER_INODE_ID;
|
((Number) m.get("fileId")).longValue() :
|
||||||
|
HdfsConstants.GRANDFATHER_INODE_ID;
|
||||||
final int childrenNum = getInt(m, "childrenNum", -1);
|
final int childrenNum = getInt(m, "childrenNum", -1);
|
||||||
final byte storagePolicy = m.containsKey("storagePolicy") ?
|
final byte storagePolicy = m.containsKey("storagePolicy") ?
|
||||||
(byte) ((Number) m.get("storagePolicy")).longValue() :
|
(byte) ((Number) m.get("storagePolicy")).longValue() :
|
||||||
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
|
||||||
return new HdfsFileStatus(len, type == WebHdfsConstants.PathType.DIRECTORY, replication,
|
return new HdfsFileStatus(len, type == WebHdfsConstants.PathType.DIRECTORY,
|
||||||
blockSize, mTime, aTime, permission, owner, group,
|
replication, blockSize, mTime, aTime, permission, owner, group,
|
||||||
symlink, DFSUtilClient.string2Bytes(localName),
|
symlink, DFSUtilClient.string2Bytes(localName),
|
||||||
fileId, childrenNum, null,
|
fileId, childrenNum, null,
|
||||||
storagePolicy);
|
storagePolicy);
|
||||||
|
@ -185,7 +189,7 @@ class JsonUtilClient {
|
||||||
|
|
||||||
/** Convert a Json map to an DatanodeInfo object. */
|
/** Convert a Json map to an DatanodeInfo object. */
|
||||||
static DatanodeInfo toDatanodeInfo(final Map<?, ?> m)
|
static DatanodeInfo toDatanodeInfo(final Map<?, ?> m)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (m == null) {
|
if (m == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -304,7 +308,8 @@ class JsonUtilClient {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
|
final Map<?, ?> m = (Map<?, ?>)json.get(
|
||||||
|
ContentSummary.class.getSimpleName());
|
||||||
final long length = ((Number) m.get("length")).longValue();
|
final long length = ((Number) m.get("length")).longValue();
|
||||||
final long fileCount = ((Number) m.get("fileCount")).longValue();
|
final long fileCount = ((Number) m.get("fileCount")).longValue();
|
||||||
final long directoryCount = ((Number) m.get("directoryCount")).longValue();
|
final long directoryCount = ((Number) m.get("directoryCount")).longValue();
|
||||||
|
@ -312,9 +317,13 @@ class JsonUtilClient {
|
||||||
final long spaceConsumed = ((Number) m.get("spaceConsumed")).longValue();
|
final long spaceConsumed = ((Number) m.get("spaceConsumed")).longValue();
|
||||||
final long spaceQuota = ((Number) m.get("spaceQuota")).longValue();
|
final long spaceQuota = ((Number) m.get("spaceQuota")).longValue();
|
||||||
|
|
||||||
return new ContentSummary.Builder().length(length).fileCount(fileCount).
|
return new ContentSummary.Builder()
|
||||||
directoryCount(directoryCount).quota(quota).spaceConsumed(spaceConsumed).
|
.length(length)
|
||||||
spaceQuota(spaceQuota).build();
|
.fileCount(fileCount)
|
||||||
|
.directoryCount(directoryCount)
|
||||||
|
.quota(quota)
|
||||||
|
.spaceConsumed(spaceConsumed)
|
||||||
|
.spaceQuota(spaceQuota).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Convert a Json map to a MD5MD5CRC32FileChecksum. */
|
/** Convert a Json map to a MD5MD5CRC32FileChecksum. */
|
||||||
|
@ -329,21 +338,22 @@ class JsonUtilClient {
|
||||||
final int length = ((Number) m.get("length")).intValue();
|
final int length = ((Number) m.get("length")).intValue();
|
||||||
final byte[] bytes = StringUtils.hexStringToByte((String) m.get("bytes"));
|
final byte[] bytes = StringUtils.hexStringToByte((String) m.get("bytes"));
|
||||||
|
|
||||||
final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
|
final DataInputStream in = new DataInputStream(
|
||||||
|
new ByteArrayInputStream(bytes));
|
||||||
final DataChecksum.Type crcType =
|
final DataChecksum.Type crcType =
|
||||||
MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
|
MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
|
||||||
final MD5MD5CRC32FileChecksum checksum;
|
final MD5MD5CRC32FileChecksum checksum;
|
||||||
|
|
||||||
// Recreate what DFSClient would have returned.
|
// Recreate what DFSClient would have returned.
|
||||||
switch(crcType) {
|
switch(crcType) {
|
||||||
case CRC32:
|
case CRC32:
|
||||||
checksum = new MD5MD5CRC32GzipFileChecksum();
|
checksum = new MD5MD5CRC32GzipFileChecksum();
|
||||||
break;
|
break;
|
||||||
case CRC32C:
|
case CRC32C:
|
||||||
checksum = new MD5MD5CRC32CastagnoliFileChecksum();
|
checksum = new MD5MD5CRC32CastagnoliFileChecksum();
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new IOException("Unknown algorithm: " + algorithm);
|
throw new IOException("Unknown algorithm: " + algorithm);
|
||||||
}
|
}
|
||||||
checksum.readFields(in);
|
checksum.readFields(in);
|
||||||
|
|
||||||
|
@ -390,14 +400,12 @@ class JsonUtilClient {
|
||||||
return aclStatusBuilder.build();
|
return aclStatusBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
static String getPath(final Map<?, ?> json)
|
static String getPath(final Map<?, ?> json) {
|
||||||
throws IOException {
|
|
||||||
if (json == null) {
|
if (json == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
String path = (String) json.get("Path");
|
return (String) json.get("Path");
|
||||||
return path;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static byte[] getXAttr(final Map<?, ?> json, final String name)
|
static byte[] getXAttr(final Map<?, ?> json, final String name)
|
||||||
|
@ -446,7 +454,7 @@ class JsonUtilClient {
|
||||||
ObjectReader reader = new ObjectMapper().reader(List.class);
|
ObjectReader reader = new ObjectMapper().reader(List.class);
|
||||||
final List<Object> xattrs = reader.readValue(namesInJson);
|
final List<Object> xattrs = reader.readValue(namesInJson);
|
||||||
final List<String> names =
|
final List<String> names =
|
||||||
Lists.newArrayListWithCapacity(json.keySet().size());
|
Lists.newArrayListWithCapacity(json.keySet().size());
|
||||||
|
|
||||||
for (Object xattr : xattrs) {
|
for (Object xattr : xattrs) {
|
||||||
names.add((String) xattr);
|
names.add((String) xattr);
|
||||||
|
@ -495,7 +503,8 @@ class JsonUtilClient {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
|
final Map<?, ?> m = (Map<?, ?>)json.get(
|
||||||
|
LocatedBlocks.class.getSimpleName());
|
||||||
final long fileLength = ((Number) m.get("fileLength")).longValue();
|
final long fileLength = ((Number) m.get("fileLength")).longValue();
|
||||||
final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
|
final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
|
||||||
final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
|
final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
|
||||||
|
|
|
@ -54,38 +54,40 @@ public class URLConnectionFactory {
|
||||||
/**
|
/**
|
||||||
* Timeout for socket connects and reads
|
* Timeout for socket connects and reads
|
||||||
*/
|
*/
|
||||||
public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
|
public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
|
||||||
private final ConnectionConfigurator connConfigurator;
|
private final ConnectionConfigurator connConfigurator;
|
||||||
|
|
||||||
private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR = new ConnectionConfigurator() {
|
private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR
|
||||||
@Override
|
= new ConnectionConfigurator() {
|
||||||
public HttpURLConnection configure(HttpURLConnection conn)
|
@Override
|
||||||
throws IOException {
|
public HttpURLConnection configure(HttpURLConnection conn)
|
||||||
URLConnectionFactory.setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
|
throws IOException {
|
||||||
return conn;
|
URLConnectionFactory.setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
|
||||||
}
|
return conn;
|
||||||
};
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The URLConnectionFactory that sets the default timeout and it only trusts
|
* The URLConnectionFactory that sets the default timeout and it only trusts
|
||||||
* Java's SSL certificates.
|
* Java's SSL certificates.
|
||||||
*/
|
*/
|
||||||
public static final URLConnectionFactory DEFAULT_SYSTEM_CONNECTION_FACTORY = new URLConnectionFactory(
|
public static final URLConnectionFactory DEFAULT_SYSTEM_CONNECTION_FACTORY =
|
||||||
DEFAULT_TIMEOUT_CONN_CONFIGURATOR);
|
new URLConnectionFactory(DEFAULT_TIMEOUT_CONN_CONFIGURATOR);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Construct a new URLConnectionFactory based on the configuration. It will
|
* Construct a new URLConnectionFactory based on the configuration. It will
|
||||||
* try to load SSL certificates when it is specified.
|
* try to load SSL certificates when it is specified.
|
||||||
*/
|
*/
|
||||||
public static URLConnectionFactory newDefaultURLConnectionFactory(Configuration conf) {
|
public static URLConnectionFactory newDefaultURLConnectionFactory(
|
||||||
|
Configuration conf) {
|
||||||
ConnectionConfigurator conn = getSSLConnectionConfiguration(conf);
|
ConnectionConfigurator conn = getSSLConnectionConfiguration(conf);
|
||||||
|
|
||||||
return new URLConnectionFactory(conn);
|
return new URLConnectionFactory(conn);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ConnectionConfigurator
|
private static ConnectionConfigurator getSSLConnectionConfiguration(
|
||||||
getSSLConnectionConfiguration(Configuration conf) {
|
Configuration conf) {
|
||||||
ConnectionConfigurator conn = null;
|
ConnectionConfigurator conn;
|
||||||
try {
|
try {
|
||||||
conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
|
conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -103,9 +105,9 @@ public class URLConnectionFactory {
|
||||||
* Construct a new URLConnectionFactory that supports OAut-based connections.
|
* Construct a new URLConnectionFactory that supports OAut-based connections.
|
||||||
* It will also try to load the SSL configuration when they are specified.
|
* It will also try to load the SSL configuration when they are specified.
|
||||||
*/
|
*/
|
||||||
public static URLConnectionFactory
|
public static URLConnectionFactory newOAuth2URLConnectionFactory(
|
||||||
newOAuth2URLConnectionFactory(Configuration conf) throws IOException {
|
Configuration conf) throws IOException {
|
||||||
ConnectionConfigurator conn = null;
|
ConnectionConfigurator conn;
|
||||||
try {
|
try {
|
||||||
ConnectionConfigurator sslConnConfigurator
|
ConnectionConfigurator sslConnConfigurator
|
||||||
= newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
|
= newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
|
||||||
|
@ -125,8 +127,9 @@ public class URLConnectionFactory {
|
||||||
/**
|
/**
|
||||||
* Create a new ConnectionConfigurator for SSL connections
|
* Create a new ConnectionConfigurator for SSL connections
|
||||||
*/
|
*/
|
||||||
private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
|
private static ConnectionConfigurator newSslConnConfigurator(
|
||||||
Configuration conf) throws IOException, GeneralSecurityException {
|
final int timeout, Configuration conf)
|
||||||
|
throws IOException, GeneralSecurityException {
|
||||||
final SSLFactory factory;
|
final SSLFactory factory;
|
||||||
final SSLSocketFactory sf;
|
final SSLSocketFactory sf;
|
||||||
final HostnameVerifier hv;
|
final HostnameVerifier hv;
|
||||||
|
|
|
@ -89,15 +89,20 @@ import com.google.common.collect.Lists;
|
||||||
|
|
||||||
/** A FileSystem for HDFS over the web. */
|
/** A FileSystem for HDFS over the web. */
|
||||||
public class WebHdfsFileSystem extends FileSystem
|
public class WebHdfsFileSystem extends FileSystem
|
||||||
implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
|
implements DelegationTokenRenewer.Renewable,
|
||||||
|
TokenAspect.TokenManagementDelegator {
|
||||||
public static final Logger LOG = LoggerFactory
|
public static final Logger LOG = LoggerFactory
|
||||||
.getLogger(WebHdfsFileSystem.class);
|
.getLogger(WebHdfsFileSystem.class);
|
||||||
/** WebHdfs version. */
|
/** WebHdfs version. */
|
||||||
public static final int VERSION = 1;
|
public static final int VERSION = 1;
|
||||||
/** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
|
/** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
|
||||||
public static final String PATH_PREFIX = "/" + WebHdfsConstants.WEBHDFS_SCHEME + "/v" + VERSION;
|
public static final String PATH_PREFIX = "/" + WebHdfsConstants.WEBHDFS_SCHEME
|
||||||
|
+ "/v" + VERSION;
|
||||||
|
|
||||||
/** Default connection factory may be overridden in tests to use smaller timeout values */
|
/**
|
||||||
|
* Default connection factory may be overridden in tests to use smaller
|
||||||
|
* timeout values
|
||||||
|
*/
|
||||||
protected URLConnectionFactory connectionFactory;
|
protected URLConnectionFactory connectionFactory;
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
|
@ -141,7 +146,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void initialize(URI uri, Configuration conf
|
public synchronized void initialize(URI uri, Configuration conf
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
super.initialize(uri, conf);
|
super.initialize(uri, conf);
|
||||||
setConf(conf);
|
setConf(conf);
|
||||||
/** set user pattern based on configuration file */
|
/** set user pattern based on configuration file */
|
||||||
|
@ -206,7 +211,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
failoverSleepMaxMillis);
|
failoverSleepMaxMillis);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.workingDir = makeQualified(new Path(getHomeDirectoryString(ugi)));
|
this.workingDir = makeQualified(getHomeDirectory());
|
||||||
this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
|
this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
|
||||||
this.disallowFallbackToInsecureCluster = !conf.getBoolean(
|
this.disallowFallbackToInsecureCluster = !conf.getBoolean(
|
||||||
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
|
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
|
||||||
|
@ -326,7 +331,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
String result = absolutePath.toUri().getPath();
|
String result = absolutePath.toUri().getPath();
|
||||||
if (!DFSUtilClient.isValidName(result)) {
|
if (!DFSUtilClient.isValidName(result)) {
|
||||||
throw new IllegalArgumentException("Invalid DFS directory name " +
|
throw new IllegalArgumentException("Invalid DFS directory name " +
|
||||||
result);
|
result);
|
||||||
}
|
}
|
||||||
workingDir = absolutePath;
|
workingDir = absolutePath;
|
||||||
}
|
}
|
||||||
|
@ -335,14 +340,16 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
return f.isAbsolute()? f: new Path(workingDir, f);
|
return f.isAbsolute()? f: new Path(workingDir, f);
|
||||||
}
|
}
|
||||||
|
|
||||||
static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
|
static Map<?, ?> jsonParse(final HttpURLConnection c,
|
||||||
) throws IOException {
|
final boolean useErrorStream) throws IOException {
|
||||||
if (c.getContentLength() == 0) {
|
if (c.getContentLength() == 0) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
|
final InputStream in = useErrorStream ?
|
||||||
|
c.getErrorStream() : c.getInputStream();
|
||||||
if (in == null) {
|
if (in == null) {
|
||||||
throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
|
throw new IOException("The " + (useErrorStream? "error": "input") +
|
||||||
|
" stream is null.");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
final String contentType = c.getContentType();
|
final String contentType = c.getContentType();
|
||||||
|
@ -362,7 +369,8 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
|
private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
|
||||||
final HttpURLConnection conn, boolean unwrapException) throws IOException {
|
final HttpURLConnection conn, boolean unwrapException)
|
||||||
|
throws IOException {
|
||||||
final int code = conn.getResponseCode();
|
final int code = conn.getResponseCode();
|
||||||
// server is demanding an authentication we don't support
|
// server is demanding an authentication we don't support
|
||||||
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
|
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
|
||||||
|
@ -444,7 +452,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
private URL getNamenodeURL(String path, String query) throws IOException {
|
private URL getNamenodeURL(String path, String query) throws IOException {
|
||||||
InetSocketAddress nnAddr = getCurrentNNAddr();
|
InetSocketAddress nnAddr = getCurrentNNAddr();
|
||||||
final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
|
final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
|
||||||
nnAddr.getPort(), path + '?' + query);
|
nnAddr.getPort(), path + '?' + query);
|
||||||
LOG.trace("url={}", url);
|
LOG.trace("url={}", url);
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
@ -493,7 +501,8 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
protected final HttpOpParam.Op op;
|
protected final HttpOpParam.Op op;
|
||||||
private final boolean redirected;
|
private final boolean redirected;
|
||||||
protected ExcludeDatanodesParam excludeDatanodes = new ExcludeDatanodesParam("");
|
protected ExcludeDatanodesParam excludeDatanodes =
|
||||||
|
new ExcludeDatanodesParam("");
|
||||||
|
|
||||||
private boolean checkRetry;
|
private boolean checkRetry;
|
||||||
|
|
||||||
|
@ -530,7 +539,8 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
*
|
*
|
||||||
* Create/Append:
|
* Create/Append:
|
||||||
* Step 1) Submit a Http request with neither auto-redirect nor data.
|
* Step 1) Submit a Http request with neither auto-redirect nor data.
|
||||||
* Step 2) Submit another Http request with the URL from the Location header with data.
|
* Step 2) Submit another Http request with the URL from the Location header
|
||||||
|
* with data.
|
||||||
*
|
*
|
||||||
* The reason of having two-step create/append is for preventing clients to
|
* The reason of having two-step create/append is for preventing clients to
|
||||||
* send out the data before the redirect. This issue is addressed by the
|
* send out the data before the redirect. This issue is addressed by the
|
||||||
|
@ -589,26 +599,25 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
conn.setRequestMethod(op.getType().toString());
|
conn.setRequestMethod(op.getType().toString());
|
||||||
conn.setInstanceFollowRedirects(false);
|
conn.setInstanceFollowRedirects(false);
|
||||||
switch (op.getType()) {
|
switch (op.getType()) {
|
||||||
// if not sending a message body for a POST or PUT operation, need
|
// if not sending a message body for a POST or PUT operation, need
|
||||||
// to ensure the server/proxy knows this
|
// to ensure the server/proxy knows this
|
||||||
case POST:
|
case POST:
|
||||||
case PUT: {
|
case PUT: {
|
||||||
conn.setDoOutput(true);
|
conn.setDoOutput(true);
|
||||||
if (!doOutput) {
|
if (!doOutput) {
|
||||||
// explicitly setting content-length to 0 won't do spnego!!
|
// explicitly setting content-length to 0 won't do spnego!!
|
||||||
// opening and closing the stream will send "Content-Length: 0"
|
// opening and closing the stream will send "Content-Length: 0"
|
||||||
conn.getOutputStream().close();
|
conn.getOutputStream().close();
|
||||||
} else {
|
} else {
|
||||||
conn.setRequestProperty("Content-Type",
|
conn.setRequestProperty("Content-Type",
|
||||||
MediaType.APPLICATION_OCTET_STREAM);
|
MediaType.APPLICATION_OCTET_STREAM);
|
||||||
conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
|
conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default: {
|
|
||||||
conn.setDoOutput(doOutput);
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
conn.setDoOutput(doOutput);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
conn.connect();
|
conn.connect();
|
||||||
return conn;
|
return conn;
|
||||||
|
@ -658,21 +667,22 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
}
|
}
|
||||||
|
|
||||||
private void shouldRetry(final IOException ioe, final int retry
|
private void shouldRetry(final IOException ioe, final int retry
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
InetSocketAddress nnAddr = getCurrentNNAddr();
|
InetSocketAddress nnAddr = getCurrentNNAddr();
|
||||||
if (checkRetry) {
|
if (checkRetry) {
|
||||||
try {
|
try {
|
||||||
final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
|
final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
|
||||||
ioe, retry, 0, true);
|
ioe, retry, 0, true);
|
||||||
|
|
||||||
boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
|
boolean isRetry =
|
||||||
|
a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
|
||||||
boolean isFailoverAndRetry =
|
boolean isFailoverAndRetry =
|
||||||
a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
|
a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
|
||||||
|
|
||||||
if (isRetry || isFailoverAndRetry) {
|
if (isRetry || isFailoverAndRetry) {
|
||||||
LOG.info("Retrying connect to namenode: {}. Already tried {}"
|
LOG.info("Retrying connect to namenode: {}. Already tried {}"
|
||||||
+ " time(s); retry policy is {}, delay {}ms.", nnAddr, retry,
|
+ " time(s); retry policy is {}, delay {}ms.",
|
||||||
retryPolicy, a.delayMillis);
|
nnAddr, retry, retryPolicy, a.delayMillis);
|
||||||
|
|
||||||
if (isFailoverAndRetry) {
|
if (isFailoverAndRetry) {
|
||||||
resetStateToFailOver();
|
resetStateToFailOver();
|
||||||
|
@ -795,7 +805,8 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
/**
|
/**
|
||||||
* Handle create/append output streams
|
* Handle create/append output streams
|
||||||
*/
|
*/
|
||||||
class FsPathOutputStreamRunner extends AbstractFsPathRunner<FSDataOutputStream> {
|
class FsPathOutputStreamRunner
|
||||||
|
extends AbstractFsPathRunner<FSDataOutputStream> {
|
||||||
private final int bufferSize;
|
private final int bufferSize;
|
||||||
|
|
||||||
FsPathOutputStreamRunner(Op op, Path fspath, int bufferSize,
|
FsPathOutputStreamRunner(Op op, Path fspath, int bufferSize,
|
||||||
|
@ -846,7 +857,8 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected URLRunner(final HttpOpParam.Op op, final URL url, boolean redirected) {
|
protected URLRunner(final HttpOpParam.Op op, final URL url,
|
||||||
|
boolean redirected) {
|
||||||
super(op, redirected);
|
super(op, redirected);
|
||||||
this.url = url;
|
this.url = url;
|
||||||
}
|
}
|
||||||
|
@ -920,7 +932,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
* Create a symlink pointing to the destination path.
|
* Create a symlink pointing to the destination path.
|
||||||
*/
|
*/
|
||||||
public void createSymlink(Path destination, Path f, boolean createParent
|
public void createSymlink(Path destination, Path f, boolean createParent
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
|
final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
|
||||||
new FsPathRunner(op, f,
|
new FsPathRunner(op, f,
|
||||||
|
@ -1029,7 +1041,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setOwner(final Path p, final String owner, final String group
|
public void setOwner(final Path p, final String owner, final String group
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
if (owner == null && group == null) {
|
if (owner == null && group == null) {
|
||||||
throw new IOException("owner == null && group == null");
|
throw new IOException("owner == null && group == null");
|
||||||
}
|
}
|
||||||
|
@ -1043,7 +1055,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setPermission(final Path p, final FsPermission permission
|
public void setPermission(final Path p, final FsPermission permission
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
|
final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
|
||||||
new FsPathRunner(op, p,new PermissionParam(permission)).run();
|
new FsPathRunner(op, p,new PermissionParam(permission)).run();
|
||||||
|
@ -1092,14 +1104,13 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
throws IOException {
|
throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.CREATESNAPSHOT;
|
final HttpOpParam.Op op = PutOpParam.Op.CREATESNAPSHOT;
|
||||||
Path spath = new FsPathResponseRunner<Path>(op, path,
|
return new FsPathResponseRunner<Path>(op, path,
|
||||||
new SnapshotNameParam(snapshotName)) {
|
new SnapshotNameParam(snapshotName)) {
|
||||||
@Override
|
@Override
|
||||||
Path decodeResponse(Map<?,?> json) {
|
Path decodeResponse(Map<?,?> json) {
|
||||||
return new Path((String) json.get(Path.class.getSimpleName()));
|
return new Path((String) json.get(Path.class.getSimpleName()));
|
||||||
}
|
}
|
||||||
}.run();
|
}.run();
|
||||||
return spath;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1121,7 +1132,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean setReplication(final Path p, final short replication
|
public boolean setReplication(final Path p, final short replication
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
|
final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
|
||||||
return new FsPathBooleanRunner(op, p,
|
return new FsPathBooleanRunner(op, p,
|
||||||
|
@ -1131,7 +1142,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setTimes(final Path p, final long mtime, final long atime
|
public void setTimes(final Path p, final long mtime, final long atime
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementWriteOps(1);
|
statistics.incrementWriteOps(1);
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
|
final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
|
||||||
new FsPathRunner(op, p,
|
new FsPathRunner(op, p,
|
||||||
|
@ -1222,7 +1233,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public FSDataInputStream open(final Path f, final int buffersize
|
public FSDataInputStream open(final Path f, final int buffersize
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementReadOps(1);
|
statistics.incrementReadOps(1);
|
||||||
final HttpOpParam.Op op = GetOpParam.Op.OPEN;
|
final HttpOpParam.Op op = GetOpParam.Op.OPEN;
|
||||||
// use a runner so the open can recover from an invalid token
|
// use a runner so the open can recover from an invalid token
|
||||||
|
@ -1320,7 +1331,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
/** Remove offset parameter before returning the resolved url. */
|
/** Remove offset parameter before returning the resolved url. */
|
||||||
@Override
|
@Override
|
||||||
protected URL getResolvedUrl(final HttpURLConnection connection
|
protected URL getResolvedUrl(final HttpURLConnection connection
|
||||||
) throws MalformedURLException {
|
) throws MalformedURLException {
|
||||||
return removeOffsetParam(connection.getURL());
|
return removeOffsetParam(connection.getURL());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1333,16 +1344,19 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
return new FsPathResponseRunner<FileStatus[]>(op, f) {
|
return new FsPathResponseRunner<FileStatus[]>(op, f) {
|
||||||
@Override
|
@Override
|
||||||
FileStatus[] decodeResponse(Map<?,?> json) {
|
FileStatus[] decodeResponse(Map<?,?> json) {
|
||||||
final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
|
final Map<?, ?> rootmap =
|
||||||
|
(Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
|
||||||
final List<?> array = JsonUtilClient.getList(rootmap,
|
final List<?> array = JsonUtilClient.getList(rootmap,
|
||||||
FileStatus.class.getSimpleName());
|
FileStatus.class.getSimpleName());
|
||||||
|
|
||||||
//convert FileStatus
|
//convert FileStatus
|
||||||
|
assert array != null;
|
||||||
final FileStatus[] statuses = new FileStatus[array.size()];
|
final FileStatus[] statuses = new FileStatus[array.size()];
|
||||||
int i = 0;
|
int i = 0;
|
||||||
for (Object object : array) {
|
for (Object object : array) {
|
||||||
final Map<?, ?> m = (Map<?, ?>) object;
|
final Map<?, ?> m = (Map<?, ?>) object;
|
||||||
statuses[i++] = makeQualified(JsonUtilClient.toFileStatus(m, false), f);
|
statuses[i++] = makeQualified(JsonUtilClient.toFileStatus(m, false),
|
||||||
|
f);
|
||||||
}
|
}
|
||||||
return statuses;
|
return statuses;
|
||||||
}
|
}
|
||||||
|
@ -1356,12 +1370,12 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
Token<DelegationTokenIdentifier> token =
|
Token<DelegationTokenIdentifier> token =
|
||||||
new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
|
new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
|
||||||
op, null, new RenewerParam(renewer)) {
|
op, null, new RenewerParam(renewer)) {
|
||||||
@Override
|
@Override
|
||||||
Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
|
Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return JsonUtilClient.toDelegationToken(json);
|
return JsonUtilClient.toDelegationToken(json);
|
||||||
}
|
}
|
||||||
}.run();
|
}.run();
|
||||||
if (token != null) {
|
if (token != null) {
|
||||||
token.setService(tokenServiceName);
|
token.setService(tokenServiceName);
|
||||||
} else {
|
} else {
|
||||||
|
@ -1387,7 +1401,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized long renewDelegationToken(final Token<?> token
|
public synchronized long renewDelegationToken(final Token<?> token
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
|
final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
|
||||||
return new FsPathResponseRunner<Long>(op, null,
|
return new FsPathResponseRunner<Long>(op, null,
|
||||||
new TokenArgumentParam(token.encodeToUrlString())) {
|
new TokenArgumentParam(token.encodeToUrlString())) {
|
||||||
|
@ -1400,7 +1414,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void cancelDelegationToken(final Token<?> token
|
public synchronized void cancelDelegationToken(final Token<?> token
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
|
final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
|
||||||
new FsPathRunner(op, null,
|
new FsPathRunner(op, null,
|
||||||
new TokenArgumentParam(token.encodeToUrlString())
|
new TokenArgumentParam(token.encodeToUrlString())
|
||||||
|
@ -1453,7 +1467,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
|
public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
|
||||||
) throws IOException {
|
) throws IOException {
|
||||||
statistics.incrementReadOps(1);
|
statistics.incrementReadOps(1);
|
||||||
|
|
||||||
final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
|
final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
|
||||||
|
@ -1471,11 +1485,11 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
* an HA cluster with its logical name, the resolver further resolves the
|
* an HA cluster with its logical name, the resolver further resolves the
|
||||||
* logical name(i.e., the authority in the URL) into real namenode addresses.
|
* logical name(i.e., the authority in the URL) into real namenode addresses.
|
||||||
*/
|
*/
|
||||||
private InetSocketAddress[] resolveNNAddr() throws IOException {
|
private InetSocketAddress[] resolveNNAddr() {
|
||||||
Configuration conf = getConf();
|
Configuration conf = getConf();
|
||||||
final String scheme = uri.getScheme();
|
final String scheme = uri.getScheme();
|
||||||
|
|
||||||
ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>();
|
ArrayList<InetSocketAddress> ret = new ArrayList<>();
|
||||||
|
|
||||||
if (!HAUtilClient.isLogicalUri(conf, uri)) {
|
if (!HAUtilClient.isLogicalUri(conf, uri)) {
|
||||||
InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(),
|
InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(),
|
||||||
|
|
|
@ -59,7 +59,6 @@ public class ConcatSourcesParam extends StringParam {
|
||||||
|
|
||||||
/** @return the absolute path. */
|
/** @return the absolute path. */
|
||||||
public final String[] getAbsolutePaths() {
|
public final String[] getAbsolutePaths() {
|
||||||
final String[] paths = getValue().split(",");
|
return getValue().split(",");
|
||||||
return paths;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class CreateFlagParam extends EnumSetParam<CreateFlag> {
|
||||||
|
|
||||||
public static final String DEFAULT = "";
|
public static final String DEFAULT = "";
|
||||||
|
|
||||||
private static final Domain<CreateFlag> DOMAIN = new Domain<CreateFlag>(
|
private static final Domain<CreateFlag> DOMAIN = new Domain<>(
|
||||||
NAME, CreateFlag.class);
|
NAME, CreateFlag.class);
|
||||||
|
|
||||||
public CreateFlagParam(final EnumSet<CreateFlag> createFlags) {
|
public CreateFlagParam(final EnumSet<CreateFlag> createFlags) {
|
||||||
|
|
|
@ -22,7 +22,7 @@ import java.net.HttpURLConnection;
|
||||||
/** Http DELETE operation parameter. */
|
/** Http DELETE operation parameter. */
|
||||||
public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
|
public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
|
||||||
/** Delete operations. */
|
/** Delete operations. */
|
||||||
public static enum Op implements HttpOpParam.Op {
|
public enum Op implements HttpOpParam.Op {
|
||||||
DELETE(HttpURLConnection.HTTP_OK),
|
DELETE(HttpURLConnection.HTTP_OK),
|
||||||
DELETESNAPSHOT(HttpURLConnection.HTTP_OK),
|
DELETESNAPSHOT(HttpURLConnection.HTTP_OK),
|
||||||
|
|
||||||
|
@ -65,7 +65,7 @@ public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
|
private static final Domain<Op> DOMAIN = new Domain<>(NAME, Op.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor.
|
* Constructor.
|
||||||
|
|
|
@ -20,7 +20,8 @@ package org.apache.hadoop.hdfs.web.resources;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
|
||||||
abstract class EnumParam<E extends Enum<E>> extends Param<E, EnumParam.Domain<E>> {
|
abstract class EnumParam<E extends Enum<E>>
|
||||||
|
extends Param<E, EnumParam.Domain<E>> {
|
||||||
EnumParam(final Domain<E> domain, final E value) {
|
EnumParam(final Domain<E> domain, final E value) {
|
||||||
super(domain, value);
|
super(domain, value);
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,8 @@ import java.util.EnumSet;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
|
||||||
abstract class EnumSetParam<E extends Enum<E>> extends Param<EnumSet<E>, EnumSetParam.Domain<E>> {
|
abstract class EnumSetParam<E extends Enum<E>>
|
||||||
|
extends Param<EnumSet<E>, EnumSetParam.Domain<E>> {
|
||||||
/** Convert an EnumSet to a string of comma separated values. */
|
/** Convert an EnumSet to a string of comma separated values. */
|
||||||
static <E extends Enum<E>> String toString(EnumSet<E> set) {
|
static <E extends Enum<E>> String toString(EnumSet<E> set) {
|
||||||
if (set == null || set.isEmpty()) {
|
if (set == null || set.isEmpty()) {
|
||||||
|
|
|
@ -22,7 +22,7 @@ import java.net.HttpURLConnection;
|
||||||
/** Http GET operation parameter. */
|
/** Http GET operation parameter. */
|
||||||
public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
|
public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
|
||||||
/** Get operations. */
|
/** Get operations. */
|
||||||
public static enum Op implements HttpOpParam.Op {
|
public enum Op implements HttpOpParam.Op {
|
||||||
OPEN(true, HttpURLConnection.HTTP_OK),
|
OPEN(true, HttpURLConnection.HTTP_OK),
|
||||||
|
|
||||||
GETFILESTATUS(false, HttpURLConnection.HTTP_OK),
|
GETFILESTATUS(false, HttpURLConnection.HTTP_OK),
|
||||||
|
@ -52,7 +52,7 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
|
||||||
}
|
}
|
||||||
|
|
||||||
Op(final boolean redirect, final int expectedHttpResponseCode,
|
Op(final boolean redirect, final int expectedHttpResponseCode,
|
||||||
final boolean requireAuth) {
|
final boolean requireAuth) {
|
||||||
this.redirect = redirect;
|
this.redirect = redirect;
|
||||||
this.expectedHttpResponseCode = expectedHttpResponseCode;
|
this.expectedHttpResponseCode = expectedHttpResponseCode;
|
||||||
this.requireAuth = requireAuth;
|
this.requireAuth = requireAuth;
|
||||||
|
@ -89,7 +89,7 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
|
private static final Domain<Op> DOMAIN = new Domain<>(NAME, Op.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor.
|
* Constructor.
|
||||||
|
|
|
@ -34,29 +34,29 @@ public abstract class HttpOpParam<E extends Enum<E> & HttpOpParam.Op>
|
||||||
public static final String DEFAULT = NULL;
|
public static final String DEFAULT = NULL;
|
||||||
|
|
||||||
/** Http operation types */
|
/** Http operation types */
|
||||||
public static enum Type {
|
public enum Type {
|
||||||
GET, PUT, POST, DELETE;
|
GET, PUT, POST, DELETE
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Http operation interface. */
|
/** Http operation interface. */
|
||||||
public static interface Op {
|
public interface Op {
|
||||||
/** @return the Http operation type. */
|
/** @return the Http operation type. */
|
||||||
public Type getType();
|
Type getType();
|
||||||
|
|
||||||
/** @return true if the operation cannot use a token */
|
/** @return true if the operation cannot use a token */
|
||||||
public boolean getRequireAuth();
|
boolean getRequireAuth();
|
||||||
|
|
||||||
/** @return true if the operation will do output. */
|
/** @return true if the operation will do output. */
|
||||||
public boolean getDoOutput();
|
boolean getDoOutput();
|
||||||
|
|
||||||
/** @return true if the operation will be redirected. */
|
/** @return true if the operation will be redirected. */
|
||||||
public boolean getRedirect();
|
boolean getRedirect();
|
||||||
|
|
||||||
/** @return true the expected http response code. */
|
/** @return true the expected http response code. */
|
||||||
public int getExpectedHttpResponseCode();
|
int getExpectedHttpResponseCode();
|
||||||
|
|
||||||
/** @return a URI query string. */
|
/** @return a URI query string. */
|
||||||
public String toQueryString();
|
String toQueryString();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Expects HTTP response 307 "Temporary Redirect". */
|
/** Expects HTTP response 307 "Temporary Redirect". */
|
||||||
|
|
|
@ -19,7 +19,8 @@ package org.apache.hadoop.hdfs.web.resources;
|
||||||
|
|
||||||
/** Long parameter. */
|
/** Long parameter. */
|
||||||
abstract class LongParam extends Param<Long, LongParam.Domain> {
|
abstract class LongParam extends Param<Long, LongParam.Domain> {
|
||||||
LongParam(final Domain domain, final Long value, final Long min, final Long max) {
|
LongParam(final Domain domain, final Long value, final Long min,
|
||||||
|
final Long max) {
|
||||||
super(domain, value);
|
super(domain, value);
|
||||||
checkRange(min, max);
|
checkRange(min, max);
|
||||||
}
|
}
|
||||||
|
|
|
@ -47,16 +47,16 @@ public abstract class Param<T, D extends Param.Domain<T>> {
|
||||||
try {
|
try {
|
||||||
for(Param<?, ?> p : parameters) {
|
for(Param<?, ?> p : parameters) {
|
||||||
if (p.getValue() != null) {
|
if (p.getValue() != null) {
|
||||||
b.append(separator).append(
|
b.append(separator)
|
||||||
URLEncoder.encode(p.getName(), "UTF-8")
|
.append(URLEncoder.encode(p.getName(), "UTF-8"))
|
||||||
+ "="
|
.append("=")
|
||||||
+ URLEncoder.encode(p.getValueString(), "UTF-8"));
|
.append(URLEncoder.encode(p.getValueString(), "UTF-8"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (UnsupportedEncodingException e) {
|
} catch (UnsupportedEncodingException e) {
|
||||||
// Sane systems know about UTF-8, so this should never happen.
|
// Sane systems know about UTF-8, so this should never happen.
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return b.toString();
|
return b.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ import java.net.HttpURLConnection;
|
||||||
/** Http POST operation parameter. */
|
/** Http POST operation parameter. */
|
||||||
public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
|
public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
|
||||||
/** Post operations. */
|
/** Post operations. */
|
||||||
public static enum Op implements HttpOpParam.Op {
|
public enum Op implements HttpOpParam.Op {
|
||||||
APPEND(true, HttpURLConnection.HTTP_OK),
|
APPEND(true, HttpURLConnection.HTTP_OK),
|
||||||
|
|
||||||
CONCAT(false, HttpURLConnection.HTTP_OK),
|
CONCAT(false, HttpURLConnection.HTTP_OK),
|
||||||
|
@ -71,7 +71,7 @@ public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Domain<Op> DOMAIN = new Domain<PostOpParam.Op>(NAME, Op.class);
|
private static final Domain<Op> DOMAIN = new Domain<>(NAME, Op.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor.
|
* Constructor.
|
||||||
|
|
|
@ -22,7 +22,7 @@ import java.net.HttpURLConnection;
|
||||||
/** Http POST operation parameter. */
|
/** Http POST operation parameter. */
|
||||||
public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
|
public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
|
||||||
/** Put operations. */
|
/** Put operations. */
|
||||||
public static enum Op implements HttpOpParam.Op {
|
public enum Op implements HttpOpParam.Op {
|
||||||
CREATE(true, HttpURLConnection.HTTP_CREATED),
|
CREATE(true, HttpURLConnection.HTTP_CREATED),
|
||||||
|
|
||||||
MKDIRS(false, HttpURLConnection.HTTP_OK),
|
MKDIRS(false, HttpURLConnection.HTTP_OK),
|
||||||
|
@ -60,7 +60,7 @@ public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
|
||||||
}
|
}
|
||||||
|
|
||||||
Op(final boolean doOutputAndRedirect, final int expectedHttpResponseCode,
|
Op(final boolean doOutputAndRedirect, final int expectedHttpResponseCode,
|
||||||
final boolean requireAuth) {
|
final boolean requireAuth) {
|
||||||
this.doOutputAndRedirect = doOutputAndRedirect;
|
this.doOutputAndRedirect = doOutputAndRedirect;
|
||||||
this.expectedHttpResponseCode = expectedHttpResponseCode;
|
this.expectedHttpResponseCode = expectedHttpResponseCode;
|
||||||
this.requireAuth = requireAuth;
|
this.requireAuth = requireAuth;
|
||||||
|
@ -97,7 +97,7 @@ public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
|
private static final Domain<Op> DOMAIN = new Domain<>(NAME, Op.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor.
|
* Constructor.
|
||||||
|
|
|
@ -26,7 +26,7 @@ public class RenameOptionSetParam extends EnumSetParam<Options.Rename> {
|
||||||
/** Default parameter value. */
|
/** Default parameter value. */
|
||||||
public static final String DEFAULT = "";
|
public static final String DEFAULT = "";
|
||||||
|
|
||||||
private static final Domain<Options.Rename> DOMAIN = new Domain<Options.Rename>(
|
private static final Domain<Options.Rename> DOMAIN = new Domain<>(
|
||||||
NAME, Options.Rename.class);
|
NAME, Options.Rename.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue