HADOOP-10628. Javadoc and few code style improvement for Crypto input and output streams. (yliu via clamb)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1598429 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Charles Lamb 2014-05-29 22:09:51 +00:00
parent b20180ffa6
commit 2e5ae1aad7
11 changed files with 194 additions and 203 deletions

View File

@ -11,6 +11,9 @@ fs-encryption (Unreleased)
HADOOP-10603. Crypto input and output streams implementing Hadoop stream HADOOP-10603. Crypto input and output streams implementing Hadoop stream
interfaces. (Yi Liu and Charles Lamb) interfaces. (Yi Liu and Charles Lamb)
HADOOP-10628. Javadoc and few code style improvement for Crypto
input and output streams. (yliu via clamb)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -40,15 +40,15 @@ public abstract class AESCTRCryptoCodec extends CryptoCodec {
} }
/** /**
* IV is produced by combining initial IV and the counter using addition. * The IV is produced by adding the initial IV to the counter. IV length
* IV length should be the same as {@link #AES_BLOCK_SIZE} * should be the same as {@link #AES_BLOCK_SIZE}
*/ */
@Override @Override
public void calculateIV(byte[] initIV, long counter, byte[] IV) { public void calculateIV(byte[] initIV, long counter, byte[] IV) {
Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE); Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE);
Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE); Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE);
ByteBuffer buf = ByteBuffer.wrap(IV); final ByteBuffer buf = ByteBuffer.wrap(IV);
buf.put(initIV); buf.put(initIV);
buf.order(ByteOrder.BIG_ENDIAN); buf.order(ByteOrder.BIG_ENDIAN);
counter += buf.getLong(AES_BLOCK_SIZE - 8); counter += buf.getLong(AES_BLOCK_SIZE - 8);

View File

@ -41,37 +41,37 @@ public abstract class CryptoCodec implements Configurable {
} }
/** /**
* Get block size of a block cipher. * Get the block size of a block cipher.
* For different algorithms, the block size may be different. * For different algorithms, the block size may be different.
* @return int block size * @return int the block size
*/ */
public abstract int getAlgorithmBlockSize(); public abstract int getAlgorithmBlockSize();
/** /**
* Get a {@link #org.apache.hadoop.crypto.Encryptor}. * Get an {@link #org.apache.hadoop.crypto.Encryptor}.
* @return Encryptor * @return Encryptor the encryptor
*/ */
public abstract Encryptor getEncryptor() throws GeneralSecurityException; public abstract Encryptor getEncryptor() throws GeneralSecurityException;
/** /**
* Get a {@link #org.apache.hadoop.crypto.Decryptor}. * Get a {@link #org.apache.hadoop.crypto.Decryptor}.
* @return Decryptor * @return Decryptor the decryptor
*/ */
public abstract Decryptor getDecryptor() throws GeneralSecurityException; public abstract Decryptor getDecryptor() throws GeneralSecurityException;
/** /**
* This interface is only for Counter (CTR) mode. Typically calculating * This interface is only for Counter (CTR) mode. Generally the Encryptor
* IV(Initialization Vector) is up to Encryptor or Decryptor, for * or Decryptor calculates the IV and maintain encryption context internally.
* example {@link #javax.crypto.Cipher} will maintain encryption context * For example a {@link #javax.crypto.Cipher} will maintain its encryption
* internally when do encryption/decryption continuously using its * context internally when we do encryption/decryption using the
* Cipher#update interface. * Cipher#update interface.
* <p/> * <p/>
* In Hadoop, multiple nodes may read splits of a file, so decrypting of * Encryption/Decryption is not always on the entire file. For example,
* file is not continuous, even for encrypting may be not continuous. For * in Hadoop, a node may only decrypt a portion of a file (i.e. a split).
* each part, we need to calculate the counter through file position. * In these situations, the counter is derived from the file position.
* <p/> * <p/>
* Typically IV for a file position is produced by combining initial IV and * The IV can be calculated by combining the initial IV and the counter with
* the counter using any lossless operation (concatenation, addition, or XOR). * a lossless operation (concatenation, addition, or XOR).
* @see http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29 * @see http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29
* *
* @param initIV initial IV * @param initIV initial IV

View File

@ -63,26 +63,30 @@ public class CryptoInputStream extends FilterInputStream implements
private static final byte[] oneByteBuf = new byte[1]; private static final byte[] oneByteBuf = new byte[1];
private final CryptoCodec codec; private final CryptoCodec codec;
private final Decryptor decryptor; private final Decryptor decryptor;
/** /**
* Input data buffer. The data starts at inBuffer.position() and ends at * Input data buffer. The data starts at inBuffer.position() and ends at
* to inBuffer.limit(). * to inBuffer.limit().
*/ */
private ByteBuffer inBuffer; private ByteBuffer inBuffer;
/** /**
* The decrypted data buffer. The data starts at outBuffer.position() and * The decrypted data buffer. The data starts at outBuffer.position() and
* ends at outBuffer.limit(); * ends at outBuffer.limit();
*/ */
private ByteBuffer outBuffer; private ByteBuffer outBuffer;
private long streamOffset = 0; // Underlying stream offset. private long streamOffset = 0; // Underlying stream offset.
/** /**
* Whether underlying stream supports * Whether the underlying stream supports
* {@link #org.apache.hadoop.fs.ByteBufferReadable} * {@link #org.apache.hadoop.fs.ByteBufferReadable}
*/ */
private Boolean usingByteBufferRead = null; private Boolean usingByteBufferRead = null;
/** /**
* Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer} * Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer}
* before any other data goes in. The purpose of padding is to put input data * before any other data goes in. The purpose of padding is to put the input
* at proper position. * data at proper position.
*/ */
private byte padding; private byte padding;
private boolean closed; private boolean closed;
@ -144,14 +148,15 @@ public class CryptoInputStream extends FilterInputStream implements
return 0; return 0;
} }
int remaining = outBuffer.remaining(); final int remaining = outBuffer.remaining();
if (remaining > 0) { if (remaining > 0) {
int n = Math.min(len, remaining); int n = Math.min(len, remaining);
outBuffer.get(b, off, n); outBuffer.get(b, off, n);
return n; return n;
} else { } else {
int n = 0; int n = 0;
/**
/*
* Check whether the underlying stream is {@link ByteBufferReadable}, * Check whether the underlying stream is {@link ByteBufferReadable},
* it can avoid bytes copy. * it can avoid bytes copy.
*/ */
@ -186,11 +191,11 @@ public class CryptoInputStream extends FilterInputStream implements
} }
} }
// Read data from underlying stream. /** Read data from underlying stream. */
private int readFromUnderlyingStream() throws IOException { private int readFromUnderlyingStream() throws IOException {
int toRead = inBuffer.remaining(); final int toRead = inBuffer.remaining();
byte[] tmp = getTmpBuf(); final byte[] tmp = getTmpBuf();
int n = in.read(tmp, 0, toRead); final int n = in.read(tmp, 0, toRead);
if (n > 0) { if (n > 0) {
inBuffer.put(tmp, 0, n); inBuffer.put(tmp, 0, n);
} }
@ -221,19 +226,19 @@ public class CryptoInputStream extends FilterInputStream implements
inBuffer.clear(); inBuffer.clear();
outBuffer.flip(); outBuffer.flip();
if (padding > 0) { if (padding > 0) {
/** /*
* The plain text and cipher text have 1:1 mapping, they start at same * The plain text and cipher text have a 1:1 mapping, they start at the
* position. * same position.
*/ */
outBuffer.position(padding); outBuffer.position(padding);
padding = 0; padding = 0;
} }
if (decryptor.isContextReset()) { if (decryptor.isContextReset()) {
/** /*
* Typically we will not get here. To improve performance in CTR mode, * This code is generally not executed since the decryptor usually
* we rely on the decryptor maintaining context, for example calculating * maintains decryption context (e.g. the counter) internally. However,
* the counter. Unfortunately, some bad implementations can't maintain * some implementations can't maintain context so a re-init is necessary
* context so we need to re-init after doing decryption. * after each decryption call.
*/ */
updateDecryptor(); updateDecryptor();
} }
@ -243,7 +248,7 @@ public class CryptoInputStream extends FilterInputStream implements
* Update the {@link #decryptor}. Calculate the counter and {@link #padding}. * Update the {@link #decryptor}. Calculate the counter and {@link #padding}.
*/ */
private void updateDecryptor() throws IOException { private void updateDecryptor() throws IOException {
long counter = streamOffset / codec.getAlgorithmBlockSize(); final long counter = streamOffset / codec.getAlgorithmBlockSize();
padding = (byte)(streamOffset % codec.getAlgorithmBlockSize()); padding = (byte)(streamOffset % codec.getAlgorithmBlockSize());
inBuffer.position(padding); // Set proper position for input data. inBuffer.position(padding); // Set proper position for input data.
codec.calculateIV(initIV, counter, iv); codec.calculateIV(initIV, counter, iv);
@ -251,8 +256,8 @@ public class CryptoInputStream extends FilterInputStream implements
} }
/** /**
* Reset the underlying stream offset; and clear {@link #inBuffer} and * Reset the underlying stream offset, and clear {@link #inBuffer} and
* {@link #outBuffer}. Typically this happens when doing {@link #seek(long)} * {@link #outBuffer}. This Typically happens during {@link #seek(long)}
* or {@link #skip(long)}. * or {@link #skip(long)}.
*/ */
private void resetStreamOffset(long offset) throws IOException { private void resetStreamOffset(long offset) throws IOException {
@ -274,30 +279,29 @@ public class CryptoInputStream extends FilterInputStream implements
closed = true; closed = true;
} }
/** /** Forcibly free the direct buffer. */
* Free the direct buffer manually.
*/
private void freeBuffers() { private void freeBuffers() {
sun.misc.Cleaner inBufferCleaner = final sun.misc.Cleaner inBufferCleaner =
((sun.nio.ch.DirectBuffer) inBuffer).cleaner(); ((sun.nio.ch.DirectBuffer) inBuffer).cleaner();
inBufferCleaner.clean(); inBufferCleaner.clean();
sun.misc.Cleaner outBufferCleaner = final sun.misc.Cleaner outBufferCleaner =
((sun.nio.ch.DirectBuffer) outBuffer).cleaner(); ((sun.nio.ch.DirectBuffer) outBuffer).cleaner();
outBufferCleaner.clean(); outBufferCleaner.clean();
} }
// Positioned read. /** Positioned read. */
@Override @Override
public int read(long position, byte[] buffer, int offset, int length) public int read(long position, byte[] buffer, int offset, int length)
throws IOException { throws IOException {
checkStream(); checkStream();
try { try {
int n = ((PositionedReadable) in).read(position, buffer, offset, length); final int n = ((PositionedReadable) in).read(position, buffer, offset,
length);
if (n > 0) { if (n > 0) {
/** /*
* Since this operation does not change the current offset of a file, * Since this operation does not change the current offset of a file,
* streamOffset should be not changed and we need to restore the * streamOffset should not be changed. We need to restore the decryptor
* decryptor and outBuffer after decryption. * and outBuffer after decryption.
*/ */
decrypt(position, buffer, offset, length); decrypt(position, buffer, offset, length);
} }
@ -310,24 +314,23 @@ public class CryptoInputStream extends FilterInputStream implements
} }
/** /**
* Decrypt given length of data in buffer: start from offset. * Decrypt length bytes in buffer starting at offset. Output is also put
* Output is also buffer and start from same offset. Restore the * into buffer starting at offset. Restore the {@link #decryptor} and
* {@link #decryptor} and {@link #outBuffer} after decryption. * {@link #outBuffer} after the decryption.
*/ */
private void decrypt(long position, byte[] buffer, int offset, int length) private void decrypt(long position, byte[] buffer, int offset, int length)
throws IOException { throws IOException {
final byte[] tmp = getTmpBuf();
byte[] tmp = getTmpBuf();
int unread = outBuffer.remaining(); int unread = outBuffer.remaining();
if (unread > 0) { // Cache outBuffer if (unread > 0) { // Cache outBuffer
outBuffer.get(tmp, 0, unread); outBuffer.get(tmp, 0, unread);
} }
long curOffset = streamOffset; final long curOffset = streamOffset;
resetStreamOffset(position); resetStreamOffset(position);
int n = 0; int n = 0;
while (n < length) { while (n < length) {
int toDecrypt = Math.min(length - n, inBuffer.remaining()); final int toDecrypt = Math.min(length - n, inBuffer.remaining());
inBuffer.put(buffer, offset + n, toDecrypt); inBuffer.put(buffer, offset + n, toDecrypt);
// Do decryption // Do decryption
decrypt(); decrypt();
@ -344,7 +347,7 @@ public class CryptoInputStream extends FilterInputStream implements
} }
} }
// Positioned read fully. /** Positioned read fully. */
@Override @Override
public void readFully(long position, byte[] buffer, int offset, int length) public void readFully(long position, byte[] buffer, int offset, int length)
throws IOException { throws IOException {
@ -352,9 +355,9 @@ public class CryptoInputStream extends FilterInputStream implements
try { try {
((PositionedReadable) in).readFully(position, buffer, offset, length); ((PositionedReadable) in).readFully(position, buffer, offset, length);
if (length > 0) { if (length > 0) {
/** /*
* Since this operation does not change the current offset of a file, * Since this operation does not change the current offset of the file,
* streamOffset should be not changed and we need to restore the decryptor * streamOffset should not be changed. We need to restore the decryptor
* and outBuffer after decryption. * and outBuffer after decryption.
*/ */
decrypt(position, buffer, offset, length); decrypt(position, buffer, offset, length);
@ -370,13 +373,16 @@ public class CryptoInputStream extends FilterInputStream implements
readFully(position, buffer, 0, buffer.length); readFully(position, buffer, 0, buffer.length);
} }
// Seek to a position. /** Seek to a position. */
@Override @Override
public void seek(long pos) throws IOException { public void seek(long pos) throws IOException {
Preconditions.checkArgument(pos >= 0, "Cannot seek to negative offset."); Preconditions.checkArgument(pos >= 0, "Cannot seek to negative offset.");
checkStream(); checkStream();
try { try {
// If target pos we have already read and decrypt. /*
* If data of target pos in the underlying stream has already been read
* and decrypted in outBuffer, we just need to re-position outBuffer.
*/
if (pos <= streamOffset && pos >= (streamOffset - outBuffer.remaining())) { if (pos <= streamOffset && pos >= (streamOffset - outBuffer.remaining())) {
int forward = (int) (pos - (streamOffset - outBuffer.remaining())); int forward = (int) (pos - (streamOffset - outBuffer.remaining()));
if (forward > 0) { if (forward > 0) {
@ -392,7 +398,7 @@ public class CryptoInputStream extends FilterInputStream implements
} }
} }
// Skip n bytes /** Skip n bytes */
@Override @Override
public long skip(long n) throws IOException { public long skip(long n) throws IOException {
Preconditions.checkArgument(n >= 0, "Negative skip length."); Preconditions.checkArgument(n >= 0, "Negative skip length.");
@ -405,11 +411,11 @@ public class CryptoInputStream extends FilterInputStream implements
outBuffer.position(pos); outBuffer.position(pos);
return n; return n;
} else { } else {
/** /*
* Subtract outBuffer.remaining() to see how many bytes we need to * Subtract outBuffer.remaining() to see how many bytes we need to
* skip in underlying stream. We get real skipped bytes number of * skip in the underlying stream. Add outBuffer.remaining() to the
* underlying stream then add outBuffer.remaining() to get skipped * actual number of skipped bytes in the underlying stream to get the
* bytes number from user's view. * number of skipped bytes from the user's point of view.
*/ */
n -= outBuffer.remaining(); n -= outBuffer.remaining();
long skipped = in.skip(n); long skipped = in.skip(n);
@ -423,7 +429,7 @@ public class CryptoInputStream extends FilterInputStream implements
} }
} }
// Get underlying stream position. /** Get underlying stream position. */
@Override @Override
public long getPos() throws IOException { public long getPos() throws IOException {
checkStream(); checkStream();
@ -431,16 +437,16 @@ public class CryptoInputStream extends FilterInputStream implements
return streamOffset - outBuffer.remaining(); return streamOffset - outBuffer.remaining();
} }
// ByteBuffer read. /** ByteBuffer read. */
@Override @Override
public int read(ByteBuffer buf) throws IOException { public int read(ByteBuffer buf) throws IOException {
checkStream(); checkStream();
if (in instanceof ByteBufferReadable) { if (in instanceof ByteBufferReadable) {
int unread = outBuffer.remaining(); final int unread = outBuffer.remaining();
if (unread > 0) { // Have unread decrypted data in buffer. if (unread > 0) { // Have unread decrypted data in buffer.
int toRead = buf.remaining(); int toRead = buf.remaining();
if (toRead <= unread) { if (toRead <= unread) {
int limit = outBuffer.limit(); final int limit = outBuffer.limit();
outBuffer.limit(outBuffer.position() + toRead); outBuffer.limit(outBuffer.position() + toRead);
buf.put(outBuffer); buf.put(outBuffer);
outBuffer.limit(limit); outBuffer.limit(limit);
@ -450,8 +456,8 @@ public class CryptoInputStream extends FilterInputStream implements
} }
} }
int pos = buf.position(); final int pos = buf.position();
int n = ((ByteBufferReadable) in).read(buf); final int n = ((ByteBufferReadable) in).read(buf);
if (n > 0) { if (n > 0) {
streamOffset += n; // Read n bytes streamOffset += n; // Read n bytes
decrypt(buf, n, pos); decrypt(buf, n, pos);
@ -470,8 +476,8 @@ public class CryptoInputStream extends FilterInputStream implements
*/ */
private void decrypt(ByteBuffer buf, int n, int start) private void decrypt(ByteBuffer buf, int n, int start)
throws IOException { throws IOException {
int pos = buf.position(); final int pos = buf.position();
int limit = buf.limit(); final int limit = buf.limit();
int len = 0; int len = 0;
while (len < n) { while (len < n) {
buf.position(start + len); buf.position(start + len);
@ -535,13 +541,13 @@ public class CryptoInputStream extends FilterInputStream implements
((Seekable) in).seek(getPos()); ((Seekable) in).seek(getPos());
resetStreamOffset(getPos()); resetStreamOffset(getPos());
} }
ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in). final ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in).
read(bufferPool, maxLength, opts); read(bufferPool, maxLength, opts);
if (buffer != null) { if (buffer != null) {
int n = buffer.remaining(); final int n = buffer.remaining();
if (n > 0) { if (n > 0) {
streamOffset += buffer.remaining(); // Read n bytes streamOffset += buffer.remaining(); // Read n bytes
int pos = buffer.position(); final int pos = buffer.position();
decrypt(buffer, n, pos); decrypt(buffer, n, pos);
} }
} }

View File

@ -52,17 +52,20 @@ public class CryptoOutputStream extends FilterOutputStream implements
private static final byte[] oneByteBuf = new byte[1]; private static final byte[] oneByteBuf = new byte[1];
private final CryptoCodec codec; private final CryptoCodec codec;
private final Encryptor encryptor; private final Encryptor encryptor;
/** /**
* Input data buffer. The data starts at inBuffer.position() and ends at * Input data buffer. The data starts at inBuffer.position() and ends at
* inBuffer.limit(). * inBuffer.limit().
*/ */
private ByteBuffer inBuffer; private ByteBuffer inBuffer;
/** /**
* Encrypted data buffer. The data starts at outBuffer.position() and ends at * Encrypted data buffer. The data starts at outBuffer.position() and ends at
* outBuffer.limit(); * outBuffer.limit();
*/ */
private ByteBuffer outBuffer; private ByteBuffer outBuffer;
private long streamOffset = 0; // Underlying stream offset. private long streamOffset = 0; // Underlying stream offset.
/** /**
* Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer} * Padding = pos%(algorithm blocksize); Padding is put into {@link #inBuffer}
* before any other data goes in. The purpose of padding is to put input data * before any other data goes in. The purpose of padding is to put input data
@ -134,7 +137,7 @@ public class CryptoOutputStream extends FilterOutputStream implements
throw new IndexOutOfBoundsException(); throw new IndexOutOfBoundsException();
} }
while (len > 0) { while (len > 0) {
int remaining = inBuffer.remaining(); final int remaining = inBuffer.remaining();
if (len < remaining) { if (len < remaining) {
inBuffer.put(b, off, len); inBuffer.put(b, off, len);
len = 0; len = 0;
@ -163,15 +166,16 @@ public class CryptoOutputStream extends FilterOutputStream implements
inBuffer.clear(); inBuffer.clear();
outBuffer.flip(); outBuffer.flip();
if (padding > 0) { if (padding > 0) {
/** /*
* The plain text and cipher text have 1:1 mapping, they start at same * The plain text and cipher text have a 1:1 mapping, they start at the
* position. * same position.
*/ */
outBuffer.position(padding); outBuffer.position(padding);
padding = 0; padding = 0;
} }
int len = outBuffer.remaining(); final int len = outBuffer.remaining();
/**
/*
* If underlying stream supports {@link ByteBuffer} write in future, needs * If underlying stream supports {@link ByteBuffer} write in future, needs
* refine here. * refine here.
*/ */
@ -181,12 +185,11 @@ public class CryptoOutputStream extends FilterOutputStream implements
streamOffset += len; streamOffset += len;
if (encryptor.isContextReset()) { if (encryptor.isContextReset()) {
/** /*
* We will generally not get here. For CTR mode, to improve * This code is generally not executed since the encryptor usually
* performance, we rely on the encryptor maintaining context, for * maintains encryption context (e.g. the counter) internally. However,
* example to calculate the counter. But some bad implementations * some implementations can't maintain context so a re-init is necessary
* can't maintain context, and need us to re-init after doing * after each encryption call.
* encryption.
*/ */
updateEncryptor(); updateEncryptor();
} }
@ -196,7 +199,7 @@ public class CryptoOutputStream extends FilterOutputStream implements
* Update the {@link #encryptor}: calculate counter and {@link #padding}. * Update the {@link #encryptor}: calculate counter and {@link #padding}.
*/ */
private void updateEncryptor() throws IOException { private void updateEncryptor() throws IOException {
long counter = streamOffset / codec.getAlgorithmBlockSize(); final long counter = streamOffset / codec.getAlgorithmBlockSize();
padding = (byte)(streamOffset % codec.getAlgorithmBlockSize()); padding = (byte)(streamOffset % codec.getAlgorithmBlockSize());
inBuffer.position(padding); // Set proper position for input data. inBuffer.position(padding); // Set proper position for input data.
codec.calculateIV(initIV, counter, iv); codec.calculateIV(initIV, counter, iv);
@ -222,21 +225,19 @@ public class CryptoOutputStream extends FilterOutputStream implements
closed = true; closed = true;
} }
/** /** Forcibly free the direct buffer. */
* Free the direct buffer manually.
*/
private void freeBuffers() { private void freeBuffers() {
sun.misc.Cleaner inBufferCleaner = final sun.misc.Cleaner inBufferCleaner =
((sun.nio.ch.DirectBuffer) inBuffer).cleaner(); ((sun.nio.ch.DirectBuffer) inBuffer).cleaner();
inBufferCleaner.clean(); inBufferCleaner.clean();
sun.misc.Cleaner outBufferCleaner = final sun.misc.Cleaner outBufferCleaner =
((sun.nio.ch.DirectBuffer) outBuffer).cleaner(); ((sun.nio.ch.DirectBuffer) outBuffer).cleaner();
outBufferCleaner.clean(); outBufferCleaner.clean();
} }
/** /**
* To flush, we need to encrypt the data in buffer and write to underlying * To flush, we need to encrypt the data in the buffer and write to the
* stream, then do the flush. * underlying stream, then do the flush.
*/ */
@Override @Override
public void flush() throws IOException { public void flush() throws IOException {

View File

@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
public interface Decryptor { public interface Decryptor {
/** /**
* Initialize the decryptor, the internal decryption context will be * Initialize the decryptor and the internal decryption context.
* reset. * reset.
* @param key decryption key. * @param key decryption key.
* @param iv decryption initialization vector * @param iv decryption initialization vector
@ -37,37 +37,34 @@ public interface Decryptor {
public void init(byte[] key, byte[] iv) throws IOException; public void init(byte[] key, byte[] iv) throws IOException;
/** /**
* Indicate whether decryption context is reset. * Indicate whether the decryption context is reset.
* <p/> * <p/>
* It's useful for some mode like CTR which requires different IV for * Certain modes, like CTR, require a different IV depending on the
* different parts of data. Usually decryptor can maintain the context * position in the stream. Generally, the decryptor maintains any necessary
* internally such as calculating IV/counter, then continue a multiple-part * context for calculating the IV and counter so that no reinit is necessary
* decryption operation without reinit the decryptor using key and the new * during the decryption. Reinit before each operation is inefficient.
* IV. For mode like CTR, if context is reset after each decryption, the
* decryptor should be reinit before each operation, that's not efficient.
* @return boolean whether context is reset. * @return boolean whether context is reset.
*/ */
public boolean isContextReset(); public boolean isContextReset();
/** /**
* This exposes a direct interface for record decryption with direct byte * This presents a direct interface decrypting with direct ByteBuffers.
* buffers.
* <p/> * <p/>
* The decrypt() function need not always consume the buffers provided, * This function does not always decrypt the entire buffer and may potentially
* it will need to be called multiple times to decrypt an entire buffer * need to be called multiple times to process an entire buffer. The object
* and the object will hold the decryption context internally. * may hold the decryption context internally.
* <p/> * <p/>
* Some implementation may need enough space in the destination buffer to * Some implementations may require sufficient space in the destination
* decrypt an entire input. * buffer to decrypt the entire input buffer.
* <p/> * <p/>
* The end result will move inBuffer.position() by the bytes-read and * Upon return, inBuffer.position() will be advanced by the number of bytes
* outBuffer.position() by the bytes-written. It should not modify the * read and outBuffer.position() by bytes written. Implementations should
* inBuffer.limit() or outBuffer.limit() to maintain consistency of operation. * not modify inBuffer.limit() and outBuffer.limit().
* <p/> * <p/>
* @param inBuffer in direct {@link ByteBuffer} for reading from. Requires * @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
* inBuffer != null and inBuffer.remaining() > 0 * not be null and inBuffer.remaining() must be > 0
* @param outBuffer out direct {@link ByteBuffer} for storing the results * @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
* into. Requires outBuffer != null and outBuffer.remaining() > 0 * not be null and outBuffer.remaining() must be > 0
* @throws IOException if decryption fails * @throws IOException if decryption fails
*/ */
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)

View File

@ -28,8 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
public interface Encryptor { public interface Encryptor {
/** /**
* Initialize the encryptor, the internal encryption context will be * Initialize the encryptor and the internal encryption context.
* reset.
* @param key encryption key. * @param key encryption key.
* @param iv encryption initialization vector * @param iv encryption initialization vector
* @throws IOException if initialization fails * @throws IOException if initialization fails
@ -37,37 +36,34 @@ public interface Encryptor {
public void init(byte[] key, byte[] iv) throws IOException; public void init(byte[] key, byte[] iv) throws IOException;
/** /**
* Indicate whether encryption context is reset. * Indicate whether the encryption context is reset.
* <p/> * <p/>
* It's useful for some mode like CTR which requires different IV for * Certain modes, like CTR, require a different IV depending on the
* different parts of data. Usually encryptor can maintain the context * position in the stream. Generally, the encryptor maintains any necessary
* internally such as calculating IV/counter, then continue a multiple-part * context for calculating the IV and counter so that no reinit is necessary
* encryption operation without reinit the encryptor using key and the new * during the encryption. Reinit before each operation is inefficient.
* IV. For mode like CTR, if context is reset after each encryption, the
* encryptor should be reinit before each operation, that's not efficient.
* @return boolean whether context is reset. * @return boolean whether context is reset.
*/ */
public boolean isContextReset(); public boolean isContextReset();
/** /**
* This exposes a direct interface for record encryption with direct byte * This presents a direct interface encrypting with direct ByteBuffers.
* buffers.
* <p/> * <p/>
* The encrypt() function need not always consume the buffers provided, * This function does not always encrypt the entire buffer and may potentially
* it will need to be called multiple times to encrypt an entire buffer * need to be called multiple times to process an entire buffer. The object
* and the object will hold the encryption context internally. * may hold the encryption context internally.
* <p/> * <p/>
* Some implementation may need enough space in the destination buffer to * Some implementations may require sufficient space in the destination
* encrypt an entire input. * buffer to encrypt the entire input buffer.
* <p/> * <p/>
* The end result will move inBuffer.position() by the bytes-read and * Upon return, inBuffer.position() will be advanced by the number of bytes
* outBuffer.position() by the bytes-written. It should not modify the * read and outBuffer.position() by bytes written. Implementations should
* inBuffer.limit() or outBuffer.limit() to maintain consistency of operation. * not modify inBuffer.limit() and outBuffer.limit().
* <p/> * <p/>
* @param inBuffer in direct {@link ByteBuffer} for reading from. Requires * @param inBuffer a direct {@link ByteBuffer} to read from. inBuffer may
* inBuffer != null and inBuffer.remaining() > 0 * not be null and inBuffer.remaining() must be > 0
* @param outBuffer out direct {@link ByteBuffer} for storing the results * @param outBuffer a direct {@link ByteBuffer} to write to. outBuffer may
* into. Requires outBuffer != null and outBuffer.remaining() > 0 * not be null and outBuffer.remaining() must be > 0
* @throws IOException if encryption fails * @throws IOException if encryption fails
*/ */
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)

View File

@ -53,8 +53,8 @@ public class JCEAESCTRDecryptor implements Decryptor {
} }
/** /**
* For AES-CTR, will consume all input data and needs enough space in the * AES-CTR will consume all of the input data. It requires enough space in
* destination buffer to decrypt entire input data. * the destination buffer to decrypt entire input buffer.
*/ */
@Override @Override
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)

View File

@ -53,8 +53,8 @@ public class JCEAESCTREncryptor implements Encryptor {
} }
/** /**
* For AES-CTR, will consume all input data and needs enough space in the * AES-CTR will consume all of the input data. It requires enough space in
* destination buffer to encrypt entire input data. * the destination buffer to encrypt entire input buffer.
*/ */
@Override @Override
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)

View File

@ -1370,8 +1370,7 @@
<name>hadoop.security.crypto.buffer.size</name> <name>hadoop.security.crypto.buffer.size</name>
<value>8192</value> <value>8192</value>
<description> <description>
The buffer size used in Crypto InputStream and OutputStream, and default The buffer size used by CryptoInputStream and CryptoOutputStream.
value is 8192.
</description> </description>
</property> </property>
</configuration> </configuration>

View File

@ -42,7 +42,7 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
public abstract class CryptoStreamsTestBase { public abstract class CryptoStreamsTestBase {
protected static final Log LOG= LogFactory.getLog( protected static final Log LOG = LogFactory.getLog(
CryptoStreamsTestBase.class); CryptoStreamsTestBase.class);
protected static CryptoCodec codec; protected static CryptoCodec codec;
@ -60,13 +60,13 @@ public abstract class CryptoStreamsTestBase {
@Before @Before
public void setUp() throws IOException { public void setUp() throws IOException {
// Generate data // Generate data
int seed = new Random().nextInt(); final int seed = new Random().nextInt();
DataOutputBuffer dataBuf = new DataOutputBuffer(); final DataOutputBuffer dataBuf = new DataOutputBuffer();
RandomDatum.Generator generator = new RandomDatum.Generator(seed); final RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for(int i=0; i < count; ++i) { for(int i = 0; i < count; ++i) {
generator.next(); generator.next();
RandomDatum key = generator.getKey(); final RandomDatum key = generator.getKey();
RandomDatum value = generator.getValue(); final RandomDatum value = generator.getValue();
key.write(dataBuf); key.write(dataBuf);
value.write(dataBuf); value.write(dataBuf);
@ -114,9 +114,7 @@ public abstract class CryptoStreamsTestBase {
protected abstract InputStream getInputStream(int bufferSize, byte[] key, protected abstract InputStream getInputStream(int bufferSize, byte[] key,
byte[] iv) throws IOException; byte[] iv) throws IOException;
/** /** Test crypto reading with different buffer size. */
* Test crypto reading with different buffer size.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testRead() throws Exception { public void testRead() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -148,9 +146,7 @@ public abstract class CryptoStreamsTestBase {
in.close(); in.close();
} }
/** /** Test crypto with different IV. */
* Test crypto with different IV.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testCryptoIV() throws Exception { public void testCryptoIV() throws Exception {
byte[] iv1 = iv.clone(); byte[] iv1 = iv.clone();
@ -202,7 +198,7 @@ public abstract class CryptoStreamsTestBase {
private void syncableCheck() throws IOException { private void syncableCheck() throws IOException {
OutputStream out = getOutputStream(smallBufferSize); OutputStream out = getOutputStream(smallBufferSize);
try { try {
int bytesWritten = dataLen/3; int bytesWritten = dataLen / 3;
out.write(data, 0, bytesWritten); out.write(data, 0, bytesWritten);
((Syncable) out).hflush(); ((Syncable) out).hflush();
@ -223,9 +219,9 @@ public abstract class CryptoStreamsTestBase {
private void verify(InputStream in, int bytesToVerify, private void verify(InputStream in, int bytesToVerify,
byte[] expectedBytes) throws IOException { byte[] expectedBytes) throws IOException {
byte[] readBuf = new byte[bytesToVerify]; final byte[] readBuf = new byte[bytesToVerify];
readAll(in, readBuf, 0, bytesToVerify); readAll(in, readBuf, 0, bytesToVerify);
for (int i=0; i<bytesToVerify; i++) { for (int i = 0; i < bytesToVerify; i++) {
Assert.assertEquals(expectedBytes[i], readBuf[i]); Assert.assertEquals(expectedBytes[i], readBuf[i]);
} }
} }
@ -246,9 +242,7 @@ public abstract class CryptoStreamsTestBase {
return total; return total;
} }
/** /** Test positioned read. */
* Test positioned read.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testPositionedRead() throws Exception { public void testPositionedRead() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -256,10 +250,10 @@ public abstract class CryptoStreamsTestBase {
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
// Pos: 1/3 dataLen // Pos: 1/3 dataLen
positionedReadCheck(in , dataLen/3); positionedReadCheck(in , dataLen / 3);
// Pos: 1/2 dataLen // Pos: 1/2 dataLen
positionedReadCheck(in, dataLen/2); positionedReadCheck(in, dataLen / 2);
in.close(); in.close();
} }
@ -275,25 +269,23 @@ public abstract class CryptoStreamsTestBase {
Assert.assertArrayEquals(readData, expectedData); Assert.assertArrayEquals(readData, expectedData);
} }
/** /** Test read fully */
* Test read fully
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testReadFully() throws Exception { public void testReadFully() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
writeData(out); writeData(out);
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
final int len1 = dataLen/4; final int len1 = dataLen / 4;
// Read len1 bytes // Read len1 bytes
byte [] readData = new byte[len1]; byte[] readData = new byte[len1];
readAll(in, readData, 0, len1); readAll(in, readData, 0, len1);
byte[] expectedData = new byte[len1]; byte[] expectedData = new byte[len1];
System.arraycopy(data, 0, expectedData, 0, len1); System.arraycopy(data, 0, expectedData, 0, len1);
Assert.assertArrayEquals(readData, expectedData); Assert.assertArrayEquals(readData, expectedData);
// Pos: 1/3 dataLen // Pos: 1/3 dataLen
readFullyCheck(in, dataLen/3); readFullyCheck(in, dataLen / 3);
// Read len1 bytes // Read len1 bytes
readData = new byte[len1]; readData = new byte[len1];
@ -303,7 +295,7 @@ public abstract class CryptoStreamsTestBase {
Assert.assertArrayEquals(readData, expectedData); Assert.assertArrayEquals(readData, expectedData);
// Pos: 1/2 dataLen // Pos: 1/2 dataLen
readFullyCheck(in, dataLen/2); readFullyCheck(in, dataLen / 2);
// Read len1 bytes // Read len1 bytes
readData = new byte[len1]; readData = new byte[len1];
@ -331,9 +323,7 @@ public abstract class CryptoStreamsTestBase {
} }
} }
/** /** Test seek to different position. */
* Test seek to different position.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testSeek() throws Exception { public void testSeek() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -341,13 +331,15 @@ public abstract class CryptoStreamsTestBase {
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
// Pos: 1/3 dataLen // Pos: 1/3 dataLen
seekCheck(in, dataLen/3); seekCheck(in, dataLen / 3);
// Pos: 0 // Pos: 0
seekCheck(in, 0); seekCheck(in, 0);
// Pos: 1/2 dataLen // Pos: 1/2 dataLen
seekCheck(in, dataLen/2); seekCheck(in, dataLen / 2);
final long pos = ((Seekable) in).getPos();
// Pos: -3 // Pos: -3
try { try {
@ -355,8 +347,9 @@ public abstract class CryptoStreamsTestBase {
Assert.fail("Seek to negative offset should fail."); Assert.fail("Seek to negative offset should fail.");
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
GenericTestUtils.assertExceptionContains("Cannot seek to negative " + GenericTestUtils.assertExceptionContains("Cannot seek to negative " +
"offset", e); "offset", e);
} }
Assert.assertEquals(pos, ((Seekable) in).getPos());
// Pos: dataLen + 3 // Pos: dataLen + 3
try { try {
@ -365,6 +358,7 @@ public abstract class CryptoStreamsTestBase {
} catch (IOException e) { } catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot seek after EOF", e); GenericTestUtils.assertExceptionContains("Cannot seek after EOF", e);
} }
Assert.assertEquals(pos, ((Seekable) in).getPos());
in.close(); in.close();
} }
@ -382,9 +376,7 @@ public abstract class CryptoStreamsTestBase {
Assert.assertArrayEquals(readData, expectedData); Assert.assertArrayEquals(readData, expectedData);
} }
/** /** Test get position. */
* Test get position.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testGetPos() throws Exception { public void testGetPos() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -393,7 +385,7 @@ public abstract class CryptoStreamsTestBase {
// Default buffer size // Default buffer size
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
byte[] result = new byte[dataLen]; byte[] result = new byte[dataLen];
int n1 = readAll(in, result, 0, dataLen/3); int n1 = readAll(in, result, 0, dataLen / 3);
Assert.assertEquals(n1, ((Seekable) in).getPos()); Assert.assertEquals(n1, ((Seekable) in).getPos());
int n2 = readAll(in, result, n1, dataLen - n1); int n2 = readAll(in, result, n1, dataLen - n1);
@ -409,7 +401,7 @@ public abstract class CryptoStreamsTestBase {
// Default buffer size // Default buffer size
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
byte[] result = new byte[dataLen]; byte[] result = new byte[dataLen];
int n1 = readAll(in, result, 0, dataLen/3); int n1 = readAll(in, result, 0, dataLen / 3);
Assert.assertEquals(in.available(), dataLen - n1); Assert.assertEquals(in.available(), dataLen - n1);
int n2 = readAll(in, result, n1, dataLen - n1); int n2 = readAll(in, result, n1, dataLen - n1);
@ -417,9 +409,7 @@ public abstract class CryptoStreamsTestBase {
in.close(); in.close();
} }
/** /** Test skip. */
* Test skip.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testSkip() throws Exception { public void testSkip() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -428,10 +418,10 @@ public abstract class CryptoStreamsTestBase {
// Default buffer size // Default buffer size
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
byte[] result = new byte[dataLen]; byte[] result = new byte[dataLen];
int n1 = readAll(in, result, 0, dataLen/3); int n1 = readAll(in, result, 0, dataLen / 3);
Assert.assertEquals(n1, ((Seekable) in).getPos()); Assert.assertEquals(n1, ((Seekable) in).getPos());
long skipped = in.skip(dataLen/3); long skipped = in.skip(dataLen / 3);
int n2 = readAll(in, result, 0, dataLen); int n2 = readAll(in, result, 0, dataLen);
Assert.assertEquals(dataLen, n1 + skipped + n2); Assert.assertEquals(dataLen, n1 + skipped + n2);
@ -468,9 +458,7 @@ public abstract class CryptoStreamsTestBase {
Assert.assertArrayEquals(readData, expectedData); Assert.assertArrayEquals(readData, expectedData);
} }
/** /** Test byte buffer read with different buffer size. */
* Test byte buffer read with different buffer size.
*/
@Test(timeout=120000) @Test(timeout=120000)
public void testByteBufferRead() throws Exception { public void testByteBufferRead() throws Exception {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
@ -530,8 +518,8 @@ public abstract class CryptoStreamsTestBase {
OutputStream out = getOutputStream(defaultBufferSize); OutputStream out = getOutputStream(defaultBufferSize);
writeData(out); writeData(out);
final int len1 = dataLen/8; final int len1 = dataLen / 8;
final int len2 = dataLen/10; final int len2 = dataLen / 10;
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
// Read len1 data. // Read len1 data.
@ -551,7 +539,7 @@ public abstract class CryptoStreamsTestBase {
Assert.assertEquals(len2, n); Assert.assertEquals(len2, n);
// Pos: 1/4 dataLen // Pos: 1/4 dataLen
positionedReadCheck(in , dataLen/4); positionedReadCheck(in , dataLen / 4);
// Pos should be len1 + len2 + len2 // Pos should be len1 + len2 + len2
pos = ((Seekable) in).getPos(); pos = ((Seekable) in).getPos();
@ -572,7 +560,7 @@ public abstract class CryptoStreamsTestBase {
Assert.assertEquals(len1 + 2 * len2 + nRead, pos); Assert.assertEquals(len1 + 2 * len2 + nRead, pos);
// Pos: 1/3 dataLen // Pos: 1/3 dataLen
positionedReadCheck(in , dataLen/3); positionedReadCheck(in , dataLen / 3);
// Read forward len1 // Read forward len1
readData = new byte[len1]; readData = new byte[len1];
@ -611,18 +599,18 @@ public abstract class CryptoStreamsTestBase {
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
final int len1 = dataLen/8; final int len1 = dataLen / 8;
byte[] readData = new byte[len1]; byte[] readData = new byte[len1];
readAll(in, readData, 0, len1); readAll(in, readData, 0, len1);
// Pos: 1/3 dataLen // Pos: 1/3 dataLen
seekToNewSourceCheck(in, dataLen/3); seekToNewSourceCheck(in, dataLen / 3);
// Pos: 0 // Pos: 0
seekToNewSourceCheck(in, 0); seekToNewSourceCheck(in, 0);
// Pos: 1/2 dataLen // Pos: 1/2 dataLen
seekToNewSourceCheck(in, dataLen/2); seekToNewSourceCheck(in, dataLen / 2);
// Pos: -3 // Pos: -3
try { try {
@ -638,7 +626,8 @@ public abstract class CryptoStreamsTestBase {
seekToNewSourceCheck(in, dataLen + 3); seekToNewSourceCheck(in, dataLen + 3);
Assert.fail("Seek after EOF should fail."); Assert.fail("Seek after EOF should fail.");
} catch (IOException e) { } catch (IOException e) {
GenericTestUtils.assertExceptionContains("Attempted to read past end of file", e); GenericTestUtils.assertExceptionContains("Attempted to read past " +
"end of file", e);
} }
in.close(); in.close();
@ -677,7 +666,7 @@ public abstract class CryptoStreamsTestBase {
writeData(out); writeData(out);
InputStream in = getInputStream(defaultBufferSize); InputStream in = getInputStream(defaultBufferSize);
final int len1 = dataLen/8; final int len1 = dataLen / 8;
// ByteBuffer size is len1 // ByteBuffer size is len1
ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in).read( ByteBuffer buffer = ((HasEnhancedByteBufferAccess) in).read(
getBufferPool(), len1, EnumSet.of(ReadOption.SKIP_CHECKSUMS)); getBufferPool(), len1, EnumSet.of(ReadOption.SKIP_CHECKSUMS));