Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15757. ABFS: remove dependency on common-codec Base64. 

Contributed by Da Zhou.
This commit is contained in:
Thomas Marquardt 2018-09-14 22:34:19 +00:00
parent 26211019c8
commit b4c23043d3
7 changed files with 344 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
View File

@ -44,4 +44,6 @@
<suppressions>
<suppress checks="ParameterNumber|MagicNumber"
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]AzureBlobFileSystemStore.java"/>
<suppress checks="ParameterNumber|MagicNumber"
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]utils[\\/]Base64.java"/>
</suppressions>

6
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
View File

@ -41,7 +41,6 @@ import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.DatatypeConverter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -77,6 +76,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@ -865,7 +865,7 @@ public class AzureBlobFileSystemStore {
throw new CharacterCodingException();
}
String encodedPropertyValue = DatatypeConverter.printBase64Binary(encoder.encode(CharBuffer.wrap(value)).array());
String encodedPropertyValue = Base64.encode(encoder.encode(CharBuffer.wrap(value)).array());
commaSeparatedProperties.append(key)
.append(AbfsHttpConstants.EQUAL)
.append(encodedPropertyValue);
@ -903,7 +903,7 @@ public class AzureBlobFileSystemStore {
throw new InvalidFileSystemPropertyException(xMsProperties);
}
byte[] decodedValue = DatatypeConverter.parseBase64Binary(nameValue[1]);
byte[] decodedValue = Base64.decode(nameValue[1]);
final String value;
try {

4
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/diagnostics/Base64StringConfigurationBasicValidator.java
View File

@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.azurebfs.contracts.diagnostics.ConfigurationValidator;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
/**
* String Base64 configuration value Validator.
@ -42,7 +42,7 @@ public class Base64StringConfigurationBasicValidator extends ConfigurationBasicV
return result;
}
if (!Base64.isBase64(configValue)) {
if (!Base64.validateIsBase64String(configValue)) {
throw new InvalidConfigurationValueException(getConfigKey());
}
return configValue;

9
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SharedKeyCredentials.java
View File

@ -39,10 +39,9 @@ import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.codec.Charsets;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
/**
* Represents the shared key credentials used to access an Azure Storage
@ -52,8 +51,6 @@ public class SharedKeyCredentials {
private static final int EXPECTED_BLOB_QUEUE_CANONICALIZED_STRING_LENGTH = 300;
private static final Pattern CRLF = Pattern.compile("\r\n", Pattern.LITERAL);
private static final String HMAC_SHA256 = "HmacSHA256";
private static final Base64 BASE_64 = new Base64();
/**
* Stores a reference to the RFC1123 date/time pattern.
*/
@ -73,7 +70,7 @@ public class SharedKeyCredentials {
throw new IllegalArgumentException("Invalid account key.");
}
this.accountName = accountName;
this.accountKey = BASE_64.decode(accountKey);
this.accountKey = Base64.decode(accountKey);
initializeMac();
}
@ -100,7 +97,7 @@ public class SharedKeyCredentials {
synchronized (this) {
hmac = hmacSha256.doFinal(utf8Bytes);
}
return new String(BASE_64.encode(hmac), Charsets.UTF_8);
return Base64.encode(hmac);
}
/**

329
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/Base64.java Normal file
View File

@ -0,0 +1,329 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azurebfs.utils;
/**
* Base64
*/
public final class Base64 {
/**
* The Base 64 Characters.
*/
private static final String BASE_64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
/**
* Decoded values, -1 is invalid character, -2 is = pad character.
*/
private static final byte[] DECODE_64 = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0-15
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /*
* 16- 31
*/
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, /*
* 32- 47
*/
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, /*
* 48- 63
*/
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 64-79 */
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, /*
* 80- 95
*/
-1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, /*
* 96- 111
*/
41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1 /*
* 112- 127
*/
};
/**
* Decodes a given Base64 string into its corresponding byte array.
*
* @param data
* the Base64 string, as a <code>String</code> object, to decode
*
* @return the corresponding decoded byte array
* @throws IllegalArgumentException
* If the string is not a valid base64 encoded string
*/
public static byte[] decode(final String data) {
if (data == null) {
throw new IllegalArgumentException("The data parameter is not a valid base64-encoded string.");
}
int byteArrayLength = 3 * data.length() / 4;
if (data.endsWith("==")) {
byteArrayLength -= 2;
}
else if (data.endsWith("=")) {
byteArrayLength -= 1;
}
final byte[] retArray = new byte[byteArrayLength];
int byteDex = 0;
int charDex = 0;
for (; charDex < data.length(); charDex += 4) {
// get 4 chars, convert to 3 bytes
final int char1 = DECODE_64[(byte) data.charAt(charDex)];
final int char2 = DECODE_64[(byte) data.charAt(charDex + 1)];
final int char3 = DECODE_64[(byte) data.charAt(charDex + 2)];
final int char4 = DECODE_64[(byte) data.charAt(charDex + 3)];
if (char1 < 0 || char2 < 0 || char3 == -1 || char4 == -1) {
// invalid character(-1), or bad padding (-2)
throw new IllegalArgumentException("The data parameter is not a valid base64-encoded string.");
}
int tVal = char1 << 18;
tVal += char2 << 12;
tVal += (char3 & 0xff) << 6;
tVal += char4 & 0xff;
if (char3 == -2) {
// two "==" pad chars, check bits 12-24
tVal &= 0x00FFF000;
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
}
else if (char4 == -2) {
// one pad char "=" , check bits 6-24.
tVal &= 0x00FFFFC0;
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
retArray[byteDex++] = (byte) (tVal >> 8 & 0xFF);
}
else {
// No pads take all 3 bytes, bits 0-24
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
retArray[byteDex++] = (byte) (tVal >> 8 & 0xFF);
retArray[byteDex++] = (byte) (tVal & 0xFF);
}
}
return retArray;
}
/**
* Decodes a given Base64 string into its corresponding byte array.
*
* @param data
* the Base64 string, as a <code>String</code> object, to decode
*
* @return the corresponding decoded byte array
* @throws IllegalArgumentException
* If the string is not a valid base64 encoded string
*/
public static Byte[] decodeAsByteObjectArray(final String data) {
int byteArrayLength = 3 * data.length() / 4;
if (data.endsWith("==")) {
byteArrayLength -= 2;
}
else if (data.endsWith("=")) {
byteArrayLength -= 1;
}
final Byte[] retArray = new Byte[byteArrayLength];
int byteDex = 0;
int charDex = 0;
for (; charDex < data.length(); charDex += 4) {
// get 4 chars, convert to 3 bytes
final int char1 = DECODE_64[(byte) data.charAt(charDex)];
final int char2 = DECODE_64[(byte) data.charAt(charDex + 1)];
final int char3 = DECODE_64[(byte) data.charAt(charDex + 2)];
final int char4 = DECODE_64[(byte) data.charAt(charDex + 3)];
if (char1 < 0 || char2 < 0 || char3 == -1 || char4 == -1) {
// invalid character(-1), or bad padding (-2)
throw new IllegalArgumentException("The data parameter is not a valid base64-encoded string.");
}
int tVal = char1 << 18;
tVal += char2 << 12;
tVal += (char3 & 0xff) << 6;
tVal += char4 & 0xff;
if (char3 == -2) {
// two "==" pad chars, check bits 12-24
tVal &= 0x00FFF000;
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
}
else if (char4 == -2) {
// one pad char "=" , check bits 6-24.
tVal &= 0x00FFFFC0;
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
retArray[byteDex++] = (byte) (tVal >> 8 & 0xFF);
}
else {
// No pads take all 3 bytes, bits 0-24
retArray[byteDex++] = (byte) (tVal >> 16 & 0xFF);
retArray[byteDex++] = (byte) (tVal >> 8 & 0xFF);
retArray[byteDex++] = (byte) (tVal & 0xFF);
}
}
return retArray;
}
/**
* Encodes a byte array as a Base64 string.
*
* @param data
* the byte array to encode
* @return the Base64-encoded string, as a <code>String</code> object
*/
public static String encode(final byte[] data) {
final StringBuilder builder = new StringBuilder();
final int dataRemainder = data.length % 3;
int j = 0;
int n = 0;
for (; j < data.length; j += 3) {
if (j < data.length - dataRemainder) {
n = ((data[j] & 0xFF) << 16) + ((data[j + 1] & 0xFF) << 8) + (data[j + 2] & 0xFF);
}
else {
if (dataRemainder == 1) {
n = (data[j] & 0xFF) << 16;
}
else if (dataRemainder == 2) {
n = ((data[j] & 0xFF) << 16) + ((data[j + 1] & 0xFF) << 8);
}
}
// Left here for readability
// byte char1 = (byte) ((n >>> 18) & 0x3F);
// byte char2 = (byte) ((n >>> 12) & 0x3F);
// byte char3 = (byte) ((n >>> 6) & 0x3F);
// byte char4 = (byte) (n & 0x3F);
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 18) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 12) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 6) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) (n & 0x3F)));
}
final int bLength = builder.length();
// append '=' to pad
if (data.length % 3 == 1) {
builder.replace(bLength - 2, bLength, "==");
}
else if (data.length % 3 == 2) {
builder.replace(bLength - 1, bLength, "=");
}
return builder.toString();
}
/**
* Encodes a byte array as a Base64 string.
*
* @param data
* the byte array to encode
* @return the Base64-encoded string, as a <code>String</code> object
*/
public static String encode(final Byte[] data) {
final StringBuilder builder = new StringBuilder();
final int dataRemainder = data.length % 3;
int j = 0;
int n = 0;
for (; j < data.length; j += 3) {
if (j < data.length - dataRemainder) {
n = ((data[j] & 0xFF) << 16) + ((data[j + 1] & 0xFF) << 8) + (data[j + 2] & 0xFF);
}
else {
if (dataRemainder == 1) {
n = (data[j] & 0xFF) << 16;
}
else if (dataRemainder == 2) {
n = ((data[j] & 0xFF) << 16) + ((data[j + 1] & 0xFF) << 8);
}
}
// Left here for readability
// byte char1 = (byte) ((n >>> 18) & 0x3F);
// byte char2 = (byte) ((n >>> 12) & 0x3F);
// byte char3 = (byte) ((n >>> 6) & 0x3F);
// byte char4 = (byte) (n & 0x3F);
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 18) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 12) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) ((n >>> 6) & 0x3F)));
builder.append(BASE_64_CHARS.charAt((byte) (n & 0x3F)));
}
final int bLength = builder.length();
// append '=' to pad
if (data.length % 3 == 1) {
builder.replace(bLength - 2, bLength, "==");
}
else if (data.length % 3 == 2) {
builder.replace(bLength - 1, bLength, "=");
}
return builder.toString();
}
/**
* Determines whether the given string contains only Base64 characters.
*
* @param data
* the string, as a <code>String</code> object, to validate
* @return <code>true</code> if <code>data</code> is a valid Base64 string, otherwise <code>false</code>
*/
public static boolean validateIsBase64String(final String data) {
if (data == null || data.length() % 4 != 0) {
return false;
}
for (int m = 0; m < data.length(); m++) {
final byte charByte = (byte) data.charAt(m);
// pad char detected
if (DECODE_64[charByte] == -2) {
if (m < data.length() - 2) {
return false;
}
else if (m == data.length() - 2 && DECODE_64[(byte) data.charAt(m + 1)] != -2) {
return false;
}
}
if (charByte < 0 || DECODE_64[charByte] == -1) {
return false;
}
}
return true;
}
/**
* Private Default Ctor.
*/
private Base64() {
// No op
}
}

7
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidati
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
@ -42,7 +43,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.D
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_AZURE_BLOCK_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.AZURE_BLOCK_LOCATION_HOST_DEFAULT;
import org.apache.commons.codec.binary.Base64;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
@ -97,10 +97,9 @@ public class TestAbfsConfigurationFieldsValidation {
public TestAbfsConfigurationFieldsValidation() throws Exception {
super();
Base64 base64 = new Base64();
this.accountName = "testaccount1.blob.core.windows.net";
this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
this.encodedString = Base64.encode("base64Value".getBytes(Charsets.UTF_8));
this.encodedAccountKey = Base64.encode("someAccountKey".getBytes(Charsets.UTF_8));
Configuration configuration = new Configuration();
configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
configuration.set(INT_KEY, "1234565");

7
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
View File

@ -18,14 +18,11 @@
package org.apache.hadoop.fs.azurebfs.diagnostics;
import org.apache.commons.codec.Charsets;
import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
@ -109,7 +106,7 @@ public class TestConfigurationValidators extends Assert {
@Test
public void testBase64StringConfigValidator() throws Exception {
String encodedVal = new String(new Base64().encode("someValue".getBytes()), Charsets.UTF_8);
String encodedVal = Base64.encode("someValue".getBytes());
Base64StringConfigurationBasicValidator base64StringConfigurationValidator = new Base64StringConfigurationBasicValidator(FAKE_KEY, "", false);
assertEquals("", base64StringConfigurationValidator.validate(null));