Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1225489 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Harsh J 2011-12-29 10:27:14 +00:00
parent 7afb9aca70
commit 07027b80f3
5 changed files with 111 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -12,6 +12,8 @@ Trunk (unreleased changes)
HADOOP-7875. Add helper class to unwrap protobuf ServiceException.
(suresh)
HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh)
IMPROVEMENTS
HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -737,6 +737,27 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return Long.parseLong(valueString);
}
/**
* Get the value of the <code>name</code> property as a <code>long</code> or
* human readable format. If no such property exists, the provided default
* value is returned, or if the specified value is not a valid
* <code>long</code> or human readable format, then an error is thrown. You
* can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
* t(tera), p(peta), e(exa)
*
* @param name property name.
* @param defaultValue default value.
* @throws NumberFormatException when the value is invalid
* @return property value as a <code>long</code>,
* or <code>defaultValue</code>.
*/
public long getLongBytes(String name, long defaultValue) {
String valueString = getTrimmed(name);
if (valueString == null)
return defaultValue;
return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
}
private String getHexDigits(String value) {
boolean negative = false;
String str = value;

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
View File

@ -661,7 +661,14 @@ public class StringUtils {
if (Character.isDigit(lastchar))
return Long.parseLong(s);
else {
long prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
long prefix;
try {
prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Invalid size prefix '" + lastchar
+ "' in '" + s
+ "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)");
}
long num = Long.parseLong(s.substring(0, lastpos));
if (num > (Long.MAX_VALUE/prefix) || num < (Long.MIN_VALUE/prefix)) {
throw new IllegalArgumentException(s + " does not fit in a Long");

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -405,12 +405,16 @@ public class TestConfiguration extends TestCase {
conf.addResource(fileResource);
assertEquals(20, conf.getInt("test.int1", 0));
assertEquals(20, conf.getLong("test.int1", 0));
assertEquals(20, conf.getLongBytes("test.int1", 0));
assertEquals(20, conf.getInt("test.int2", 0));
assertEquals(20, conf.getLong("test.int2", 0));
assertEquals(20, conf.getLongBytes("test.int2", 0));
assertEquals(-20, conf.getInt("test.int3", 0));
assertEquals(-20, conf.getLong("test.int3", 0));
assertEquals(-20, conf.getLongBytes("test.int3", 0));
assertEquals(-20, conf.getInt("test.int4", 0));
assertEquals(-20, conf.getLong("test.int4", 0));
assertEquals(-20, conf.getLongBytes("test.int4", 0));
try {
conf.getInt("test.int5", 0);
fail("Property had invalid int value, but was read successfully.");
@ -419,6 +423,26 @@ public class TestConfiguration extends TestCase {
}
}
public void testHumanReadableValues() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.humanReadableValue1", "1m");
appendProperty("test.humanReadableValue2", "1M");
appendProperty("test.humanReadableValue5", "1MBCDE");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(1048576, conf.getLongBytes("test.humanReadableValue1", 0));
assertEquals(1048576, conf.getLongBytes("test.humanReadableValue2", 0));
try {
conf.getLongBytes("test.humanReadableValue5", 0);
fail("Property had invalid human readable value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
public void testBooleanValues() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();

58
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
View File

@ -143,8 +143,62 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
}
assertEquals(0L, StringUtils.TraditionalBinaryPrefix.string2long("0"));
assertEquals(-1259520L, StringUtils.TraditionalBinaryPrefix.string2long("-1230k"));
assertEquals(956703965184L, StringUtils.TraditionalBinaryPrefix.string2long("891g"));
assertEquals(1024L, StringUtils.TraditionalBinaryPrefix.string2long("1k"));
assertEquals(-1024L, StringUtils.TraditionalBinaryPrefix.string2long("-1k"));
assertEquals(1259520L,
StringUtils.TraditionalBinaryPrefix.string2long("1230K"));
assertEquals(-1259520L,
StringUtils.TraditionalBinaryPrefix.string2long("-1230K"));
assertEquals(104857600L,
StringUtils.TraditionalBinaryPrefix.string2long("100m"));
assertEquals(-104857600L,
StringUtils.TraditionalBinaryPrefix.string2long("-100M"));
assertEquals(956703965184L,
StringUtils.TraditionalBinaryPrefix.string2long("891g"));
assertEquals(-956703965184L,
StringUtils.TraditionalBinaryPrefix.string2long("-891G"));
assertEquals(501377302265856L,
StringUtils.TraditionalBinaryPrefix.string2long("456t"));
assertEquals(-501377302265856L,
StringUtils.TraditionalBinaryPrefix.string2long("-456T"));
assertEquals(11258999068426240L,
StringUtils.TraditionalBinaryPrefix.string2long("10p"));
assertEquals(-11258999068426240L,
StringUtils.TraditionalBinaryPrefix.string2long("-10P"));
assertEquals(1152921504606846976L,
StringUtils.TraditionalBinaryPrefix.string2long("1e"));
assertEquals(-1152921504606846976L,
StringUtils.TraditionalBinaryPrefix.string2long("-1E"));
String tooLargeNumStr = "10e";
try {
StringUtils.TraditionalBinaryPrefix.string2long(tooLargeNumStr);
fail("Test passed for a number " + tooLargeNumStr + " too large");
} catch (IllegalArgumentException e) {
assertEquals(tooLargeNumStr + " does not fit in a Long", e.getMessage());
}
String tooSmallNumStr = "-10e";
try {
StringUtils.TraditionalBinaryPrefix.string2long(tooSmallNumStr);
fail("Test passed for a number " + tooSmallNumStr + " too small");
} catch (IllegalArgumentException e) {
assertEquals(tooSmallNumStr + " does not fit in a Long", e.getMessage());
}
String invalidFormatNumStr = "10kb";
char invalidPrefix = 'b';
try {
StringUtils.TraditionalBinaryPrefix.string2long(invalidFormatNumStr);
fail("Test passed for a number " + invalidFormatNumStr
+ " has invalid format");
} catch (IllegalArgumentException e) {
assertEquals("Invalid size prefix '" + invalidPrefix + "' in '"
+ invalidFormatNumStr
+ "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)",
e.getMessage());
}
}
@Test