Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-7350. Use ServiceLoader to discover compression codec classes. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1328083 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White 2012-04-19 19:20:31 +00:00
parent e3806060ce
commit 0363ca0489
5 changed files with 84 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -360,6 +360,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh
existence for starting start-yarn.sh. (Devaraj K via eli)
HADOOP-7350. Use ServiceLoader to discover compression codec classes.
(tomwhite)
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)

38
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
View File

@ -37,6 +37,9 @@ public class CompressionCodecFactory {
public static final Log LOG =
LogFactory.getLog(CompressionCodecFactory.class.getName());
private static final ServiceLoader<CompressionCodec> CODEC_PROVIDERS =
ServiceLoader.load(CompressionCodec.class);
/**
* A map from the reversed filename suffixes to the codecs.
* This is probably overkill, because the maps should be small, but it
@ -95,16 +98,23 @@ public String toString() {
}
/**
* Get the list of codecs listed in the configuration
* Get the list of codecs discovered via a Java ServiceLoader, or
* listed in the configuration. Codecs specified in configuration come
* later in the returned list, and are considered to override those
* from the ServiceLoader.
* @param conf the configuration to look in
* @return a list of the Configuration classes or null if the attribute
* was not set
* @return a list of the {@link CompressionCodec} classes
*/
public static List<Class<? extends CompressionCodec>> getCodecClasses(Configuration conf) {
List<Class<? extends CompressionCodec>> result
= new ArrayList<Class<? extends CompressionCodec>>();
// Add codec classes discovered via service loading
for (CompressionCodec codec : CODEC_PROVIDERS) {
result.add(codec.getClass());
}
// Add codec classes from configuration
String codecsString = conf.get("io.compression.codecs");
if (codecsString != null) {
List<Class<? extends CompressionCodec>> result
= new ArrayList<Class<? extends CompressionCodec>>();
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
String codecSubstring = codecSplit.nextToken();
@ -123,14 +133,14 @@ public static List<Class<? extends CompressionCodec>> getCodecClasses(Configurat
}
}
}
return result;
} else {
return null;
}
return result;
}
/**
* Sets a list of codec classes in the configuration.
* Sets a list of codec classes in the configuration. In addition to any
* classes specified using this method, {@link CompressionCodec} classes on
* the classpath are discovered using a Java ServiceLoader.
* @param conf the configuration to modify
* @param classes the list of classes to set
*/
@ -151,21 +161,19 @@ public static void setCodecClasses(Configuration conf,
/**
* Find the codecs specified in the config value io.compression.codecs
* and register them. Defaults to gzip and zip.
* and register them. Defaults to gzip and deflate.
*/
public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
codecsByClassName = new HashMap<String, CompressionCodec>();
codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
if (codecClasses == null) {
if (codecClasses == null || codecClasses.isEmpty()) {
addCodec(new GzipCodec());
addCodec(new DefaultCodec());
} else {
Iterator<Class<? extends CompressionCodec>> itr = codecClasses.iterator();
while (itr.hasNext()) {
CompressionCodec codec = ReflectionUtils.newInstance(itr.next(), conf);
addCodec(codec);
for (Class<? extends CompressionCodec> codecClass : codecClasses) {
addCodec(ReflectionUtils.newInstance(codecClass, conf));
}
}
}

20
hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
org.apache.hadoop.io.compress.BZip2Codec
org.apache.hadoop.io.compress.DefaultCodec
org.apache.hadoop.io.compress.DeflateCodec
org.apache.hadoop.io.compress.GzipCodec
org.apache.hadoop.io.compress.Lz4Codec
org.apache.hadoop.io.compress.SnappyCodec

8
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -268,9 +268,11 @@
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
<description>A list of the compression codec classes that can be used
for compression/decompression.</description>
<value></value>
<description>A comma-separated list of the compression codec classes that can
be used for compression/decompression. In addition to any classes specified
with this property (which take precedence), codec classes on the classpath
are discovered using a Java ServiceLoader.</description>
</property>
<property>

43
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
View File

@ -101,6 +101,12 @@ public String getDefaultExtension() {
}
}
private static class NewGzipCodec extends BaseCodec {
public String getDefaultExtension() {
return ".gz";
}
}
/**
* Returns a factory for a given set of codecs
* @param classes the codec classes to include
@ -167,32 +173,43 @@ public static void testFinding() {
checkCodec("default factory for deflate codec", DeflateCodec.class, codec);
factory = setClasses(new Class[0]);
// gz, bz2, snappy, lz4 are picked up by service loader, but bar isn't
codec = factory.getCodec(new Path("/tmp/foo.bar"));
assertEquals("empty codec bar codec", null, codec);
assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName());
assertEquals("empty codec bar codec", null, codec);
assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodec(new Path("/tmp/foo.gz"));
assertEquals("empty codec gz codec", null, codec);
checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
assertEquals("empty codec gz codec", null, codec);
checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
assertEquals("empty factory for .bz2", null, codec);
checkCodec("empty factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
assertEquals("empty factory for bzip2 codec", null, codec);
checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.snappy"));
checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName());
checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.lz4"));
checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName());
checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
factory = setClasses(new Class[]{BarCodec.class, FooCodec.class,
FooBarCodec.class});
codec = factory.getCodec(new Path("/tmp/.foo.bar.gz"));
assertEquals("full factory gz codec", null, codec);
checkCodec("full factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
assertEquals("full codec gz codec", null, codec);
checkCodec("full codec gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
assertEquals("full factory for .bz2", null, codec);
checkCodec("full factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
assertEquals("full codec bzip2 codec", null, codec);
checkCodec("full codec bzip2 codec", BZip2Codec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bar"));
checkCodec("full factory bar codec", BarCodec.class, codec);
@ -220,5 +237,11 @@ public static void testFinding() {
checkCodec("full factory foo codec", FooCodec.class, codec);
codec = factory.getCodecByName("FOO");
checkCodec("full factory foo codec", FooCodec.class, codec);
factory = setClasses(new Class[]{NewGzipCodec.class});
codec = factory.getCodec(new Path("/tmp/foo.gz"));
checkCodec("overridden factory for .gz", NewGzipCodec.class, codec);
codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName());
checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
}
}