diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ce4e3242cf8..5caa5dcdff7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -360,6 +360,9 @@ Release 2.0.0 - UNRELEASED HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh existence for starting start-yarn.sh. (Devaraj K via eli) + HADOOP-7350. Use ServiceLoader to discover compression codec classes. + (tomwhite) + BREAKDOWN OF HADOOP-7454 SUBTASKS HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java index 6fd443ebb1a..e12dcfeeb4a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java @@ -36,6 +36,9 @@ public class CompressionCodecFactory { public static final Log LOG = LogFactory.getLog(CompressionCodecFactory.class.getName()); + + private static final ServiceLoader CODEC_PROVIDERS = + ServiceLoader.load(CompressionCodec.class); /** * A map from the reversed filename suffixes to the codecs. @@ -95,16 +98,23 @@ public String toString() { } /** - * Get the list of codecs listed in the configuration + * Get the list of codecs discovered via a Java ServiceLoader, or + * listed in the configuration. Codecs specified in configuration come + * later in the returned list, and are considered to override those + * from the ServiceLoader. * @param conf the configuration to look in - * @return a list of the Configuration classes or null if the attribute - * was not set + * @return a list of the {@link CompressionCodec} classes */ public static List> getCodecClasses(Configuration conf) { + List> result + = new ArrayList>(); + // Add codec classes discovered via service loading + for (CompressionCodec codec : CODEC_PROVIDERS) { + result.add(codec.getClass()); + } + // Add codec classes from configuration String codecsString = conf.get("io.compression.codecs"); if (codecsString != null) { - List> result - = new ArrayList>(); StringTokenizer codecSplit = new StringTokenizer(codecsString, ","); while (codecSplit.hasMoreElements()) { String codecSubstring = codecSplit.nextToken(); @@ -123,14 +133,14 @@ public static List> getCodecClasses(Configurat } } } - return result; - } else { - return null; } + return result; } /** - * Sets a list of codec classes in the configuration. + * Sets a list of codec classes in the configuration. In addition to any + * classes specified using this method, {@link CompressionCodec} classes on + * the classpath are discovered using a Java ServiceLoader. * @param conf the configuration to modify * @param classes the list of classes to set */ @@ -151,21 +161,19 @@ public static void setCodecClasses(Configuration conf, /** * Find the codecs specified in the config value io.compression.codecs - * and register them. Defaults to gzip and zip. + * and register them. Defaults to gzip and deflate. */ public CompressionCodecFactory(Configuration conf) { codecs = new TreeMap(); codecsByClassName = new HashMap(); codecsByName = new HashMap(); List> codecClasses = getCodecClasses(conf); - if (codecClasses == null) { + if (codecClasses == null || codecClasses.isEmpty()) { addCodec(new GzipCodec()); addCodec(new DefaultCodec()); } else { - Iterator> itr = codecClasses.iterator(); - while (itr.hasNext()) { - CompressionCodec codec = ReflectionUtils.newInstance(itr.next(), conf); - addCodec(codec); + for (Class codecClass : codecClasses) { + addCodec(ReflectionUtils.newInstance(codecClass, conf)); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec new file mode 100644 index 00000000000..df46e32b3ba --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -0,0 +1,20 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.io.compress.BZip2Codec +org.apache.hadoop.io.compress.DefaultCodec +org.apache.hadoop.io.compress.DeflateCodec +org.apache.hadoop.io.compress.GzipCodec +org.apache.hadoop.io.compress.Lz4Codec +org.apache.hadoop.io.compress.SnappyCodec + diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 1202002ed5f..a1a91877920 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -268,9 +268,11 @@ io.compression.codecs - org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec - A list of the compression codec classes that can be used - for compression/decompression. + + A comma-separated list of the compression codec classes that can + be used for compression/decompression. In addition to any classes specified + with this property (which take precedence), codec classes on the classpath + are discovered using a Java ServiceLoader. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java index c7bcc0faa14..2caef859e64 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java @@ -101,6 +101,12 @@ public String getDefaultExtension() { } } + private static class NewGzipCodec extends BaseCodec { + public String getDefaultExtension() { + return ".gz"; + } + } + /** * Returns a factory for a given set of codecs * @param classes the codec classes to include @@ -167,32 +173,43 @@ public static void testFinding() { checkCodec("default factory for deflate codec", DeflateCodec.class, codec); factory = setClasses(new Class[0]); + // gz, bz2, snappy, lz4 are picked up by service loader, but bar isn't codec = factory.getCodec(new Path("/tmp/foo.bar")); - assertEquals("empty codec bar codec", null, codec); + assertEquals("empty factory bar codec", null, codec); codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName()); - assertEquals("empty codec bar codec", null, codec); + assertEquals("empty factory bar codec", null, codec); codec = factory.getCodec(new Path("/tmp/foo.gz")); - assertEquals("empty codec gz codec", null, codec); + checkCodec("empty factory gz codec", GzipCodec.class, codec); codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName()); - assertEquals("empty codec gz codec", null, codec); + checkCodec("empty factory gz codec", GzipCodec.class, codec); codec = factory.getCodec(new Path("/tmp/foo.bz2")); - assertEquals("empty factory for .bz2", null, codec); + checkCodec("empty factory for .bz2", BZip2Codec.class, codec); codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName()); - assertEquals("empty factory for bzip2 codec", null, codec); + checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec); + + codec = factory.getCodec(new Path("/tmp/foo.snappy")); + checkCodec("empty factory snappy codec", SnappyCodec.class, codec); + codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName()); + checkCodec("empty factory snappy codec", SnappyCodec.class, codec); + + codec = factory.getCodec(new Path("/tmp/foo.lz4")); + checkCodec("empty factory lz4 codec", Lz4Codec.class, codec); + codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName()); + checkCodec("empty factory lz4 codec", Lz4Codec.class, codec); factory = setClasses(new Class[]{BarCodec.class, FooCodec.class, FooBarCodec.class}); codec = factory.getCodec(new Path("/tmp/.foo.bar.gz")); - assertEquals("full factory gz codec", null, codec); + checkCodec("full factory gz codec", GzipCodec.class, codec); codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName()); - assertEquals("full codec gz codec", null, codec); + checkCodec("full codec gz codec", GzipCodec.class, codec); codec = factory.getCodec(new Path("/tmp/foo.bz2")); - assertEquals("full factory for .bz2", null, codec); + checkCodec("full factory for .bz2", BZip2Codec.class, codec); codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName()); - assertEquals("full codec bzip2 codec", null, codec); + checkCodec("full codec bzip2 codec", BZip2Codec.class, codec); codec = factory.getCodec(new Path("/tmp/foo.bar")); checkCodec("full factory bar codec", BarCodec.class, codec); @@ -220,5 +237,11 @@ public static void testFinding() { checkCodec("full factory foo codec", FooCodec.class, codec); codec = factory.getCodecByName("FOO"); checkCodec("full factory foo codec", FooCodec.class, codec); + + factory = setClasses(new Class[]{NewGzipCodec.class}); + codec = factory.getCodec(new Path("/tmp/foo.gz")); + checkCodec("overridden factory for .gz", NewGzipCodec.class, codec); + codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName()); + checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec); } }