From 769194d5b39598ff6a5f6811b9eac926257dd173 Mon Sep 17 00:00:00 2001 From: Harsh J Date: Mon, 16 Jul 2012 09:03:24 +0000 Subject: [PATCH] Backport of HADOOP-8531 from trunk. SequenceFile Writer can throw out a better error if a serializer or deserializer isn't available. Contributed by Madhukara Phatak. (harsh) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1361943 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 4 ++ .../org/apache/hadoop/io/SequenceFile.java | 45 +++++++++++++++ .../apache/hadoop/io/TestSequenceFile.java | 55 +++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e5e58d2a313..46411ae7761 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -369,6 +369,10 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-7818. DiskChecker#checkDir should fail if the directory is not executable. (Madhukara Phatak via harsh) + HADOOP-8531. SequenceFile Writer can throw out a better error if a + serializer or deserializer isn't available + (Madhukara Phatak via harsh) + BUG FIXES HADOOP-8199. Fix issues in start-all.sh and stop-all.sh (Devaraj K via umamahesh) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index 8213a1b9336..6ffa436cea0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -1161,8 +1161,26 @@ public class SequenceFile { this.metadata = metadata; SerializationFactory serializationFactory = new SerializationFactory(conf); this.keySerializer = serializationFactory.getSerializer(keyClass); + if (this.keySerializer == null) { + throw new IOException( + "Could not find a serializer for the Key class: '" + + keyClass.getCanonicalName() + "'. " + + "Please ensure that the configuration '" + + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is " + + "properly configured, if you're using" + + "custom serialization."); + } this.keySerializer.open(buffer); this.uncompressedValSerializer = serializationFactory.getSerializer(valClass); + if (this.uncompressedValSerializer == null) { + throw new IOException( + "Could not find a serializer for the Value class: '" + + valClass.getCanonicalName() + "'. " + + "Please ensure that the configuration '" + + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is " + + "properly configured, if you're using" + + "custom serialization."); + } this.uncompressedValSerializer.open(buffer); if (this.codec != null) { ReflectionUtils.setConf(this.codec, this.conf); @@ -1171,6 +1189,15 @@ public class SequenceFile { this.deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter)); this.compressedValSerializer = serializationFactory.getSerializer(valClass); + if (this.compressedValSerializer == null) { + throw new IOException( + "Could not find a serializer for the Value class: '" + + valClass.getCanonicalName() + "'. " + + "Please ensure that the configuration '" + + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is " + + "properly configured, if you're using" + + "custom serialization."); + } this.compressedValSerializer.open(deflateOut); } writeFileHeader(); @@ -1898,6 +1925,15 @@ public class SequenceFile { new SerializationFactory(conf); this.keyDeserializer = getDeserializer(serializationFactory, getKeyClass()); + if (this.keyDeserializer == null) { + throw new IOException( + "Could not find a deserializer for the Key class: '" + + getKeyClass().getCanonicalName() + "'. " + + "Please ensure that the configuration '" + + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is " + + "properly configured, if you're using " + + "custom serialization."); + } if (!blockCompressed) { this.keyDeserializer.open(valBuffer); } else { @@ -1905,6 +1941,15 @@ public class SequenceFile { } this.valDeserializer = getDeserializer(serializationFactory, getValueClass()); + if (this.valDeserializer == null) { + throw new IOException( + "Could not find a deserializer for the Value class: '" + + getValueClass().getCanonicalName() + "'. " + + "Please ensure that the configuration '" + + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is " + + "properly configured, if you're using " + + "custom serialization."); + } this.valDeserializer.open(valIn); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java index 79b68b86c5e..fe33fefd91c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.SequenceFile.Metadata; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; +import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.conf.*; import org.mockito.Mockito; @@ -557,6 +558,60 @@ public class TestSequenceFile extends TestCase { // should succeed, fails if exception thrown } + public void testSerializationAvailability() throws IOException { + Configuration conf = new Configuration(); + Path path = new Path(System.getProperty("test.build.data", "."), + "serializationAvailability"); + // Check if any serializers aren't found. + try { + SequenceFile.createWriter( + conf, + SequenceFile.Writer.file(path), + SequenceFile.Writer.keyClass(String.class), + SequenceFile.Writer.valueClass(NullWritable.class)); + // Note: This may also fail someday if JavaSerialization + // is activated by default. + fail("Must throw IOException for missing serializer for the Key class"); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith( + "Could not find a serializer for the Key class: '" + + String.class.getName() + "'.")); + } + try { + SequenceFile.createWriter( + conf, + SequenceFile.Writer.file(path), + SequenceFile.Writer.keyClass(NullWritable.class), + SequenceFile.Writer.valueClass(String.class)); + // Note: This may also fail someday if JavaSerialization + // is activated by default. + fail("Must throw IOException for missing serializer for the Value class"); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith( + "Could not find a serializer for the Value class: '" + + String.class.getName() + "'.")); + } + + // Write a simple file to test deserialization failures with + writeTest(FileSystem.get(conf), 1, 1, path, CompressionType.NONE, null); + + // Remove Writable serializations, to enforce error. + conf.setStrings(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, + AvroReflectSerialization.class.getName()); + + // Now check if any deserializers aren't found. + try { + new SequenceFile.Reader( + conf, + SequenceFile.Reader.file(path)); + fail("Must throw IOException for missing deserializer for the Key class"); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith( + "Could not find a deserializer for the Key class: '" + + RandomDatum.class.getName() + "'.")); + } + } + /** For debugging and testing. */ public static void main(String[] args) throws Exception { int count = 1024 * 1024;