diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml deleted file mode 100644 index a7da364e72f..00000000000 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - hadoop-raid-dist - - dir - - false - - - - ${basedir}/src/main/conf - /etc/hadoop - - * - - - - ${basedir}/src/main/sbin - /sbin - - * - - 0755 - - - ${basedir}/src/main/libexec - /libexec - - * - - 0755 - - - - ${project.build.directory}/site - /share/doc/hadoop/raid - - - - - /share/hadoop/${hadoop.component}/lib - false - runtime - true - - - diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 442cdcac333..7fc6e563dbf 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -52,11 +52,6 @@ hadoop-yarn-api provided - - org.apache.hadoop - hadoop-hdfs-raid - provided - @@ -125,7 +120,6 @@ run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* . - run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-raid/target/hadoop-hdfs-raid-${project.version}/* . run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* . run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* . echo diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml deleted file mode 100644 index a7fa6eb6562..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-project-dist - 3.0.0-SNAPSHOT - ../../hadoop-project-dist - - org.apache.hadoop - hadoop-hdfs-raid - 3.0.0-SNAPSHOT - jar - - Apache Hadoop HDFS Raid - Apache Hadoop HDFS Raid - - - - raid - false - - - - - junit - junit - test - - - org.apache.hadoop - hadoop-annotations - provided - - - org.apache.hadoop - hadoop-minicluster - test - - - org.apache.hadoop - hadoop-client - provided - - - org.apache.hadoop - hadoop-archives - provided - - - - - - - - maven-dependency-plugin - - - create-mrapp-generated-classpath - generate-test-resources - - build-classpath - - - - ${project.build.directory}/test-classes/mrapp-generated-classpath - - - - - - org.apache.rat - apache-rat-plugin - - - - - - - org.codehaus.mojo - findbugs-maven-plugin - - - - - - - - - - docs - - false - - - - - org.apache.maven.plugins - maven-site-plugin - - - docs - prepare-package - - site - - - - - - - - - - dist - - false - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.hadoop - hadoop-assemblies - ${project.version} - - - - - dist - prepare-package - - single - - - ${project.artifactId}-${project.version} - false - false - - hadoop-raid-dist - - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml deleted file mode 100644 index 3cf2dc66941..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - srcReplication - 3 - pick files for RAID only if their replication factor is - greater than or equal to this value. - - - - targetReplication - 2 - after RAIDing, decrease the replication factor of a file to - this value. - - - - metaReplication - 2 - the replication factor of the RAID meta file - - - - modTimePeriod - 3600000 - time (milliseconds) after a file is modified to make it a - candidate for RAIDing - - - - - - - - targetReplication - 1 - after RAIDing, decrease the replication factor of a file to - this value. - - - - metaReplication - 2 - the replication factor of the RAID meta file - - - - modTimePeriod - 3600000 - time (milliseconds) after a file is modified to make it a - candidate for RAIDing - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java deleted file mode 100644 index 985774bcdcc..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java +++ /dev/null @@ -1,509 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import java.io.IOException; -import java.io.PrintStream; -import java.net.URI; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Random; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FilterFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.raid.Decoder; -import org.apache.hadoop.raid.RaidNode; -import org.apache.hadoop.raid.ReedSolomonDecoder; -import org.apache.hadoop.raid.XORDecoder; -import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * This is an implementation of the Hadoop RAID Filesystem. This FileSystem - * wraps an instance of the DistributedFileSystem. - * If a file is corrupted, this FileSystem uses the parity blocks to - * regenerate the bad block. - */ - -public class DistributedRaidFileSystem extends FilterFileSystem { - - // these are alternate locations that can be used for read-only access - DecodeInfo[] alternates; - Configuration conf; - int stripeLength; - - DistributedRaidFileSystem() throws IOException { - } - - DistributedRaidFileSystem(FileSystem fs) throws IOException { - super(fs); - alternates = null; - stripeLength = 0; - } - - // Information required for decoding a source file - static private class DecodeInfo { - final Path destPath; - final ErasureCodeType type; - final Configuration conf; - final int stripeLength; - private DecodeInfo(Configuration conf, ErasureCodeType type, Path destPath) { - this.conf = conf; - this.type = type; - this.destPath = destPath; - this.stripeLength = RaidNode.getStripeLength(conf); - } - - Decoder createDecoder() { - if (this.type == ErasureCodeType.XOR) { - return new XORDecoder(conf, stripeLength); - } else if (this.type == ErasureCodeType.RS) { - return new ReedSolomonDecoder(conf, stripeLength, - RaidNode.rsParityLength(conf)); - } - return null; - } - } - - /* Initialize a Raid FileSystem - */ - public void initialize(URI name, Configuration conf) throws IOException { - this.conf = conf; - - Class clazz = conf.getClass("fs.raid.underlyingfs.impl", - DistributedFileSystem.class); - if (clazz == null) { - throw new IOException("No FileSystem for fs.raid.underlyingfs.impl."); - } - - this.fs = (FileSystem)ReflectionUtils.newInstance(clazz, null); - super.initialize(name, conf); - - // find stripe length configured - stripeLength = RaidNode.getStripeLength(conf); - if (stripeLength == 0) { - LOG.info("dfs.raid.stripeLength is incorrectly defined to be " + - stripeLength + " Ignoring..."); - return; - } - - // Put XOR and RS in alternates - alternates= new DecodeInfo[2]; - Path xorPath = RaidNode.xorDestinationPath(conf, fs); - alternates[0] = new DecodeInfo(conf, ErasureCodeType.XOR, xorPath); - Path rsPath = RaidNode.rsDestinationPath(conf, fs); - alternates[1] = new DecodeInfo(conf, ErasureCodeType.RS, rsPath); - } - - /* - * Returns the underlying filesystem - */ - public FileSystem getFileSystem() throws IOException { - return fs; - } - - @Override - public FSDataInputStream open(Path f, int bufferSize) throws IOException { - ExtFSDataInputStream fd = new ExtFSDataInputStream(conf, this, alternates, f, - stripeLength, bufferSize); - return fd; - } - - public void close() throws IOException { - if (fs != null) { - try { - fs.close(); - } catch(IOException ie) { - //this might already be closed, ignore - } - } - super.close(); - } - - /** - * Layered filesystem input stream. This input stream tries reading - * from alternate locations if it encoumters read errors in the primary location. - */ - private static class ExtFSDataInputStream extends FSDataInputStream { - - private static class UnderlyingBlock { - // File that holds this block. Need not be the same as outer file. - public Path path; - // Offset within path where this block starts. - public long actualFileOffset; - // Offset within the outer file where this block starts. - public long originalFileOffset; - // Length of the block (length <= blk sz of outer file). - public long length; - public UnderlyingBlock(Path path, long actualFileOffset, - long originalFileOffset, long length) { - this.path = path; - this.actualFileOffset = actualFileOffset; - this.originalFileOffset = originalFileOffset; - this.length = length; - } - } - - /** - * Create an input stream that wraps all the reads/positions/seeking. - */ - private static class ExtFsInputStream extends FSInputStream { - - // Extents of "good" underlying data that can be read. - private UnderlyingBlock[] underlyingBlocks; - private long currentOffset; - private FSDataInputStream currentStream; - private UnderlyingBlock currentBlock; - private byte[] oneBytebuff = new byte[1]; - private int nextLocation; - private DistributedRaidFileSystem lfs; - private Path path; - private FileStatus stat; - private final DecodeInfo[] alternates; - private final int buffersize; - private final Configuration conf; - private final int stripeLength; - - ExtFsInputStream(Configuration conf, DistributedRaidFileSystem lfs, - DecodeInfo[] alternates, Path path, int stripeLength, int buffersize) - throws IOException { - this.path = path; - this.nextLocation = 0; - // Construct array of blocks in file. - this.stat = lfs.getFileStatus(path); - long numBlocks = (this.stat.getLen() % this.stat.getBlockSize() == 0) ? - this.stat.getLen() / this.stat.getBlockSize() : - 1 + this.stat.getLen() / this.stat.getBlockSize(); - this.underlyingBlocks = new UnderlyingBlock[(int)numBlocks]; - for (int i = 0; i < numBlocks; i++) { - long actualFileOffset = i * stat.getBlockSize(); - long originalFileOffset = i * stat.getBlockSize(); - long length = Math.min( - stat.getBlockSize(), stat.getLen() - originalFileOffset); - this.underlyingBlocks[i] = new UnderlyingBlock( - path, actualFileOffset, originalFileOffset, length); - } - this.currentOffset = 0; - this.currentBlock = null; - this.alternates = alternates; - this.buffersize = buffersize; - this.conf = conf; - this.lfs = lfs; - this.stripeLength = stripeLength; - // Open a stream to the first block. - openCurrentStream(); - } - - private void closeCurrentStream() throws IOException { - if (currentStream != null) { - currentStream.close(); - currentStream = null; - } - } - - /** - * Open a stream to the file containing the current block - * and seek to the appropriate offset - */ - private void openCurrentStream() throws IOException { - int blockIdx = (int)(currentOffset/stat.getBlockSize()); - UnderlyingBlock block = underlyingBlocks[blockIdx]; - // If the current path is the same as we want. - if (currentBlock == block || - currentBlock != null && currentBlock.path == block.path) { - // If we have a valid stream, nothing to do. - if (currentStream != null) { - currentBlock = block; - return; - } - } else { - closeCurrentStream(); - } - currentBlock = block; - currentStream = lfs.fs.open(currentBlock.path, buffersize); - long offset = block.actualFileOffset + - (currentOffset - block.originalFileOffset); - currentStream.seek(offset); - } - - /** - * Returns the number of bytes available in the current block. - */ - private int blockAvailable() { - return (int) (currentBlock.length - - (currentOffset - currentBlock.originalFileOffset)); - } - - @Override - public synchronized int available() throws IOException { - // Application should not assume that any bytes are buffered here. - nextLocation = 0; - return Math.min(blockAvailable(), currentStream.available()); - } - - @Override - public synchronized void close() throws IOException { - closeCurrentStream(); - super.close(); - } - - @Override - public boolean markSupported() { return false; } - - @Override - public void mark(int readLimit) { - // Mark and reset are not supported. - nextLocation = 0; - } - - @Override - public void reset() throws IOException { - // Mark and reset are not supported. - nextLocation = 0; - } - - @Override - public synchronized int read() throws IOException { - int value = read(oneBytebuff); - if (value < 0) { - return value; - } else { - return oneBytebuff[0]; - } - } - - @Override - public synchronized int read(byte[] b) throws IOException { - int value = read(b, 0, b.length); - nextLocation = 0; - return value; - } - - @Override - public synchronized int read(byte[] b, int offset, int len) - throws IOException { - while (true) { - openCurrentStream(); - try{ - int limit = Math.min(blockAvailable(), len); - int value = currentStream.read(b, offset, limit); - currentOffset += value; - nextLocation = 0; - return value; - } catch (BlockMissingException e) { - setAlternateLocations(e, currentOffset); - } catch (ChecksumException e) { - setAlternateLocations(e, currentOffset); - } - } - } - - @Override - public synchronized int read(long position, byte[] b, int offset, int len) - throws IOException { - long oldPos = currentOffset; - seek(position); - try { - return read(b, offset, len); - } finally { - seek(oldPos); - } - } - - @Override - public synchronized long skip(long n) throws IOException { - long skipped = 0; - while (skipped < n) { - int val = read(); - if (val < 0) { - break; - } - skipped++; - } - nextLocation = 0; - return skipped; - } - - @Override - public synchronized long getPos() throws IOException { - nextLocation = 0; - return currentOffset; - } - - @Override - public synchronized void seek(long pos) throws IOException { - if (pos != currentOffset) { - closeCurrentStream(); - currentOffset = pos; - openCurrentStream(); - } - nextLocation = 0; - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - seek(targetPos); - boolean value = currentStream.seekToNewSource(currentStream.getPos()); - nextLocation = 0; - return value; - } - - /** - * position readable again. - */ - @Override - public void readFully(long pos, byte[] b, int offset, int length) - throws IOException { - long oldPos = currentOffset; - seek(pos); - try { - while (true) { - // This loop retries reading until successful. Unrecoverable errors - // cause exceptions. - // currentOffset is changed by read(). - try { - while (length > 0) { - int n = read(b, offset, length); - if (n < 0) { - throw new IOException("Premature EOF"); - } - offset += n; - length -= n; - } - nextLocation = 0; - return; - } catch (BlockMissingException e) { - setAlternateLocations(e, currentOffset); - } catch (ChecksumException e) { - setAlternateLocations(e, currentOffset); - } - } - } finally { - seek(oldPos); - } - } - - @Override - public void readFully(long pos, byte[] b) throws IOException { - readFully(pos, b, 0, b.length); - nextLocation = 0; - } - - /** - * Extract good block from RAID - * @throws IOException if all alternate locations are exhausted - */ - private void setAlternateLocations(IOException curexp, long offset) - throws IOException { - while (alternates != null && nextLocation < alternates.length) { - try { - int idx = nextLocation++; - // Start offset of block. - long corruptOffset = - (offset / stat.getBlockSize()) * stat.getBlockSize(); - // Make sure we use DFS and not DistributedRaidFileSystem for unRaid. - Configuration clientConf = new Configuration(conf); - Class clazz = conf.getClass("fs.raid.underlyingfs.impl", - DistributedFileSystem.class); - clientConf.set("fs.hdfs.impl", clazz.getName()); - // Disable caching so that a previously cached RaidDfs is not used. - clientConf.setBoolean("fs.hdfs.impl.disable.cache", true); - Path npath = RaidNode.unRaidCorruptBlock(clientConf, path, - alternates[idx].destPath, - alternates[idx].createDecoder(), - stripeLength, corruptOffset); - if (npath == null) - continue; - try { - String outdir = conf.get("fs.raid.recoverylogdir"); - if (outdir != null) { - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - java.util.Date date = new java.util.Date(); - String fname = path.getName() + dateFormat.format(date) + - (new Random()).nextInt() + ".txt"; - Path outputunraid = new Path(outdir, fname); - FileSystem fs = outputunraid.getFileSystem(conf); - FSDataOutputStream dout = fs.create(outputunraid); - PrintStream ps = new PrintStream(dout); - ps.println("Recovery attempt log"); - ps.println("Source path : " + path ); - ps.println("Alternate path : " + alternates[idx].destPath); - ps.println("Stripe lentgh : " + stripeLength); - ps.println("Corrupt offset : " + corruptOffset); - String output = (npath==null) ? "UNSUCCESSFUL" : npath.toString(); - ps.println("Output from unRaid : " + output); - ps.close(); - } - } catch (Exception exc) { - LOG.info("Error while creating recovery log: " + exc); - } - - closeCurrentStream(); - LOG.info("Using block at offset " + corruptOffset + " from " + - npath); - currentBlock.path = npath; - currentBlock.actualFileOffset = 0; // Single block in file. - // Dont change currentOffset, in case the user had done a seek? - openCurrentStream(); - - return; - } catch (Exception e) { - LOG.info("Error in using alternate path " + path + ". " + e + - " Ignoring..."); - } - } - throw curexp; - } - - /** - * The name of the file system that is immediately below the - * DistributedRaidFileSystem. This is specified by the - * configuration parameter called fs.raid.underlyingfs.impl. - * If this parameter is not specified in the configuration, then - * the default class DistributedFileSystem is returned. - * @param conf the configuration object - * @return the filesystem object immediately below DistributedRaidFileSystem - * @throws IOException if all alternate locations are exhausted - */ - private FileSystem getUnderlyingFileSystem(Configuration conf) { - Class clazz = conf.getClass("fs.raid.underlyingfs.impl", DistributedFileSystem.class); - FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf); - return fs; - } - } - - /** - * constructor for ext input stream. - * @param fs the underlying filesystem - * @param p the path in the underlying file system - * @param buffersize the size of IO - * @throws IOException - */ - public ExtFSDataInputStream(Configuration conf, DistributedRaidFileSystem lfs, - DecodeInfo[] alternates, Path p, int stripeLength, int buffersize) throws IOException { - super(new ExtFsInputStream(conf, lfs, alternates, p, stripeLength, buffersize)); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java deleted file mode 100644 index 58dfa34fe69..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs; - -import java.io.IOException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.LinkedList; -import java.util.List; -import java.util.HashSet; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.tools.DFSck; -import org.apache.hadoop.util.ToolRunner; - -public abstract class RaidDFSUtil { - /** - * Returns the corrupt blocks in a file. - */ - public static List corruptBlocksInFile( - DistributedFileSystem dfs, String path, long offset, long length) - throws IOException { - List corrupt = new LinkedList(); - LocatedBlocks locatedBlocks = - getBlockLocations(dfs, path, offset, length); - for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) { - if (b.isCorrupt() || - (b.getLocations().length == 0 && b.getBlockSize() > 0)) { - corrupt.add(b); - } - } - return corrupt; - } - - public static LocatedBlocks getBlockLocations( - DistributedFileSystem dfs, String path, long offset, long length) - throws IOException { - return dfs.getClient().namenode.getBlockLocations(path, offset, length); - } - - /** - * Make successive calls to listCorruptFiles to obtain all - * corrupt files. - */ - public static String[] getCorruptFiles(DistributedFileSystem dfs) - throws IOException { - Set corruptFiles = new HashSet(); - RemoteIterator cfb = dfs.listCorruptFileBlocks(new Path("/")); - while (cfb.hasNext()) { - corruptFiles.add(cfb.next().toUri().getPath()); - } - - return corruptFiles.toArray(new String[corruptFiles.size()]); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java deleted file mode 100644 index de0bee60645..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java +++ /dev/null @@ -1,632 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.blockmanagement; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.LinkedHashMap; -import java.util.ArrayList; -import java.util.Map; -import java.util.Comparator; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.server.namenode.*; -import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.net.Node; -import org.apache.hadoop.raid.RaidNode; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; - -/** - * This BlockPlacementPolicy spreads out the group of blocks which used by RAID - * for recovering each other. This is important for the availability - * of the blocks. This class can be used by multiple threads. It has to be - * thread safe. - */ -public class BlockPlacementPolicyRaid extends BlockPlacementPolicy { - public static final Log LOG = - LogFactory.getLog(BlockPlacementPolicyRaid.class); - Configuration conf; - private int stripeLength; - private int xorParityLength; - private int rsParityLength; - private String xorPrefix = null; - private String rsPrefix = null; - private String raidTempPrefix = null; - private String raidrsTempPrefix = null; - private String raidHarTempPrefix = null; - private String raidrsHarTempPrefix = null; - private FSNamesystem namesystem = null; - private BlockPlacementPolicyDefault defaultPolicy; - - CachedLocatedBlocks cachedLocatedBlocks; - CachedFullPathNames cachedFullPathNames; - - /** {@inheritDoc} */ - @Override - public void initialize(Configuration conf, FSClusterStats stats, - NetworkTopology clusterMap) { - this.conf = conf; - this.stripeLength = RaidNode.getStripeLength(conf); - this.rsParityLength = RaidNode.rsParityLength(conf); - this.xorParityLength = 1; - try { - this.xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath(); - this.rsPrefix = RaidNode.rsDestinationPath(conf).toUri().getPath(); - } catch (IOException e) { - } - if (this.xorPrefix == null) { - this.xorPrefix = RaidNode.DEFAULT_RAID_LOCATION; - } - if (this.rsPrefix == null) { - this.rsPrefix = RaidNode.DEFAULT_RAIDRS_LOCATION; - } - // Throws ClassCastException if we cannot cast here. - this.namesystem = (FSNamesystem) stats; - this.cachedLocatedBlocks = new CachedLocatedBlocks(namesystem); - this.cachedFullPathNames = new CachedFullPathNames(namesystem); - this.raidTempPrefix = RaidNode.xorTempPrefix(conf); - this.raidrsTempPrefix = RaidNode.rsTempPrefix(conf); - this.raidHarTempPrefix = RaidNode.xorHarTempPrefix(conf); - this.raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf); - defaultPolicy = new BlockPlacementPolicyDefault(conf, stats, clusterMap); - } - - @Override - DatanodeDescriptor[] chooseTarget(String srcPath, int numOfReplicas, - DatanodeDescriptor writer, List chosenNodes, - long blocksize) { - return chooseTarget(srcPath, numOfReplicas, writer, chosenNodes, - null, blocksize); - } - - @Override - public DatanodeDescriptor[] chooseTarget(String srcPath, int numOfReplicas, - DatanodeDescriptor writer, List chosenNodes, - boolean returnChosenNodes, - HashMap excludedNodes, long blocksize) { - try { - FileType type = getFileType(srcPath); - if (type == FileType.NOT_RAID) { - return defaultPolicy.chooseTarget( - srcPath, numOfReplicas, writer, chosenNodes, blocksize); - } - if (excludedNodes == null) { - excludedNodes = new HashMap(); - } - addExcludedNodes(srcPath, type, excludedNodes); - DatanodeDescriptor[] result = - defaultPolicy.chooseTarget(numOfReplicas, writer, - chosenNodes, returnChosenNodes, excludedNodes, blocksize); - // Add the added block locations in the block locations cache. - // So the rest of the blocks know about these locations. - cachedLocatedBlocks.get(srcPath). - add(new LocatedBlock(new ExtendedBlock(), result)); - return result; - } catch (Exception e) { - LOG.debug("Error happend when choosing datanode to write:" + - StringUtils.stringifyException(e)); - return defaultPolicy.chooseTarget(srcPath, numOfReplicas, writer, - chosenNodes, blocksize); - } - } - - @Override - public int verifyBlockPlacement(String srcPath, LocatedBlock lBlk, - int minRacks) { - return defaultPolicy.verifyBlockPlacement(srcPath, lBlk, minRacks); - } - - /** {@inheritDoc} */ - @Override - public DatanodeDescriptor chooseReplicaToDelete(BlockCollection bc, - Block block, short replicationFactor, - Collection first, - Collection second) { - - DatanodeDescriptor chosenNode = null; - try { - String path = cachedFullPathNames.get(bc); - FileType type = getFileType(path); - if (type == FileType.NOT_RAID) { - return defaultPolicy.chooseReplicaToDelete( - bc, block, replicationFactor, first, second); - } - List companionBlocks = - getCompanionBlocks(path, type, block); - if (companionBlocks == null || companionBlocks.size() == 0) { - // Use the default method if it is not a valid raided or parity file - return defaultPolicy.chooseReplicaToDelete( - bc, block, replicationFactor, first, second); - } - // Delete from the first collection first - // This ensures the number of unique rack of this block is not reduced - Collection all = new HashSet(); - all.addAll(first); - all.addAll(second); - chosenNode = chooseReplicaToDelete(companionBlocks, all); - if (chosenNode != null) { - return chosenNode; - } - return defaultPolicy.chooseReplicaToDelete( - bc, block, replicationFactor, first, second); - } catch (Exception e) { - LOG.debug("Error happend when choosing replica to delete" + - StringUtils.stringifyException(e)); - return defaultPolicy.chooseReplicaToDelete( - bc, block, replicationFactor, first, second); - } - } - - /** - * Obtain the excluded nodes for the current block that is being written - */ - void addExcludedNodes(String file, FileType type, HashMap excluded) - throws IOException { - Collection blocks = getCompanionBlocks(file, type, null); - if (blocks == null) { - return; - } - for (LocatedBlock b : blocks) { - for (Node n : b.getLocations()) { - excluded.put(n, n); - } - } - } - - private DatanodeDescriptor chooseReplicaToDelete( - Collection companionBlocks, - Collection dataNodes) throws IOException { - - if (dataNodes.isEmpty()) { - return null; - } - // Count the number of replicas on each node and rack - final Map nodeCompanionBlockCount = - countCompanionBlocks(companionBlocks, false); - final Map rackCompanionBlockCount = - countCompanionBlocks(companionBlocks, true); - - NodeComparator comparator = - new NodeComparator(nodeCompanionBlockCount, rackCompanionBlockCount); - return Collections.max(dataNodes, comparator); - } - - /** - * Count how many companion blocks are on each datanode or the each rack - * @param companionBlocks a collection of all the companion blocks - * @param doRackCount count the companion blocks on the racks of datanodes - * @param result the map from node name to the number of companion blocks - */ - static Map countCompanionBlocks( - Collection companionBlocks, boolean doRackCount) { - Map result = new HashMap(); - for (LocatedBlock block : companionBlocks) { - for (DatanodeInfo d : block.getLocations()) { - String name = doRackCount ? d.getParent().getName() : d.getName(); - if (result.containsKey(name)) { - int count = result.get(name) + 1; - result.put(name, count); - } else { - result.put(name, 1); - } - } - } - return result; - } - - /** - * Compares the datanodes based on the number of companion blocks on the same - * node and rack. If even, compare the remaining space on the datanodes. - */ - class NodeComparator implements Comparator { - private Map nodeBlockCount; - private Map rackBlockCount; - private NodeComparator(Map nodeBlockCount, - Map rackBlockCount) { - this.nodeBlockCount = nodeBlockCount; - this.rackBlockCount = rackBlockCount; - } - @Override - public int compare(DatanodeDescriptor d1, DatanodeDescriptor d2) { - int res = compareBlockCount(d1, d2, nodeBlockCount); - if (res != 0) { - return res; - } - res = compareBlockCount(d1.getParent(), d2.getParent(), rackBlockCount); - if (res != 0) { - return res; - } - if (d1.getRemaining() > d2.getRemaining()) { - return -1; - } - if (d1.getRemaining() < d2.getRemaining()) { - return 1; - } - return 0; - } - private int compareBlockCount(Node node1, Node node2, - Map blockCount) { - Integer count1 = blockCount.get(node1.getName()); - Integer count2 = blockCount.get(node2.getName()); - count1 = count1 == null ? 0 : count1; - count2 = count2 == null ? 0 : count2; - if (count1 > count2) { - return 1; - } - if (count1 < count2) { - return -1; - } - return 0; - } - } - - /** - * Obtain the companion blocks of the give block - * Companion blocks are defined as the blocks that can help recover each - * others by using raid decoder. - * @param path The path of the file contains the block - * @param type The type of this file - * @param block The given block - * null if it is the block which is currently being written to - * @return the block locations of companion blocks - */ - List getCompanionBlocks(String path, FileType type, - Block block) throws IOException { - switch (type) { - case NOT_RAID: - return new ArrayList(); - case XOR_HAR_TEMP_PARITY: - return getCompanionBlocksForHarParityBlock( - path, xorParityLength, block); - case RS_HAR_TEMP_PARITY: - return getCompanionBlocksForHarParityBlock( - path, rsParityLength, block); - case XOR_TEMP_PARITY: - return getCompanionBlocksForParityBlock( - getSourceFile(path, raidTempPrefix), path, xorParityLength, block); - case RS_TEMP_PARITY: - return getCompanionBlocksForParityBlock( - getSourceFile(path, raidrsTempPrefix), path, rsParityLength, block); - case XOR_PARITY: - return getCompanionBlocksForParityBlock(getSourceFile(path, xorPrefix), - path, xorParityLength, block); - case RS_PARITY: - return getCompanionBlocksForParityBlock(getSourceFile(path, rsPrefix), - path, rsParityLength, block); - case XOR_SOURCE: - return getCompanionBlocksForSourceBlock( - path, getParityFile(path), xorParityLength, block); - case RS_SOURCE: - return getCompanionBlocksForSourceBlock( - path, getParityFile(path), xorParityLength, block); - } - return new ArrayList(); - } - - private List getCompanionBlocksForHarParityBlock( - String parity, int parityLength, Block block) - throws IOException { - int blockIndex = getBlockIndex(parity, block); - // consider only parity file in this case because source file block - // location is not easy to obtain - List parityBlocks = cachedLocatedBlocks.get(parity); - List result = new ArrayList(); - synchronized (parityBlocks) { - int start = Math.max(0, blockIndex - parityLength + 1); - int end = Math.min(parityBlocks.size(), blockIndex + parityLength); - result.addAll(parityBlocks.subList(start, end)); - } - return result; - } - - private List getCompanionBlocksForParityBlock( - String src, String parity, int parityLength, Block block) - throws IOException { - int blockIndex = getBlockIndex(parity, block); - List result = new ArrayList(); - List parityBlocks = cachedLocatedBlocks.get(parity); - int stripeIndex = blockIndex / parityLength; - synchronized (parityBlocks) { - int parityStart = stripeIndex * parityLength; - int parityEnd = Math.min(parityStart + parityLength, - parityBlocks.size()); - // for parity, always consider the neighbor blocks as companion blocks - if (parityStart < parityBlocks.size()) { - result.addAll(parityBlocks.subList(parityStart, parityEnd)); - } - } - - if (src == null) { - return result; - } - List sourceBlocks = cachedLocatedBlocks.get(src); - synchronized (sourceBlocks) { - int sourceStart = stripeIndex * stripeLength; - int sourceEnd = Math.min(sourceStart + stripeLength, - sourceBlocks.size()); - if (sourceStart < sourceBlocks.size()) { - result.addAll(sourceBlocks.subList(sourceStart, sourceEnd)); - } - } - return result; - } - - private List getCompanionBlocksForSourceBlock( - String src, String parity, int parityLength, Block block) - throws IOException { - int blockIndex = getBlockIndex(src, block); - List result = new ArrayList(); - List sourceBlocks = cachedLocatedBlocks.get(src); - int stripeIndex = blockIndex / stripeLength; - synchronized (sourceBlocks) { - int sourceStart = stripeIndex * stripeLength; - int sourceEnd = Math.min(sourceStart + stripeLength, - sourceBlocks.size()); - if (sourceStart < sourceBlocks.size()) { - result.addAll(sourceBlocks.subList(sourceStart, sourceEnd)); - } - } - if (parity == null) { - return result; - } - List parityBlocks = cachedLocatedBlocks.get(parity); - synchronized (parityBlocks) { - int parityStart = stripeIndex * parityLength; - int parityEnd = Math.min(parityStart + parityLength, - parityBlocks.size()); - if (parityStart < parityBlocks.size()) { - result.addAll(parityBlocks.subList(parityStart, parityEnd)); - } - } - return result; - } - - private int getBlockIndex(String file, Block block) throws IOException { - List blocks = cachedLocatedBlocks.get(file); - synchronized (blocks) { - // null indicates that this block is currently added. Return size() - // as the index in this case - if (block == null) { - return blocks.size(); - } - for (int i = 0; i < blocks.size(); i++) { - if (blocks.get(i).getBlock().getLocalBlock().equals(block)) { - return i; - } - } - } - throw new IOException("Cannot locate " + block + " in file " + file); - } - - /** - * Cache results for getFullPathName() - */ - static class CachedFullPathNames { - FSNamesystem namesystem; - CachedFullPathNames(FSNamesystem namesystem) { - this.namesystem = namesystem; - } - private Cache cacheInternal = - new Cache() { - @Override - public String getDirectly(INodeWithHashCode inode) throws IOException { - namesystem.readLock(); - try { - return inode.getFullPathName(); - } finally { - namesystem.readUnlock(); - } - } - }; - - static private class INodeWithHashCode { - BlockCollection bc; - INodeWithHashCode(BlockCollection bc) { - this.bc= bc; - } - @Override - public boolean equals(Object obj) { - return bc== obj; - } - @Override - public int hashCode() { - return System.identityHashCode(bc); - } - String getFullPathName() { - return bc.getName(); - } - } - - public String get(BlockCollection bc) throws IOException { - return cacheInternal.get(new INodeWithHashCode(bc)); - } - } - - /** - * Cache results for FSNamesystem.getBlockLocations() - */ - static class CachedLocatedBlocks extends Cache> { - FSNamesystem namesystem; - CachedLocatedBlocks(FSNamesystem namesystem) { - this.namesystem = namesystem; - } - @Override - public List getDirectly(String file) throws IOException { - long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen(); - List result = NameNodeRaidUtil.getBlockLocations(namesystem, - file, 0L, len, false, false).getLocatedBlocks(); - if (result == null || result.isEmpty()) { - result = new ArrayList(); - } - return Collections.synchronizedList(result); - } - } - - static abstract class Cache { - private Map cache; - private static final long CACHE_TIMEOUT = 300000L; // 5 minutes - // The timeout is long but the consequence of stale value is not serious - Cache() { - Map map = new LinkedHashMap() { - private static final long serialVersionUID = 1L; - final private int MAX_ENTRIES = 50000; - @Override - protected boolean removeEldestEntry( - Map.Entry eldest) { - return size() > MAX_ENTRIES; - } - }; - this.cache = Collections.synchronizedMap(map); - } - - // Note that this method may hold FSNamesystem.readLock() and it may - // be called inside FSNamesystem.writeLock(). If we make this method - // synchronized, it will deadlock. - abstract protected V getDirectly(K key) throws IOException; - - public V get(K key) throws IOException { - // The method is not synchronized so we may get some stale value here but - // it's OK. - ValueWithTime result = cache.get(key); - long now = Time.now(); - if (result != null && - now - result.cachedTime < CACHE_TIMEOUT) { - return result.value; - } - result = new ValueWithTime(); - result.value = getDirectly(key); - result.cachedTime = now; - cache.put(key, result); - return result.value; - } - private class ValueWithTime { - V value = null; - long cachedTime = 0L; - } - } - - /** - * Get path for the corresponding source file for a valid parity - * file. Returns null if it does not exists - * @param parity the toUri path of the parity file - * @return the toUri path of the source file - */ - String getSourceFile(String parity, String prefix) throws IOException { - if (isHarFile(parity)) { - return null; - } - // remove the prefix - String src = parity.substring(prefix.length()); - if (NameNodeRaidUtil.getFileInfo(namesystem, src, true) == null) { - return null; - } - return src; - } - - /** - * Get path for the corresponding parity file for a source file. - * Returns null if it does not exists - * @param src the toUri path of the source file - * @return the toUri path of the parity file - */ - String getParityFile(String src) throws IOException { - String xorParity = getParityFile(xorPrefix, src); - if (xorParity != null) { - return xorParity; - } - String rsParity = getParityFile(rsPrefix, src); - if (rsParity != null) { - return rsParity; - } - return null; - } - - /** - * Get path for the parity file. Returns null if it does not exists - * @param parityPrefix usuall "/raid/" or "/raidrs/" - * @return the toUri path of the parity file - */ - private String getParityFile(String parityPrefix, String src) - throws IOException { - String parity = parityPrefix + src; - if (NameNodeRaidUtil.getFileInfo(namesystem, parity, true) == null) { - return null; - } - return parity; - } - - private boolean isHarFile(String path) { - return path.lastIndexOf(RaidNode.HAR_SUFFIX) != -1; - } - - enum FileType { - NOT_RAID, - XOR_HAR_TEMP_PARITY, - XOR_TEMP_PARITY, - XOR_PARITY, - XOR_SOURCE, - RS_HAR_TEMP_PARITY, - RS_TEMP_PARITY, - RS_PARITY, - RS_SOURCE, - } - - FileType getFileType(String path) throws IOException { - if (path.startsWith(raidHarTempPrefix + Path.SEPARATOR)) { - return FileType.XOR_HAR_TEMP_PARITY; - } - if (path.startsWith(raidrsHarTempPrefix + Path.SEPARATOR)) { - return FileType.RS_HAR_TEMP_PARITY; - } - if (path.startsWith(raidTempPrefix + Path.SEPARATOR)) { - return FileType.XOR_TEMP_PARITY; - } - if (path.startsWith(raidrsTempPrefix + Path.SEPARATOR)) { - return FileType.RS_TEMP_PARITY; - } - if (path.startsWith(xorPrefix + Path.SEPARATOR)) { - return FileType.XOR_PARITY; - } - if (path.startsWith(rsPrefix + Path.SEPARATOR)) { - return FileType.RS_PARITY; - } - String parity = getParityFile(path); - if (parity == null) { - return FileType.NOT_RAID; - } - if (parity.startsWith(xorPrefix + Path.SEPARATOR)) { - return FileType.XOR_SOURCE; - } - if (parity.startsWith(rsPrefix + Path.SEPARATOR)) { - return FileType.RS_SOURCE; - } - return FileType.NOT_RAID; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java deleted file mode 100644 index 7eb6e16c55a..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java +++ /dev/null @@ -1,505 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.datanode; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.SocketException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.Arrays; - -import org.apache.commons.logging.Log; -import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; -import org.apache.hadoop.hdfs.util.DataTransferThrottler; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.net.SocketOutputStream; -import org.apache.hadoop.util.DataChecksum; -import org.apache.hadoop.util.StringUtils; - -/** - * Reads a block from the disk and sends it to a recipient. - */ -public class RaidBlockSender implements java.io.Closeable { - public static final Log LOG = DataNode.LOG; - static final Log ClientTraceLog = DataNode.ClientTraceLog; - - private ExtendedBlock block; // the block to read from - - /** The visible length of a replica. */ - private final long replicaVisibleLength; - - private InputStream blockIn; // data stream - private long blockInPosition = -1; // updated while using transferTo(). - private DataInputStream checksumIn; // checksum datastream - private DataChecksum checksum; // checksum stream - private long offset; // starting position to read - /** Initial position to read */ - private long initialOffset; - private long endOffset; // ending position - private int chunkSize; // chunk size - private int checksumSize; // checksum size - private boolean corruptChecksumOk; // if need to verify checksum - private boolean chunkOffsetOK; // if need to send chunk offset - private long seqno; // sequence number of packet - - private boolean transferToAllowed = true; - private boolean blockReadFully; //set when the whole block is read - private boolean verifyChecksum; //if true, check is verified while reading - private final String clientTraceFmt; // format of client trace log message - - /** - * Minimum buffer used while sending data to clients. Used only if - * transferTo() is enabled. 64KB is not that large. It could be larger, but - * not sure if there will be much more improvement. - */ - private static final int MIN_BUFFER_WITH_TRANSFERTO = 64*1024; - private static final int TRANSFERTO_BUFFER_SIZE = Math.max( - HdfsConstants.IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO); - private volatile ChunkChecksum lastChunkChecksum = null; - - - public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, long length, - boolean corruptChecksumOk, boolean chunkOffsetOK, - boolean verifyChecksum, boolean transferToAllowed, - DataInputStream metadataIn, InputStreamFactory streamFactory - ) throws IOException { - this(block, blockLength, startOffset, length, - corruptChecksumOk, chunkOffsetOK, - verifyChecksum, transferToAllowed, - metadataIn, streamFactory, null); - } - - public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, long length, - boolean corruptChecksumOk, boolean chunkOffsetOK, - boolean verifyChecksum, boolean transferToAllowed, - DataInputStream metadataIn, InputStreamFactory streamFactory, - String clientTraceFmt) throws IOException { - try { - this.block = block; - this.chunkOffsetOK = chunkOffsetOK; - this.corruptChecksumOk = corruptChecksumOk; - this.verifyChecksum = verifyChecksum; - this.replicaVisibleLength = blockLength; - this.transferToAllowed = transferToAllowed; - this.clientTraceFmt = clientTraceFmt; - - if ( !corruptChecksumOk || metadataIn != null) { - this.checksumIn = metadataIn; - - // read and handle the common header here. For now just a version - BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); - short version = header.getVersion(); - - if (version != BlockMetadataHeader.VERSION) { - LOG.warn("Wrong version (" + version + ") for metadata file for " - + block + " ignoring ..."); - } - checksum = header.getChecksum(); - } else { - LOG.warn("Could not find metadata file for " + block); - // This only decides the buffer size. Use BUFFER_SIZE? - checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL, - 16 * 1024); - } - - /* If bytesPerChecksum is very large, then the metadata file - * is mostly corrupted. For now just truncate bytesPerchecksum to - * blockLength. - */ - int size = checksum.getBytesPerChecksum(); - if (size > 10*1024*1024 && size > replicaVisibleLength) { - checksum = DataChecksum.newDataChecksum(checksum.getChecksumType(), - Math.max((int)replicaVisibleLength, 10*1024*1024)); - size = checksum.getBytesPerChecksum(); - } - chunkSize = size; - checksumSize = checksum.getChecksumSize(); - - if (length < 0) { - length = replicaVisibleLength; - } - - endOffset = blockLength; - - if (startOffset < 0 || startOffset > endOffset - || (length + startOffset) > endOffset) { - String msg = " Offset " + startOffset + " and length " + length - + " don't match block " + block + " ( blockLen " + endOffset + " )"; - LOG.warn("sendBlock() : " + msg); - throw new IOException(msg); - } - - offset = (startOffset - (startOffset % chunkSize)); - if (length >= 0) { - // Make sure endOffset points to end of a checksumed chunk. - long tmpLen = startOffset + length; - if (tmpLen % chunkSize != 0) { - tmpLen += (chunkSize - tmpLen % chunkSize); - } - if (tmpLen < endOffset) { - // will use on-disk checksum here since the end is a stable chunk - endOffset = tmpLen; - } - } - - // seek to the right offsets - if (offset > 0) { - long checksumSkip = (offset / chunkSize) * checksumSize; - // note blockInStream is seeked when created below - if (checksumSkip > 0) { - // Should we use seek() for checksum file as well? - IOUtils.skipFully(checksumIn, checksumSkip); - } - } - seqno = 0; - - blockIn = streamFactory.createStream(offset); - } catch (IOException ioe) { - IOUtils.closeStream(this); - IOUtils.closeStream(blockIn); - throw ioe; - } - } - - /** - * close opened files. - */ - public void close() throws IOException { - IOException ioe = null; - // close checksum file - if(checksumIn!=null) { - try { - checksumIn.close(); - } catch (IOException e) { - ioe = e; - } - checksumIn = null; - } - // close data file - if(blockIn!=null) { - try { - blockIn.close(); - } catch (IOException e) { - ioe = e; - } - blockIn = null; - } - // throw IOException if there is any - if(ioe!= null) { - throw ioe; - } - } - - /** - * Converts an IOExcpetion (not subclasses) to SocketException. - * This is typically done to indicate to upper layers that the error - * was a socket error rather than often more serious exceptions like - * disk errors. - */ - private static IOException ioeToSocketException(IOException ioe) { - if (ioe.getClass().equals(IOException.class)) { - // "se" could be a new class in stead of SocketException. - IOException se = new SocketException("Original Exception : " + ioe); - se.initCause(ioe); - /* Change the stacktrace so that original trace is not truncated - * when printed.*/ - se.setStackTrace(ioe.getStackTrace()); - return se; - } - // otherwise just return the same exception. - return ioe; - } - - /** - * @param datalen Length of data - * @return number of chunks for data of given size - */ - private int numberOfChunks(long datalen) { - return (int) ((datalen + chunkSize - 1)/chunkSize); - } - - /** - * Write packet header into {@code pkt} - */ - private void writePacketHeader(ByteBuffer pkt, int dataLen, int packetLen) { - pkt.clear(); - PacketHeader header = new PacketHeader(packetLen, offset, seqno, - (dataLen == 0), dataLen, false); - header.putInBuffer(pkt); - } - - /** - * Read checksum into given buffer - * @param buf buffer to read the checksum into - * @param checksumOffset offset at which to write the checksum into buf - * @param checksumLen length of checksum to write - * @throws IOException on error - */ - private void readChecksum(byte[] buf, final int checksumOffset, - final int checksumLen) throws IOException { - if (checksumSize <= 0 && checksumIn == null) { - return; - } - try { - checksumIn.readFully(buf, checksumOffset, checksumLen); - } catch (IOException e) { - LOG.warn(" Could not read or failed to veirfy checksum for data" - + " at offset " + offset + " for block " + block, e); - IOUtils.closeStream(checksumIn); - checksumIn = null; - if (corruptChecksumOk) { - if (checksumOffset < checksumLen) { - // Just fill the array with zeros. - Arrays.fill(buf, checksumOffset, checksumLen, (byte) 0); - } - } else { - throw e; - } - } - } - - /** - * Sends a packet with up to maxChunks chunks of data. - * - * @param pkt buffer used for writing packet data - * @param maxChunks maximum number of chunks to send - * @param out stream to send data to - * @param transferTo use transferTo to send data - * @param throttler used for throttling data transfer bandwidth - */ - private int sendPacket(ByteBuffer pkt, int maxChunks, OutputStream out, - boolean transferTo, DataTransferThrottler throttler) throws IOException { - int dataLen = (int) Math.min(endOffset - offset, - (chunkSize * (long) maxChunks)); - - int numChunks = numberOfChunks(dataLen); // Number of chunks be sent in the packet - int checksumDataLen = numChunks * checksumSize; - int packetLen = dataLen + checksumDataLen + 4; - boolean lastDataPacket = offset + dataLen == endOffset && dataLen > 0; - - writePacketHeader(pkt, dataLen, packetLen); - - int checksumOff = pkt.position(); - byte[] buf = pkt.array(); - - if (checksumSize > 0 && checksumIn != null) { - readChecksum(buf, checksumOff, checksumDataLen); - - // write in progress that we need to use to get last checksum - if (lastDataPacket && lastChunkChecksum != null) { - int start = checksumOff + checksumDataLen - checksumSize; - byte[] updatedChecksum = lastChunkChecksum.getChecksum(); - - if (updatedChecksum != null) { - System.arraycopy(updatedChecksum, 0, buf, start, checksumSize); - } - } - } - - int dataOff = checksumOff + checksumDataLen; - if (!transferTo) { // normal transfer - IOUtils.readFully(blockIn, buf, dataOff, dataLen); - - if (verifyChecksum) { - verifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff); - } - } - - try { - if (transferTo) { - SocketOutputStream sockOut = (SocketOutputStream)out; - sockOut.write(buf, 0, dataOff); // First write checksum - - // no need to flush. since we know out is not a buffered stream. - sockOut.transferToFully(((FileInputStream)blockIn).getChannel(), - blockInPosition, dataLen); - blockInPosition += dataLen; - } else { - // normal transfer - out.write(buf, 0, dataOff + dataLen); - } - } catch (IOException e) { - /* Exception while writing to the client. Connection closure from - * the other end is mostly the case and we do not care much about - * it. But other things can go wrong, especially in transferTo(), - * which we do not want to ignore. - * - * The message parsing below should not be considered as a good - * coding example. NEVER do it to drive a program logic. NEVER. - * It was done here because the NIO throws an IOException for EPIPE. - */ - String ioem = e.getMessage(); - if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) { - LOG.error("BlockSender.sendChunks() exception: ", e); - } - throw ioeToSocketException(e); - } - - if (throttler != null) { // rebalancing so throttle - throttler.throttle(packetLen); - } - - return dataLen; - } - - /** - * Compute checksum for chunks and verify the checksum that is read from - * the metadata file is correct. - * - * @param buf buffer that has checksum and data - * @param dataOffset position where data is written in the buf - * @param datalen length of data - * @param numChunks number of chunks corresponding to data - * @param checksumOffset offset where checksum is written in the buf - * @throws ChecksumException on failed checksum verification - */ - public void verifyChecksum(final byte[] buf, final int dataOffset, - final int datalen, final int numChunks, final int checksumOffset) - throws ChecksumException { - int dOff = dataOffset; - int cOff = checksumOffset; - int dLeft = datalen; - - for (int i = 0; i < numChunks; i++) { - checksum.reset(); - int dLen = Math.min(dLeft, chunkSize); - checksum.update(buf, dOff, dLen); - if (!checksum.compare(buf, cOff)) { - long failedPos = offset + datalen - dLeft; - throw new ChecksumException("Checksum failed at " + failedPos, - failedPos); - } - dLeft -= dLen; - dOff += dLen; - cOff += checksumSize; - } - } - - - /** - * sendBlock() is used to read block and its metadata and stream the data to - * either a client or to another datanode. - * - * @param out stream to which the block is written to - * @param baseStream optional. if non-null, out is assumed to - * be a wrapper over this stream. This enables optimizations for - * sending the data, e.g. - * {@link SocketOutputStream#transferToFully(FileChannel, - * long, int)}. - * @return total bytes reads, including crc. - */ - public long sendBlock(DataOutputStream out, OutputStream baseStream) - throws IOException { - if (out == null) { - throw new IOException( "out stream is null" ); - } - initialOffset = offset; - long totalRead = 0; - OutputStream streamForSendChunks = out; - - final long startTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0; - try { - int maxChunksPerPacket; - int pktSize = PacketHeader.PKT_HEADER_LEN; - boolean transferTo = transferToAllowed && !verifyChecksum - && baseStream instanceof SocketOutputStream - && blockIn instanceof FileInputStream; - if (transferTo) { - FileChannel fileChannel = ((FileInputStream)blockIn).getChannel(); - blockInPosition = fileChannel.position(); - streamForSendChunks = baseStream; - maxChunksPerPacket = numberOfChunks(TRANSFERTO_BUFFER_SIZE); - - // Smaller packet size to only hold checksum when doing transferTo - pktSize += checksumSize * maxChunksPerPacket; - } else { - maxChunksPerPacket = Math.max(1, - numberOfChunks(HdfsConstants.IO_FILE_BUFFER_SIZE)); - // Packet size includes both checksum and data - pktSize += (chunkSize + checksumSize) * maxChunksPerPacket; - } - - ByteBuffer pktBuf = ByteBuffer.allocate(pktSize); - - while (endOffset > offset) { - long len = sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, - transferTo, null); - offset += len; - totalRead += len + (numberOfChunks(len) * checksumSize); - seqno++; - } - try { - // send an empty packet to mark the end of the block - sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, - null); - out.flush(); - } catch (IOException e) { //socket error - throw ioeToSocketException(e); - } - blockReadFully = true; - } finally { - if (clientTraceFmt != null) { - final long endTime = System.nanoTime(); - ClientTraceLog.info(String.format(clientTraceFmt, totalRead, - initialOffset, endTime - startTime)); - } - close(); - } - return totalRead; - } - - boolean isBlockReadFully() { - return blockReadFully; - } - - public static interface InputStreamFactory { - public InputStream createStream(long offset) throws IOException; - } - - /** - * @return the checksum type that will be used with this block transfer. - */ - public DataChecksum getChecksum() { - return checksum; - } - - private static class BlockInputStreamFactory implements InputStreamFactory { - private final ExtendedBlock block; - private final FsDatasetSpi data; - - private BlockInputStreamFactory(ExtendedBlock block, FsDatasetSpi data) { - this.block = block; - this.data = data; - } - - @Override - public InputStream createStream(long offset) throws IOException { - return data.getBlockInputStream(block, offset); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java deleted file mode 100644 index 92586961564..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import java.io.*; - -import org.apache.hadoop.classification.*; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.hdfs.protocol.*; -import org.apache.hadoop.ipc.StandbyException; -import org.apache.hadoop.security.AccessControlException; - -/** Utilities used by RAID for accessing NameNode. */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class NameNodeRaidUtil { - /** Accessing FSDirectory.getFileInfo(..) */ - public static HdfsFileStatus getFileInfo(final FSDirectory dir, - final String src, final boolean resolveLink - ) throws UnresolvedLinkException { - return dir.getFileInfo(src, resolveLink); - } - - /** Accessing FSNamesystem.getFileInfo(..) - * @throws StandbyException */ - public static HdfsFileStatus getFileInfo(final FSNamesystem namesystem, - final String src, final boolean resolveLink - ) throws AccessControlException, UnresolvedLinkException, StandbyException { - return namesystem.getFileInfo(src, resolveLink); - } - - /** Accessing FSNamesystem.getBlockLocations(..) */ - public static LocatedBlocks getBlockLocations(final FSNamesystem namesystem, - final String src, final long offset, final long length, - final boolean doAccessTime, final boolean needBlockToken - ) throws FileNotFoundException, UnresolvedLinkException, IOException { - return namesystem.getBlockLocations(src, offset, length, - doAccessTime, needBlockToken, true); - } -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java deleted file mode 100644 index dd2fb96c2ba..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java +++ /dev/null @@ -1,840 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.channels.SocketChannel; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; - -import org.apache.hadoop.util.DataChecksum; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.protocol.datatransfer.*; -import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.FSConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.RaidBlockSender; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.net.NetUtils; - -import org.apache.hadoop.raid.RaidNode; -import org.apache.hadoop.raid.RaidUtils; - - -/** - * contains the core functionality of the block fixer - * - * configuration options: - * raid.blockfix.classname - the class name of the block fixer - * implementation to use - * - * raid.blockfix.interval - interval between checks for corrupt files - * - * raid.blockfix.history.interval - interval before fixing same file again - * - * raid.blockfix.read.timeout - read time out - * - * raid.blockfix.write.timeout - write time out - */ -public abstract class BlockFixer extends Configured implements Runnable { - - public static final String BLOCKFIX_CLASSNAME = "raid.blockfix.classname"; - public static final String BLOCKFIX_INTERVAL = "raid.blockfix.interval"; - public static final String BLOCKFIX_HISTORY_INTERVAL = - "raid.blockfix.history.interval"; - public static final String BLOCKFIX_READ_TIMEOUT = - "raid.blockfix.read.timeout"; - public static final String BLOCKFIX_WRITE_TIMEOUT = - "raid.blockfix.write.timeout"; - - public static final long DEFAULT_BLOCKFIX_INTERVAL = 60 * 1000; // 1 min - public static final long DEFAULT_BLOCKFIX_HISTORY_INTERVAL = - 60 * 60 * 1000; // 60 mins - - public static BlockFixer createBlockFixer(Configuration conf) - throws ClassNotFoundException { - try { - // default to distributed block fixer - Class blockFixerClass = - conf.getClass(BLOCKFIX_CLASSNAME, DistBlockFixer.class); - if (!BlockFixer.class.isAssignableFrom(blockFixerClass)) { - throw new ClassNotFoundException("not an implementation of blockfixer"); - } - Constructor constructor = - blockFixerClass.getConstructor(new Class[] {Configuration.class} ); - return (BlockFixer) constructor.newInstance(conf); - } catch (NoSuchMethodException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (InstantiationException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (IllegalAccessException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (InvocationTargetException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } - } - - private long numFilesFixed = 0; - - public volatile boolean running = true; - - // interval between checks for corrupt files - protected long blockFixInterval; - - // interval before fixing same file again - protected long historyInterval; - - public BlockFixer(Configuration conf) { - super(conf); - blockFixInterval = - getConf().getLong(BLOCKFIX_INTERVAL, DEFAULT_BLOCKFIX_INTERVAL); - historyInterval = - getConf().getLong(BLOCKFIX_HISTORY_INTERVAL, - DEFAULT_BLOCKFIX_HISTORY_INTERVAL); - - } - - @Override - public abstract void run(); - - /** - * returns the number of files that have been fixed by this block fixer - */ - public synchronized long filesFixed() { - return numFilesFixed; - } - - /** - * increments the number of files that have been fixed by this block fixer - */ - protected synchronized void incrFilesFixed() { - numFilesFixed++; - } - - /** - * increments the number of files that have been fixed by this block fixer - */ - protected synchronized void incrFilesFixed(long incr) { - if (incr < 0) { - throw new IllegalArgumentException("cannot increment by negative value " + - incr); - } - - numFilesFixed += incr; - } - - static boolean isSourceFile(Path p, String[] destPrefixes) { - String pathStr = p.toUri().getPath(); - for (String destPrefix: destPrefixes) { - if (pathStr.startsWith(destPrefix)) { - return false; - } - } - return true; - } - - void filterUnfixableSourceFiles(Iterator it) throws IOException { - String xorPrefix = RaidNode.xorDestinationPath(getConf()).toUri().getPath(); - if (!xorPrefix.endsWith(Path.SEPARATOR)) { - xorPrefix += Path.SEPARATOR; - } - String rsPrefix = RaidNode.rsDestinationPath(getConf()).toUri().getPath(); - if (!rsPrefix.endsWith(Path.SEPARATOR)) { - rsPrefix += Path.SEPARATOR; - } - String[] destPrefixes = new String[]{xorPrefix, rsPrefix}; - while (it.hasNext()) { - Path p = it.next(); - if (isSourceFile(p, destPrefixes) && - RaidNode.xorParityForSource(p, getConf()) == null && - RaidNode.rsParityForSource(p, getConf()) == null) { - it.remove(); - } - } - } - - /** - * this class implements the actual fixing functionality - * we keep this in a separate class so that - * the distributed block fixer can use it - */ - static class BlockFixerHelper extends Configured { - - public static final Log LOG = LogFactory.getLog(BlockFixer. - BlockFixerHelper.class); - - private String xorPrefix; - private String rsPrefix; - private XOREncoder xorEncoder; - private XORDecoder xorDecoder; - private ReedSolomonEncoder rsEncoder; - private ReedSolomonDecoder rsDecoder; - - public BlockFixerHelper(Configuration conf) throws IOException { - super(conf); - - xorPrefix = RaidNode.xorDestinationPath(getConf()).toUri().getPath(); - if (!xorPrefix.endsWith(Path.SEPARATOR)) { - xorPrefix += Path.SEPARATOR; - } - rsPrefix = RaidNode.rsDestinationPath(getConf()).toUri().getPath(); - if (!rsPrefix.endsWith(Path.SEPARATOR)) { - rsPrefix += Path.SEPARATOR; - } - int stripeLength = RaidNode.getStripeLength(getConf()); - xorEncoder = new XOREncoder(getConf(), stripeLength); - xorDecoder = new XORDecoder(getConf(), stripeLength); - int parityLength = RaidNode.rsParityLength(getConf()); - rsEncoder = new ReedSolomonEncoder(getConf(), stripeLength, parityLength); - rsDecoder = new ReedSolomonDecoder(getConf(), stripeLength, parityLength); - - } - - /** - * checks whether file is xor parity file - */ - boolean isXorParityFile(Path p) { - String pathStr = p.toUri().getPath(); - if (pathStr.contains(RaidNode.HAR_SUFFIX)) { - return false; - } - return pathStr.startsWith(xorPrefix); - } - - /** - * checks whether file is rs parity file - */ - boolean isRsParityFile(Path p) { - String pathStr = p.toUri().getPath(); - if (pathStr.contains(RaidNode.HAR_SUFFIX)) { - return false; - } - return pathStr.startsWith(rsPrefix); - } - - /** - * Fix a file, do not report progess. - * - * @return true if file has been fixed, false if no fixing - * was necessary or possible. - */ - boolean fixFile(Path srcPath) throws IOException { - return fixFile(srcPath, new RaidUtils.DummyProgressable()); - } - - /** - * Fix a file, report progess. - * - * @return true if file has been fixed, false if no fixing - * was necessary or possible. - */ - boolean fixFile(Path srcPath, Progressable progress) throws IOException { - - if (RaidNode.isParityHarPartFile(srcPath)) { - return processCorruptParityHarPartFile(srcPath, progress); - } - - // The corrupted file is a XOR parity file - if (isXorParityFile(srcPath)) { - return processCorruptParityFile(srcPath, xorEncoder, progress); - } - - // The corrupted file is a ReedSolomon parity file - if (isRsParityFile(srcPath)) { - return processCorruptParityFile(srcPath, rsEncoder, progress); - } - - // The corrupted file is a source file - RaidNode.ParityFilePair ppair = - RaidNode.xorParityForSource(srcPath, getConf()); - Decoder decoder = null; - if (ppair != null) { - decoder = xorDecoder; - } else { - ppair = RaidNode.rsParityForSource(srcPath, getConf()); - if (ppair != null) { - decoder = rsDecoder; - } - } - - // If we have a parity file, process the file and fix it. - if (ppair != null) { - return processCorruptFile(srcPath, ppair, decoder, progress); - } - - // there was nothing to do - return false; - } - - /** - * Sorts source files ahead of parity files. - */ - void sortCorruptFiles(List files) { - // TODO: We should first fix the files that lose more blocks - Comparator comp = new Comparator() { - public int compare(Path p1, Path p2) { - if (isXorParityFile(p2) || isRsParityFile(p2)) { - // If p2 is a parity file, p1 is smaller. - return -1; - } - if (isXorParityFile(p1) || isRsParityFile(p1)) { - // If p1 is a parity file, p2 is smaller. - return 1; - } - // If both are source files, they are equal. - return 0; - } - }; - Collections.sort(files, comp); - } - - /** - * Returns a DistributedFileSystem hosting the path supplied. - */ - protected DistributedFileSystem getDFS(Path p) throws IOException { - return (DistributedFileSystem) p.getFileSystem(getConf()); - } - - /** - * Reads through a corrupt source file fixing corrupt blocks on the way. - * @param srcPath Path identifying the corrupt file. - * @throws IOException - * @return true if file has been fixed, false if no fixing - * was necessary or possible. - */ - boolean processCorruptFile(Path srcPath, RaidNode.ParityFilePair parityPair, - Decoder decoder, Progressable progress) - throws IOException { - LOG.info("Processing corrupt file " + srcPath); - - DistributedFileSystem srcFs = getDFS(srcPath); - FileStatus srcStat = srcFs.getFileStatus(srcPath); - long blockSize = srcStat.getBlockSize(); - long srcFileSize = srcStat.getLen(); - String uriPath = srcPath.toUri().getPath(); - - int numBlocksFixed = 0; - List corrupt = - RaidDFSUtil.corruptBlocksInFile(srcFs, uriPath, 0, srcFileSize); - if (corrupt.size() == 0) { - return false; - } - for (LocatedBlock lb: corrupt) { - ExtendedBlock corruptBlock = lb.getBlock(); - long corruptOffset = lb.getStartOffset(); - - LOG.info("Found corrupt block " + corruptBlock + - ", offset " + corruptOffset); - - final long blockContentsSize = - Math.min(blockSize, srcFileSize - corruptOffset); - File localBlockFile = - File.createTempFile(corruptBlock.getBlockName(), ".tmp"); - localBlockFile.deleteOnExit(); - - try { - decoder.recoverBlockToFile(srcFs, srcPath, parityPair.getFileSystem(), - parityPair.getPath(), blockSize, - corruptOffset, localBlockFile, - blockContentsSize); - - // We have a the contents of the block, send them. - DatanodeInfo datanode = chooseDatanode(lb.getLocations()); - computeMetadataAndSendFixedBlock(datanode, localBlockFile, - lb, blockContentsSize); - numBlocksFixed++; - } finally { - localBlockFile.delete(); - } - progress.progress(); - } - LOG.info("Fixed " + numBlocksFixed + " blocks in " + srcPath); - return true; - } - - /** - * Fixes corrupt blocks in a parity file. - * This function uses the corresponding source file to regenerate parity - * file blocks. - * @return true if file has been fixed, false if no fixing - * was necessary or possible. - */ - boolean processCorruptParityFile(Path parityPath, Encoder encoder, - Progressable progress) - throws IOException { - LOG.info("Processing corrupt file " + parityPath); - Path srcPath = sourcePathFromParityPath(parityPath); - if (srcPath == null) { - LOG.warn("Unusable parity file " + parityPath); - return false; - } - - DistributedFileSystem parityFs = getDFS(parityPath); - FileStatus parityStat = parityFs.getFileStatus(parityPath); - long blockSize = parityStat.getBlockSize(); - long parityFileSize = parityStat.getLen(); - FileStatus srcStat = getDFS(srcPath).getFileStatus(srcPath); - long srcFileSize = srcStat.getLen(); - - // Check timestamp. - if (srcStat.getModificationTime() != parityStat.getModificationTime()) { - LOG.info("Mismatching timestamp for " + srcPath + " and " + parityPath + - ", moving on..."); - return false; - } - - String uriPath = parityPath.toUri().getPath(); - int numBlocksFixed = 0; - List corrupt = - RaidDFSUtil.corruptBlocksInFile(parityFs, uriPath, 0, parityFileSize); - if (corrupt.size() == 0) { - return false; - } - for (LocatedBlock lb: corrupt) { - ExtendedBlock corruptBlock = lb.getBlock(); - long corruptOffset = lb.getStartOffset(); - - LOG.info("Found corrupt block " + corruptBlock + - ", offset " + corruptOffset); - - File localBlockFile = - File.createTempFile(corruptBlock.getBlockName(), ".tmp"); - localBlockFile.deleteOnExit(); - - try { - encoder.recoverParityBlockToFile(parityFs, srcPath, srcFileSize, - blockSize, parityPath, - corruptOffset, localBlockFile); - // We have a the contents of the block, send them. - DatanodeInfo datanode = chooseDatanode(lb.getLocations()); - computeMetadataAndSendFixedBlock(datanode, localBlockFile, lb, - blockSize); - - numBlocksFixed++; - } finally { - localBlockFile.delete(); - } - progress.progress(); - } - LOG.info("Fixed " + numBlocksFixed + " blocks in " + parityPath); - return true; - } - - /** - * Reads through a parity HAR part file, fixing corrupt blocks on the way. - * A HAR block can contain many file blocks, as long as the HAR part file - * block size is a multiple of the file block size. - * @return true if file has been fixed, false if no fixing - * was necessary or possible. - */ - boolean processCorruptParityHarPartFile(Path partFile, - Progressable progress) - throws IOException { - LOG.info("Processing corrupt file " + partFile); - // Get some basic information. - DistributedFileSystem dfs = getDFS(partFile); - FileStatus partFileStat = dfs.getFileStatus(partFile); - long partFileSize = partFileStat.getLen(); - long partFileBlockSize = partFileStat.getBlockSize(); - LOG.info(partFile + " has block size " + partFileBlockSize); - - // Find the path to the index file. - // Parity file HARs are only one level deep, so the index files is at the - // same level as the part file. - String harDirectory = partFile.toUri().getPath(); // Temporarily. - harDirectory = - harDirectory.substring(0, harDirectory.lastIndexOf(Path.SEPARATOR)); - Path indexFile = new Path(harDirectory + "/" + HarIndex.indexFileName); - FileStatus indexStat = dfs.getFileStatus(indexFile); - // Parses through the HAR index file. - HarIndex harIndex = new HarIndex(dfs.open(indexFile), indexStat.getLen()); - - String uriPath = partFile.toUri().getPath(); - int numBlocksFixed = 0; - List corrupt = - RaidDFSUtil.corruptBlocksInFile(dfs, uriPath, 0, partFileSize); - if (corrupt.size() == 0) { - return false; - } - for (LocatedBlock lb: corrupt) { - ExtendedBlock corruptBlock = lb.getBlock(); - long corruptOffset = lb.getStartOffset(); - - File localBlockFile = - File.createTempFile(corruptBlock.getBlockName(), ".tmp"); - localBlockFile.deleteOnExit(); - processCorruptParityHarPartBlock(dfs, partFile, corruptBlock, - corruptOffset, partFileStat, harIndex, - localBlockFile, progress); - // Now we have recovered the part file block locally, send it. - try { - DatanodeInfo datanode = chooseDatanode(lb.getLocations()); - computeMetadataAndSendFixedBlock(datanode, localBlockFile, - lb, localBlockFile.length()); - numBlocksFixed++; - } finally { - localBlockFile.delete(); - } - progress.progress(); - } - LOG.info("Fixed " + numBlocksFixed + " blocks in " + partFile); - return true; - } - - /** - * This fixes a single part file block by recovering in sequence each - * parity block in the part file block. - */ - private void processCorruptParityHarPartBlock(FileSystem dfs, Path partFile, - ExtendedBlock corruptBlock, - long corruptOffset, - FileStatus partFileStat, - HarIndex harIndex, - File localBlockFile, - Progressable progress) - throws IOException { - String partName = partFile.toUri().getPath(); // Temporarily. - partName = partName.substring(1 + partName.lastIndexOf(Path.SEPARATOR)); - - OutputStream out = new FileOutputStream(localBlockFile); - - try { - // A HAR part file block could map to several parity files. We need to - // use all of them to recover this block. - final long corruptEnd = Math.min(corruptOffset + - partFileStat.getBlockSize(), - partFileStat.getLen()); - for (long offset = corruptOffset; offset < corruptEnd; ) { - HarIndex.IndexEntry entry = harIndex.findEntry(partName, offset); - if (entry == null) { - String msg = "Corrupt index file has no matching index entry for " + - partName + ":" + offset; - LOG.warn(msg); - throw new IOException(msg); - } - Path parityFile = new Path(entry.fileName); - Encoder encoder; - if (isXorParityFile(parityFile)) { - encoder = xorEncoder; - } else if (isRsParityFile(parityFile)) { - encoder = rsEncoder; - } else { - String msg = "Could not figure out parity file correctly"; - LOG.warn(msg); - throw new IOException(msg); - } - Path srcFile = sourcePathFromParityPath(parityFile); - FileStatus srcStat = dfs.getFileStatus(srcFile); - if (srcStat.getModificationTime() != entry.mtime) { - String msg = "Modification times of " + parityFile + " and " + - srcFile + " do not match."; - LOG.warn(msg); - throw new IOException(msg); - } - long corruptOffsetInParity = offset - entry.startOffset; - LOG.info(partFile + ":" + offset + " maps to " + - parityFile + ":" + corruptOffsetInParity + - " and will be recovered from " + srcFile); - encoder.recoverParityBlockToStream(dfs, srcFile, srcStat.getLen(), - srcStat.getBlockSize(), parityFile, - corruptOffsetInParity, out); - // Finished recovery of one parity block. Since a parity block has the - // same size as a source block, we can move offset by source block size. - offset += srcStat.getBlockSize(); - LOG.info("Recovered " + srcStat.getBlockSize() + " part file bytes "); - if (offset > corruptEnd) { - String msg = - "Recovered block spills across part file blocks. Cannot continue."; - throw new IOException(msg); - } - progress.progress(); - } - } finally { - out.close(); - } - } - - /** - * Choose a datanode (hostname:portnumber). The datanode is chosen at - * random from the live datanodes. - * @param locationsToAvoid locations to avoid. - * @return A datanode - * @throws IOException - */ - private DatanodeInfo chooseDatanode(DatanodeInfo[] locationsToAvoid) - throws IOException { - DistributedFileSystem dfs = getDFS(new Path("/")); - DatanodeInfo[] live = - dfs.getClient().datanodeReport(DatanodeReportType.LIVE); - LOG.info("Choosing a datanode from " + live.length + - " live nodes while avoiding " + locationsToAvoid.length); - Random rand = new Random(); - DatanodeInfo chosen = null; - int maxAttempts = 1000; - for (int i = 0; i < maxAttempts && chosen == null; i++) { - int idx = rand.nextInt(live.length); - chosen = live[idx]; - for (DatanodeInfo avoid: locationsToAvoid) { - if (chosen.getName().equals(avoid.getName())) { - LOG.info("Avoiding " + avoid.getName()); - chosen = null; - break; - } - } - } - if (chosen == null) { - throw new IOException("Could not choose datanode"); - } - LOG.info("Choosing datanode " + chosen.getName()); - return chosen; - } - - /** - * Reads data from the data stream provided and computes metadata. - */ - static DataInputStream computeMetadata(Configuration conf, - InputStream dataStream) - throws IOException { - ByteArrayOutputStream mdOutBase = new ByteArrayOutputStream(1024*1024); - DataOutputStream mdOut = new DataOutputStream(mdOutBase); - - // First, write out the version. - mdOut.writeShort(BlockMetadataHeader.VERSION); - - // Create a summer and write out its header. - int bytesPerChecksum = conf.getInt("dfs.bytes-per-checksum", 512); - DataChecksum sum = - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, - bytesPerChecksum); - sum.writeHeader(mdOut); - - // Buffer to read in a chunk of data. - byte[] buf = new byte[bytesPerChecksum]; - // Buffer to store the checksum bytes. - byte[] chk = new byte[sum.getChecksumSize()]; - - // Read data till we reach the end of the input stream. - int bytesSinceFlush = 0; - while (true) { - // Read some bytes. - int bytesRead = dataStream.read(buf, bytesSinceFlush, - bytesPerChecksum-bytesSinceFlush); - if (bytesRead == -1) { - if (bytesSinceFlush > 0) { - boolean reset = true; - sum.writeValue(chk, 0, reset); // This also resets the sum. - // Write the checksum to the stream. - mdOut.write(chk, 0, chk.length); - bytesSinceFlush = 0; - } - break; - } - // Update the checksum. - sum.update(buf, bytesSinceFlush, bytesRead); - bytesSinceFlush += bytesRead; - - // Flush the checksum if necessary. - if (bytesSinceFlush == bytesPerChecksum) { - boolean reset = true; - sum.writeValue(chk, 0, reset); // This also resets the sum. - // Write the checksum to the stream. - mdOut.write(chk, 0, chk.length); - bytesSinceFlush = 0; - } - } - - byte[] mdBytes = mdOutBase.toByteArray(); - return new DataInputStream(new ByteArrayInputStream(mdBytes)); - } - - private void computeMetadataAndSendFixedBlock(DatanodeInfo datanode, - File localBlockFile, - LocatedBlock block, - long blockSize) - throws IOException { - - LOG.info("Computing metdata"); - InputStream blockContents = null; - DataInputStream blockMetadata = null; - try { - blockContents = new FileInputStream(localBlockFile); - blockMetadata = computeMetadata(getConf(), blockContents); - blockContents.close(); - // Reopen - blockContents = new FileInputStream(localBlockFile); - sendFixedBlock(datanode, blockContents, blockMetadata, block, - blockSize); - } finally { - if (blockContents != null) { - blockContents.close(); - blockContents = null; - } - if (blockMetadata != null) { - blockMetadata.close(); - blockMetadata = null; - } - } - } - - /** - * Send a generated block to a datanode. - * @param datanode Chosen datanode name in host:port form. - * @param blockContents Stream with the block contents. - * @param corruptBlock Block identifying the block to be sent. - * @param blockSize size of the block. - * @throws IOException - */ - private void sendFixedBlock(DatanodeInfo datanode, - final InputStream blockContents, - DataInputStream metadataIn, - LocatedBlock block, long blockSize) - throws IOException { - InetSocketAddress target = NetUtils.createSocketAddr(datanode.getName()); - Socket sock = SocketChannel.open().socket(); - - int readTimeout = - getConf().getInt(BLOCKFIX_READ_TIMEOUT, - HdfsServerConstants.READ_TIMEOUT); - NetUtils.connect(sock, target, readTimeout); - sock.setSoTimeout(readTimeout); - - int writeTimeout = getConf().getInt(BLOCKFIX_WRITE_TIMEOUT, - HdfsServerConstants.WRITE_TIMEOUT); - - OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout); - DataOutputStream out = - new DataOutputStream(new BufferedOutputStream(baseStream, - HdfsConstants. - SMALL_BUFFER_SIZE)); - - boolean corruptChecksumOk = false; - boolean chunkOffsetOK = false; - boolean verifyChecksum = true; - boolean transferToAllowed = false; - - try { - LOG.info("Sending block " + block.getBlock() + - " from " + sock.getLocalSocketAddress().toString() + - " to " + sock.getRemoteSocketAddress().toString() + - " " + blockSize + " bytes"); - RaidBlockSender blockSender = - new RaidBlockSender(block.getBlock(), blockSize, 0, blockSize, - corruptChecksumOk, chunkOffsetOK, verifyChecksum, - transferToAllowed, metadataIn, - new RaidBlockSender.InputStreamFactory() { - @Override - public InputStream - createStream(long offset) throws IOException { - // we are passing 0 as the offset above, - // so we can safely ignore - // the offset passed - return blockContents; - } - }); - - DatanodeInfo[] nodes = new DatanodeInfo[]{datanode}; - DataChecksum checksum = blockSender.getChecksum(); - new Sender(out).writeBlock(block.getBlock(), block.getBlockToken(), "", - nodes, null, BlockConstructionStage.PIPELINE_SETUP_CREATE, - 1, 0L, blockSize, 0L, DataChecksum.newDataChecksum( - checksum.getChecksumType(), checksum.getBytesPerChecksum())); - blockSender.sendBlock(out, baseStream); - - LOG.info("Sent block " + block.getBlock() + " to " + datanode.getName()); - } finally { - out.close(); - } - } - - /** - * returns the source file corresponding to a parity file - */ - Path sourcePathFromParityPath(Path parityPath) { - String parityPathStr = parityPath.toUri().getPath(); - if (parityPathStr.startsWith(xorPrefix)) { - // Remove the prefix to get the source file. - String src = parityPathStr.replaceFirst(xorPrefix, "/"); - return new Path(src); - } else if (parityPathStr.startsWith(rsPrefix)) { - // Remove the prefix to get the source file. - String src = parityPathStr.replaceFirst(rsPrefix, "/"); - return new Path(src); - } - return null; - } - - /** - * Returns the corrupt blocks in a file. - */ - List corruptBlocksInFile(DistributedFileSystem fs, - String uriPath, FileStatus stat) - throws IOException { - List corrupt = new LinkedList(); - LocatedBlocks locatedBlocks = - RaidDFSUtil.getBlockLocations(fs, uriPath, 0, stat.getLen()); - for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) { - if (b.isCorrupt() || - (b.getLocations().length == 0 && b.getBlockSize() > 0)) { - corrupt.add(b); - } - } - return corrupt; - } - } - -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java deleted file mode 100644 index 0ae288dcb43..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java +++ /dev/null @@ -1,408 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.w3c.dom.Text; -import org.xml.sax.SAXException; - -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.raid.protocol.PolicyList; - -/** - * Maintains the configuration xml file that is read into memory. - */ -class ConfigManager { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.ConfigManager"); - - /** Time to wait between checks of the config file */ - public static final long RELOAD_INTERVAL = 10 * 1000; - - /** Time to wait between successive runs of all policies */ - public static final long RESCAN_INTERVAL = 3600 * 1000; - - public static final long HAR_PARTFILE_SIZE = 10 * 1024 * 1024 * 1024l; - - public static final int DISTRAID_MAX_JOBS = 10; - - public static final int DISTRAID_MAX_FILES = 10000; - - /** - * Time to wait after the config file has been modified before reloading it - * (this is done to prevent loading a file that hasn't been fully written). - */ - public static final long RELOAD_WAIT = 5 * 1000; - - private Configuration conf; // Hadoop configuration - private String configFileName; // Path to config XML file - - private long lastReloadAttempt; // Last time we tried to reload the config file - private long lastSuccessfulReload; // Last time we successfully reloaded config - private boolean lastReloadAttemptFailed = false; - private long reloadInterval = RELOAD_INTERVAL; - private long periodicity; // time between runs of all policies - private long harPartfileSize; - private int maxJobsPerPolicy; // Max no. of jobs running simultaneously for - // a job. - private int maxFilesPerJob; // Max no. of files raided by a job. - - // Reload the configuration - private boolean doReload; - private Thread reloadThread; - private volatile boolean running = false; - - // Collection of all configured policies. - Collection allPolicies = new ArrayList(); - - public ConfigManager(Configuration conf) throws IOException, SAXException, - RaidConfigurationException, ClassNotFoundException, ParserConfigurationException { - this.conf = conf; - this.configFileName = conf.get("raid.config.file"); - this.doReload = conf.getBoolean("raid.config.reload", true); - this.reloadInterval = conf.getLong("raid.config.reload.interval", RELOAD_INTERVAL); - this.periodicity = conf.getLong("raid.policy.rescan.interval", RESCAN_INTERVAL); - this.harPartfileSize = conf.getLong("raid.har.partfile.size", HAR_PARTFILE_SIZE); - this.maxJobsPerPolicy = conf.getInt("raid.distraid.max.jobs", - DISTRAID_MAX_JOBS); - this.maxFilesPerJob = conf.getInt("raid.distraid.max.files", - DISTRAID_MAX_FILES); - if (configFileName == null) { - String msg = "No raid.config.file given in conf - " + - "the Hadoop Raid utility cannot run. Aborting...."; - LOG.warn(msg); - throw new IOException(msg); - } - reloadConfigs(); - lastSuccessfulReload = RaidNode.now(); - lastReloadAttempt = RaidNode.now(); - running = true; - } - - /** - * Reload config file if it hasn't been loaded in a while - * Returns true if the file was reloaded. - */ - public synchronized boolean reloadConfigsIfNecessary() { - long time = RaidNode.now(); - if (time > lastReloadAttempt + reloadInterval) { - lastReloadAttempt = time; - try { - File file = new File(configFileName); - long lastModified = file.lastModified(); - if (lastModified > lastSuccessfulReload && - time > lastModified + RELOAD_WAIT) { - reloadConfigs(); - lastSuccessfulReload = time; - lastReloadAttemptFailed = false; - return true; - } - } catch (Exception e) { - if (!lastReloadAttemptFailed) { - LOG.error("Failed to reload config file - " + - "will use existing configuration.", e); - } - lastReloadAttemptFailed = true; - } - } - return false; - } - - /** - * Updates the in-memory data structures from the config file. This file is - * expected to be in the following whitespace-separated format: - * - - - - hdfs://dfsname.myhost.com:9000/archive/ - RaidScanMonthly - - targetReplication - 2 - after RAIDing, decrease the replication factor of the file to - this value. - - - - metaReplication - 2 - the replication factor of the RAID meta file - - - - stripeLength - 10 - the number of blocks to RAID together - - - - - - * - * Blank lines and lines starting with # are ignored. - * - * @throws IOException if the config file cannot be read. - * @throws RaidConfigurationException if configuration entries are invalid. - * @throws ClassNotFoundException if user-defined policy classes cannot be loaded - * @throws ParserConfigurationException if XML parser is misconfigured. - * @throws SAXException if config file is malformed. - * @returns A new set of policy categories. - */ - void reloadConfigs() throws IOException, ParserConfigurationException, - SAXException, ClassNotFoundException, RaidConfigurationException { - - if (configFileName == null) { - return; - } - - File file = new File(configFileName); - if (!file.exists()) { - throw new RaidConfigurationException("Configuration file " + configFileName + - " does not exist."); - } - - // Create some temporary hashmaps to hold the new allocs, and we only save - // them in our fields if we have parsed the entire allocs file successfully. - List all = new ArrayList(); - long periodicityValue = periodicity; - - - // Read and parse the configuration file. - // allow include files in configuration file - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - docBuilderFactory.setIgnoringComments(true); - docBuilderFactory.setNamespaceAware(true); - try { - docBuilderFactory.setXIncludeAware(true); - } catch (UnsupportedOperationException e) { - LOG.error("Failed to set setXIncludeAware(true) for raid parser " - + docBuilderFactory + ":" + e, e); - } - LOG.error("Reloading config file " + file); - - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(file); - Element root = doc.getDocumentElement(); - if (!"configuration".equalsIgnoreCase(root.getTagName())) - throw new RaidConfigurationException("Bad configuration file: " + - "top-level element not "); - NodeList elements = root.getChildNodes(); - - Map existingPolicies = - new HashMap(); - // loop through all the configured source paths. - for (int i = 0; i < elements.getLength(); i++) { - Node node = elements.item(i); - if (!(node instanceof Element)) { - continue; - } - Element element = (Element)node; - String elementTagName = element.getTagName(); - if ("srcPath".equalsIgnoreCase(elementTagName)) { - String srcPathPrefix = element.getAttribute("prefix"); - - PolicyList policyList = null; - if (srcPathPrefix != null && srcPathPrefix.length() != 0) { - // Empty srcPath will have no effect but policies will be processed - // This allow us to define some "abstract" policies - policyList = new PolicyList(); - all.add(policyList); - policyList.setSrcPath(conf, srcPathPrefix); - } - - // loop through all the policies for this source path - NodeList policies = element.getChildNodes(); - for (int j = 0; j < policies.getLength(); j++) { - Node node1 = policies.item(j); - if (!(node1 instanceof Element)) { - continue; - } - Element policy = (Element)node1; - if (!"policy".equalsIgnoreCase(policy.getTagName())) { - throw new RaidConfigurationException("Bad configuration file: " + - "Expecting for srcPath " + srcPathPrefix); - } - String policyName = policy.getAttribute("name"); - PolicyInfo curr = new PolicyInfo(policyName, conf); - if (srcPathPrefix != null && srcPathPrefix.length() > 0) { - curr.setSrcPath(srcPathPrefix); - } - // loop through all the properties of this policy - NodeList properties = policy.getChildNodes(); - PolicyInfo parent = null; - for (int k = 0; k < properties.getLength(); k++) { - Node node2 = properties.item(k); - if (!(node2 instanceof Element)) { - continue; - } - Element property = (Element)node2; - String propertyName = property.getTagName(); - if ("erasureCode".equalsIgnoreCase(propertyName)) { - String text = ((Text)property.getFirstChild()).getData().trim(); - LOG.info(policyName + ".erasureCode = " + text); - curr.setErasureCode(text); - } else if ("description".equalsIgnoreCase(propertyName)) { - String text = ((Text)property.getFirstChild()).getData().trim(); - curr.setDescription(text); - } else if ("parentPolicy".equalsIgnoreCase(propertyName)) { - String text = ((Text)property.getFirstChild()).getData().trim(); - parent = existingPolicies.get(text); - } else if ("property".equalsIgnoreCase(propertyName)) { - NodeList nl = property.getChildNodes(); - String pname=null,pvalue=null; - for (int l = 0; l < nl.getLength(); l++){ - Node node3 = nl.item(l); - if (!(node3 instanceof Element)) { - continue; - } - Element item = (Element) node3; - String itemName = item.getTagName(); - if ("name".equalsIgnoreCase(itemName)){ - pname = ((Text)item.getFirstChild()).getData().trim(); - } else if ("value".equalsIgnoreCase(itemName)){ - pvalue = ((Text)item.getFirstChild()).getData().trim(); - } - } - if (pname != null && pvalue != null) { - LOG.info(policyName + "." + pname + " = " + pvalue); - curr.setProperty(pname,pvalue); - } - } else { - LOG.warn("Found bad property " + propertyName + - " for srcPath" + srcPathPrefix + - " policy name " + policyName + - ". Ignoring."); - } - } // done with all properties of this policy - - PolicyInfo pinfo; - if (parent != null) { - pinfo = new PolicyInfo(policyName, conf); - pinfo.copyFrom(parent); - pinfo.copyFrom(curr); - } else { - pinfo = curr; - } - if (policyList != null) { - policyList.add(pinfo); - } - existingPolicies.put(policyName, pinfo); - - } // done with all policies for this srcpath - } - } // done with all srcPaths - setAllPolicies(all); - periodicity = periodicityValue; - return; - } - - - public synchronized long getPeriodicity() { - return periodicity; - } - - public synchronized long getHarPartfileSize() { - return harPartfileSize; - } - - public synchronized int getMaxJobsPerPolicy() { - return maxJobsPerPolicy; - } - - public synchronized int getMaxFilesPerJob() { - return maxFilesPerJob; - } - - /** - * Get a collection of all policies - */ - public synchronized Collection getAllPolicies() { - return new ArrayList(allPolicies); - } - - /** - * Set a collection of all policies - */ - protected synchronized void setAllPolicies(Collection value) { - this.allPolicies = value; - } - - /** - * Start a background thread to reload the config file - */ - void startReload() { - if (doReload) { - reloadThread = new UpdateThread(); - reloadThread.start(); - } - } - - /** - * Stop the background thread that reload the config file - */ - void stopReload() throws InterruptedException { - if (reloadThread != null) { - running = false; - reloadThread.interrupt(); - reloadThread.join(); - reloadThread = null; - } - } - - /** - * A thread which reloads the config file. - */ - private class UpdateThread extends Thread { - private UpdateThread() { - super("Raid update thread"); - } - - public void run() { - while (running) { - try { - Thread.sleep(reloadInterval); - reloadConfigsIfNecessary(); - } catch (InterruptedException e) { - // do nothing - } catch (Exception e) { - LOG.error("Failed to reload config file ", e); - } - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java deleted file mode 100644 index 65060085c6f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.BlockMissingException; -import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; - -/** - * Represents a generic decoder that can be used to read a file with - * corrupt blocks by using the parity file. - * This is an abstract class, concrete subclasses need to implement - * fixErasedBlock. - */ -public abstract class Decoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.Decoder"); - protected Configuration conf; - protected int stripeSize; - protected int paritySize; - protected Random rand; - protected int bufSize; - protected byte[][] readBufs; - protected byte[][] writeBufs; - - Decoder(Configuration conf, int stripeSize, int paritySize) { - this.conf = conf; - this.stripeSize = stripeSize; - this.paritySize = paritySize; - this.rand = new Random(); - this.bufSize = conf.getInt("raid.decoder.bufsize", 1024 * 1024); - this.readBufs = new byte[stripeSize + paritySize][]; - this.writeBufs = new byte[paritySize][]; - allocateBuffers(); - } - - private void allocateBuffers() { - for (int i = 0; i < stripeSize + paritySize; i++) { - readBufs[i] = new byte[bufSize]; - } - for (int i = 0; i < paritySize; i++) { - writeBufs[i] = new byte[bufSize]; - } - } - - private void configureBuffers(long blockSize) { - if ((long)bufSize > blockSize) { - bufSize = (int)blockSize; - allocateBuffers(); - } else if (blockSize % bufSize != 0) { - bufSize = (int)(blockSize / 256L); // heuristic. - if (bufSize == 0) { - bufSize = 1024; - } - bufSize = Math.min(bufSize, 1024 * 1024); - allocateBuffers(); - } - } - - /** - * The interface to generate a decoded file using the good portion of the - * source file and the parity file. - * @param fs The filesystem containing the source file. - * @param srcFile The damaged source file. - * @param parityFs The filesystem containing the parity file. This could be - * different from fs in case the parity file is part of a HAR archive. - * @param parityFile The parity file. - * @param errorOffset Known location of error in the source file. There could - * be additional errors in the source file that are discovered during - * the decode process. - * @param decodedFile The decoded file. This will have the exact same contents - * as the source file on success. - */ - public void decodeFile( - FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, - long errorOffset, Path decodedFile) throws IOException { - - LOG.info("Create " + decodedFile + " for error at " + - srcFile + ":" + errorOffset); - FileStatus srcStat = fs.getFileStatus(srcFile); - long blockSize = srcStat.getBlockSize(); - configureBuffers(blockSize); - // Move the offset to the start of the block. - errorOffset = (errorOffset / blockSize) * blockSize; - - // Create the decoded file. - FSDataOutputStream out = fs.create( - decodedFile, false, conf.getInt("io.file.buffer.size", 64 * 1024), - srcStat.getReplication(), srcStat.getBlockSize()); - - // Open the source file. - FSDataInputStream in = fs.open( - srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); - - // Start copying data block-by-block. - for (long offset = 0; offset < srcStat.getLen(); offset += blockSize) { - long limit = Math.min(blockSize, srcStat.getLen() - offset); - long bytesAlreadyCopied = 0; - if (offset != errorOffset) { - try { - in = fs.open( - srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); - in.seek(offset); - RaidUtils.copyBytes(in, out, readBufs[0], limit); - assert(out.getPos() == offset +limit); - LOG.info("Copied till " + out.getPos() + " from " + srcFile); - continue; - } catch (BlockMissingException e) { - LOG.warn("Encountered BME at " + srcFile + ":" + offset); - bytesAlreadyCopied = out.getPos() - offset; - } catch (ChecksumException e) { - LOG.warn("Encountered CE at " + srcFile + ":" + offset); - bytesAlreadyCopied = out.getPos() - offset; - } - } - // If we are here offset == errorOffset or we got an exception. - // Recover the block starting at offset. - fixErasedBlock(fs, srcFile, parityFs, parityFile, blockSize, offset, - bytesAlreadyCopied, limit, out); - } - out.close(); - - try { - fs.setOwner(decodedFile, srcStat.getOwner(), srcStat.getGroup()); - fs.setPermission(decodedFile, srcStat.getPermission()); - fs.setTimes(decodedFile, srcStat.getModificationTime(), - srcStat.getAccessTime()); - } catch (Exception exc) { - LOG.warn("Didn't manage to copy meta information because of " + exc + - " Ignoring..."); - } - - } - - /** - * Recovers a corrupt block to local file. - * - * @param srcFs The filesystem containing the source file. - * @param srcPath The damaged source file. - * @param parityFs The filesystem containing the parity file. This could be - * different from fs in case the parity file is part of a HAR archive. - * @param parityPath The parity file. - * @param blockSize The block size of the file. - * @param blockOffset Known location of error in the source file. There could - * be additional errors in the source file that are discovered during - * the decode process. - * @param localBlockFile The file to write the block to. - * @param limit The maximum number of bytes to be written out. - * This is to prevent writing beyond the end of the file. - */ - public void recoverBlockToFile( - FileSystem srcFs, Path srcPath, FileSystem parityFs, Path parityPath, - long blockSize, long blockOffset, File localBlockFile, long limit) - throws IOException { - OutputStream out = new FileOutputStream(localBlockFile); - fixErasedBlock(srcFs, srcPath, parityFs, parityPath, - blockSize, blockOffset, 0, limit, out); - out.close(); - } - - /** - * Implementation-specific mechanism of writing a fixed block. - * @param fs The filesystem containing the source file. - * @param srcFile The damaged source file. - * @param parityFs The filesystem containing the parity file. This could be - * different from fs in case the parity file is part of a HAR archive. - * @param parityFile The parity file. - * @param blockSize The maximum size of a block. - * @param errorOffset Known location of error in the source file. There could - * be additional errors in the source file that are discovered during - * the decode process. - * @param bytesToSkip After the block is generated, these many bytes should be - * skipped before writing to the output. This is needed because the - * output may have a portion of the block written from the source file - * before a new corruption is discovered in the block. - * @param limit The maximum number of bytes to be written out, including - * bytesToSkip. This is to prevent writing beyond the end of the file. - * @param out The output. - */ - protected abstract void fixErasedBlock( - FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, - long blockSize, long errorOffset, long bytesToSkip, long limit, - OutputStream out) throws IOException; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java deleted file mode 100644 index c1ff9bccc4f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Stack; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.Semaphore; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; - -/** - * Implements depth-first traversal using a Stack object. The traversal - * can be stopped at any time and the state of traversal is saved. - */ -public class DirectoryTraversal { - public static final Log LOG = - LogFactory.getLog("org.apache.hadoop.raid.DirectoryTraversal"); - - private FileSystem fs; - private List paths; - private int pathIdx = 0; // Next path to process. - private Stack stack = new Stack(); - private ExecutorService executor; - - private int numThreads; - - /** - * A FileFilter object can be used to choose files during directory traversal. - */ - public interface FileFilter { - /** - * @return a boolean value indicating if the file passes the filter. - */ - boolean check(FileStatus f) throws IOException; - } - - /** - * Represents a directory node in directory traversal. - */ - static class Node { - private FileStatus path; // Path that this node represents. - private FileStatus[] elements; // Elements in the node. - private int idx = 0; - - public Node(FileStatus path, FileStatus[] elements) { - this.path = path; - this.elements = elements; - } - - public boolean hasNext() { - return idx < elements.length; - } - - public FileStatus next() { - return elements[idx++]; - } - - public FileStatus path() { - return this.path; - } - } - - /** - * Constructor. - * @param fs The filesystem to use. - * @param startPaths A list of paths that need to be traversed - */ - public DirectoryTraversal(FileSystem fs, List startPaths) { - this(fs, startPaths, 1); - } - - public DirectoryTraversal( - FileSystem fs, List startPaths, int numThreads) { - this.fs = fs; - paths = startPaths; - pathIdx = 0; - this.numThreads = numThreads; - executor = Executors.newFixedThreadPool(numThreads); - } - - public List getFilteredFiles(FileFilter filter, int limit) { - List filtered = new ArrayList(); - if (limit == 0) - return filtered; - - // We need this semaphore to block when the number of running workitems - // is equal to the number of threads. FixedThreadPool limits the number - // of threads, but not the queue size. This way we will limit the memory - // usage. - Semaphore slots = new Semaphore(numThreads); - - while (true) { - FilterFileWorkItem work = null; - try { - slots.acquire(); - synchronized(filtered) { - if (filtered.size() >= limit) { - slots.release(); - break; - } - } - Node next = getNextDirectoryNode(); - if (next == null) { - slots.release(); - break; - } - work = new FilterFileWorkItem(filter, next, filtered, slots); - } catch (InterruptedException ie) { - slots.release(); - break; - } catch (IOException e) { - slots.release(); - break; - } - executor.execute(work); - } - - try { - // Wait for all submitted items to finish. - slots.acquire(numThreads); - // If this traversal is finished, shutdown the executor. - if (doneTraversal()) { - executor.shutdown(); - executor.awaitTermination(1, TimeUnit.HOURS); - } - } catch (InterruptedException ie) { - } - - return filtered; - } - - class FilterFileWorkItem implements Runnable { - FileFilter filter; - Node dir; - List filtered; - Semaphore slots; - - FilterFileWorkItem(FileFilter filter, Node dir, List filtered, - Semaphore slots) { - this.slots = slots; - this.filter = filter; - this.dir = dir; - this.filtered = filtered; - } - - @SuppressWarnings("deprecation") - public void run() { - try { - LOG.info("Initiating file filtering for " + dir.path.getPath()); - for (FileStatus f: dir.elements) { - if (!f.isFile()) { - continue; - } - if (filter.check(f)) { - synchronized(filtered) { - filtered.add(f); - } - } - } - } catch (Exception e) { - LOG.error("Error in directory traversal: " - + StringUtils.stringifyException(e)); - } finally { - slots.release(); - } - } - } - - /** - * Return the next file. - * @throws IOException - */ - public FileStatus getNextFile() throws IOException { - // Check if traversal is done. - while (!doneTraversal()) { - // If traversal is not done, check if the stack is not empty. - while (!stack.isEmpty()) { - // If the stack is not empty, look at the top node. - Node node = stack.peek(); - // Check if the top node has an element. - if (node.hasNext()) { - FileStatus element = node.next(); - // Is the next element a directory. - if (!element.isDir()) { - // It is a file, return it. - return element; - } - // Next element is a directory, push it on to the stack and - // continue - try { - pushNewNode(element); - } catch (FileNotFoundException e) { - // Ignore and move to the next element. - } - continue; - } else { - // Top node has no next element, pop it and continue. - stack.pop(); - continue; - } - } - // If the stack is empty, do we have more paths? - while (!paths.isEmpty()) { - FileStatus next = paths.remove(0); - pathIdx++; - if (!next.isDir()) { - return next; - } - try { - pushNewNode(next); - } catch (FileNotFoundException e) { - continue; - } - break; - } - } - return null; - } - - /** - * Gets the next directory in the tree. The algorithm returns deeper directories - * first. - * @return A FileStatus representing the directory. - * @throws IOException - */ - public FileStatus getNextDirectory() throws IOException { - Node dirNode = getNextDirectoryNode(); - if (dirNode != null) { - return dirNode.path; - } - return null; - } - - private Node getNextDirectoryNode() throws IOException { - - // Check if traversal is done. - while (!doneTraversal()) { - // If traversal is not done, check if the stack is not empty. - while (!stack.isEmpty()) { - // If the stack is not empty, look at the top node. - Node node = stack.peek(); - // Check if the top node has an element. - if (node.hasNext()) { - FileStatus element = node.next(); - // Is the next element a directory. - if (element.isDir()) { - // Next element is a directory, push it on to the stack and - // continue - try { - pushNewNode(element); - } catch (FileNotFoundException e) { - // Ignore and move to the next element. - } - continue; - } - } else { - stack.pop(); - return node; - } - } - // If the stack is empty, do we have more paths? - while (!paths.isEmpty()) { - FileStatus next = paths.remove(0); - pathIdx++; - if (next.isDir()) { - try { - pushNewNode(next); - } catch (FileNotFoundException e) { - continue; - } - break; - } - } - } - return null; - } - - private void pushNewNode(FileStatus stat) throws IOException { - if (!stat.isDir()) { - return; - } - Path p = stat.getPath(); - FileStatus[] elements = fs.listStatus(p); - Node newNode = new Node(stat, (elements == null? new FileStatus[0]: elements)); - stack.push(newNode); - } - - public boolean doneTraversal() { - return paths.isEmpty() && stack.isEmpty(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java deleted file mode 100644 index 9779e31924e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java +++ /dev/null @@ -1,660 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.io.PrintStream; -import java.io.InputStreamReader; -import java.io.BufferedReader; -import java.util.List; -import java.util.LinkedList; -import java.util.Map; -import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Date; -import java.text.SimpleDateFormat; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; - -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.RaidDFSUtil; - -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; - -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; - -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.InputSplit; - -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.FileSplit; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; - -/** - * distributed block fixer, uses map reduce jobs to fix corrupt files - * - * configuration options - * raid.blockfix.filespertask - number of corrupt files to fix in a single - * map reduce task (i.e., at one mapper node) - * - * raid.blockfix.maxpendingfiles - maximum number of files to fix - * simultaneously - */ -public class DistBlockFixer extends BlockFixer { - // volatile should be sufficient since only the block fixer thread - // updates numJobsRunning (other threads may read) - private volatile int numJobsRunning = 0; - - private static final String WORK_DIR_PREFIX = "blockfixer"; - private static final String IN_FILE_SUFFIX = ".in"; - private static final String PART_PREFIX = "part-"; - - private static final String BLOCKFIX_FILES_PER_TASK = - "raid.blockfix.filespertask"; - private static final String BLOCKFIX_MAX_PENDING_FILES = - "raid.blockfix.maxpendingfiles"; - - // default number of files to fix in a task - private static final long DEFAULT_BLOCKFIX_FILES_PER_TASK = 10L; - - // default number of files to fix simultaneously - private static final long DEFAULT_BLOCKFIX_MAX_PENDING_FILES = 1000L; - - protected static final Log LOG = LogFactory.getLog(DistBlockFixer.class); - - // number of files to fix in a task - private long filesPerTask; - - // number of files to fix simultaneously - final private long maxPendingFiles; - - // number of files being fixed right now - private long pendingFiles; - - private long lastCheckTime; - - private final SimpleDateFormat dateFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - - private Map fileIndex = - new HashMap(); - private Map> jobIndex = - new HashMap>(); - - static enum Counter { - FILES_SUCCEEDED, FILES_FAILED, FILES_NOACTION - } - - public DistBlockFixer(Configuration conf) { - super(conf); - filesPerTask = DistBlockFixer.filesPerTask(getConf()); - maxPendingFiles = DistBlockFixer.maxPendingFiles(getConf()); - pendingFiles = 0L; - - // start off due for the first iteration - lastCheckTime = Time.now() - blockFixInterval; - } - - /** - * determines how many files to fix in a single task - */ - protected static long filesPerTask(Configuration conf) { - return conf.getLong(BLOCKFIX_FILES_PER_TASK, - DEFAULT_BLOCKFIX_FILES_PER_TASK); - - } - /** - * determines how many files to fix simultaneously - */ - protected static long maxPendingFiles(Configuration conf) { - return conf.getLong(BLOCKFIX_MAX_PENDING_FILES, - DEFAULT_BLOCKFIX_MAX_PENDING_FILES); - } - - /** - * runs the block fixer periodically - */ - public void run() { - while (running) { - // check if it is time to run the block fixer - long now = Time.now(); - if (now >= lastCheckTime + blockFixInterval) { - lastCheckTime = now; - try { - checkAndFixBlocks(now); - } catch (InterruptedException ignore) { - LOG.info("interrupted"); - } catch (Exception e) { - // log exceptions and keep running - LOG.error(StringUtils.stringifyException(e)); - } catch (Error e) { - LOG.error(StringUtils.stringifyException(e)); - throw e; - } - } - - // try to sleep for the remainder of the interval - long sleepPeriod = (lastCheckTime - Time.now()) + - blockFixInterval; - - if ((sleepPeriod > 0L) && running) { - try { - Thread.sleep(sleepPeriod); - } catch (InterruptedException ignore) { - LOG.info("interrupted"); - } - } - } - } - - /** - * checks for corrupt blocks and fixes them (if any) - */ - private void checkAndFixBlocks(long startTime) - throws IOException, InterruptedException, ClassNotFoundException { - checkJobs(); - - if (pendingFiles >= maxPendingFiles) { - return; - } - - List corruptFiles = getCorruptFiles(); - filterUnfixableSourceFiles(corruptFiles.iterator()); - - String startTimeStr = dateFormat.format(new Date(startTime)); - - LOG.info("found " + corruptFiles.size() + " corrupt files"); - - if (corruptFiles.size() > 0) { - String jobName = "blockfixer." + startTime; - startJob(jobName, corruptFiles); - } - } - - /** - * Handle a failed job. - */ - private void failJob(Job job) throws IOException { - // assume no files have been fixed - LOG.error("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() + - ") finished (failed)"); - for (CorruptFileInfo fileInfo: jobIndex.get(job)) { - fileInfo.fail(); - } - numJobsRunning--; - } - - /** - * Handle a successful job. - */ - private void succeedJob(Job job, long filesSucceeded, long filesFailed) - throws IOException { - LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() + - ") finished (succeeded)"); - - if (filesFailed == 0) { - // no files have failed - for (CorruptFileInfo fileInfo: jobIndex.get(job)) { - fileInfo.succeed(); - } - } else { - // we have to look at the output to check which files have failed - Set failedFiles = getFailedFiles(job); - - for (CorruptFileInfo fileInfo: jobIndex.get(job)) { - if (failedFiles.contains(fileInfo.getFile().toString())) { - fileInfo.fail(); - } else { - // call succeed for files that have succeeded or for which no action - // was taken - fileInfo.succeed(); - } - } - } - // report succeeded files to metrics - incrFilesFixed(filesSucceeded); - numJobsRunning--; - } - - /** - * checks if jobs have completed and updates job and file index - * returns a list of failed files for restarting - */ - private void checkJobs() throws IOException { - Iterator jobIter = jobIndex.keySet().iterator(); - while(jobIter.hasNext()) { - Job job = jobIter.next(); - - try { - if (job.isComplete()) { - long filesSucceeded = - job.getCounters().findCounter(Counter.FILES_SUCCEEDED).getValue(); - long filesFailed = - job.getCounters().findCounter(Counter.FILES_FAILED).getValue(); - long filesNoAction = - job.getCounters().findCounter(Counter.FILES_NOACTION).getValue(); - int files = jobIndex.get(job).size(); - if (job.isSuccessful() && - (filesSucceeded + filesFailed + filesNoAction == - ((long) files))) { - // job has processed all files - succeedJob(job, filesSucceeded, filesFailed); - } else { - failJob(job); - } - jobIter.remove(); - } else { - LOG.info("job " + job.getJobName() + " still running"); - } - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - failJob(job); - try { - job.killJob(); - } catch (Exception ee) { - LOG.error(StringUtils.stringifyException(ee)); - } - jobIter.remove(); - } - } - purgeFileIndex(); - } - - /** - * determines which files have failed for a given job - */ - private Set getFailedFiles(Job job) throws IOException { - Set failedFiles = new HashSet(); - - Path outDir = SequenceFileOutputFormat.getOutputPath(job); - FileSystem fs = outDir.getFileSystem(getConf()); - if (!fs.getFileStatus(outDir).isDir()) { - throw new IOException(outDir.toString() + " is not a directory"); - } - - FileStatus[] files = fs.listStatus(outDir); - - for (FileStatus f: files) { - Path fPath = f.getPath(); - if ((!f.isDir()) && (fPath.getName().startsWith(PART_PREFIX))) { - LOG.info("opening " + fPath.toString()); - SequenceFile.Reader reader = - new SequenceFile.Reader(fs, fPath, getConf()); - - Text key = new Text(); - Text value = new Text(); - while (reader.next(key, value)) { - failedFiles.add(key.toString()); - } - reader.close(); - } - } - return failedFiles; - } - - - /** - * purge expired jobs from the file index - */ - private void purgeFileIndex() { - Iterator fileIter = fileIndex.keySet().iterator(); - while(fileIter.hasNext()) { - String file = fileIter.next(); - if (fileIndex.get(file).isExpired()) { - fileIter.remove(); - } - } - - } - - /** - * creates and submits a job, updates file index and job index - */ - private Job startJob(String jobName, List corruptFiles) - throws IOException, InterruptedException, ClassNotFoundException { - Path inDir = new Path(WORK_DIR_PREFIX + "/in/" + jobName); - Path outDir = new Path(WORK_DIR_PREFIX + "/out/" + jobName); - List filesInJob = createInputFile(jobName, inDir, corruptFiles); - - Configuration jobConf = new Configuration(getConf()); - Job job = new Job(jobConf, jobName); - job.setJarByClass(getClass()); - job.setMapperClass(DistBlockFixerMapper.class); - job.setNumReduceTasks(0); - job.setInputFormatClass(DistBlockFixerInputFormat.class); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Text.class); - - DistBlockFixerInputFormat.setInputPaths(job, inDir); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - job.submit(); - LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() + - ") started"); - - // submit the job before inserting it into the index - // this way, if submit fails, we won't have added anything to the index - insertJob(job, filesInJob); - return job; - } - - /** - * inserts new job into file index and job index - */ - private void insertJob(Job job, List corruptFiles) { - List fileInfos = new LinkedList(); - - for (Path file: corruptFiles) { - CorruptFileInfo fileInfo = new CorruptFileInfo(file, job); - fileInfos.add(fileInfo); - fileIndex.put(file.toString(), fileInfo); - } - - jobIndex.put(job, fileInfos); - numJobsRunning++; - } - - /** - * creates the input file (containing the names of the files to be fixed - */ - private List createInputFile(String jobName, Path inDir, - List corruptFiles) - throws IOException { - - Path file = new Path(inDir, jobName + IN_FILE_SUFFIX); - FileSystem fs = file.getFileSystem(getConf()); - SequenceFile.Writer fileOut = SequenceFile.createWriter(fs, getConf(), file, - LongWritable.class, - Text.class); - long index = 0L; - - List filesAdded = new LinkedList(); - - for (Path corruptFile: corruptFiles) { - if (pendingFiles >= maxPendingFiles) { - break; - } - - String corruptFileName = corruptFile.toString(); - fileOut.append(new LongWritable(index++), new Text(corruptFileName)); - filesAdded.add(corruptFile); - pendingFiles++; - - if (index % filesPerTask == 0) { - fileOut.sync(); // create sync point to make sure we can split here - } - } - - fileOut.close(); - return filesAdded; - } - - /** - * gets a list of corrupt files from the name node - * and filters out files that are currently being fixed or - * that were recently fixed - */ - private List getCorruptFiles() throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem) - (new Path("/")).getFileSystem(getConf()); - - String[] files = RaidDFSUtil.getCorruptFiles(dfs); - List corruptFiles = new LinkedList(); - - for (String f: files) { - Path p = new Path(f); - // filter out files that are being fixed or that were recently fixed - if (!fileIndex.containsKey(p.toString())) { - corruptFiles.add(p); - } - } - RaidUtils.filterTrash(getConf(), corruptFiles); - - return corruptFiles; - } - - /** - * returns the number of map reduce jobs running - */ - public int jobsRunning() { - return numJobsRunning; - } - - /** - * hold information about a corrupt file that is being fixed - */ - class CorruptFileInfo { - - private Path file; - private Job job; - private boolean done; - private long time; - - public CorruptFileInfo(Path file, Job job) { - this.file = file; - this.job = job; - this.done = false; - this.time = 0; - } - - public boolean isDone() { - return done; - } - - public boolean isExpired() { - return done && ((Time.now() - time) > historyInterval); - } - - public Path getFile() { - return file; - } - - /** - * updates file index to record a failed attempt at fixing a file, - * immediately removes the entry from the file index - * (instead of letting it expire) - * so that we can retry right away - */ - public void fail() { - // remove this file from the index - CorruptFileInfo removed = fileIndex.remove(file.toString()); - if (removed == null) { - LOG.error("trying to remove file not in file index: " + - file.toString()); - } else { - LOG.error("fixing " + file.toString() + " failed"); - } - pendingFiles--; - } - - /** - * marks a file as fixed successfully - * and sets time stamp for expiry after specified interval - */ - public void succeed() { - // leave the file in the index, - // will be pruged later - job = null; - done = true; - time = Time.now(); - LOG.info("fixing " + file.toString() + " succeeded"); - pendingFiles--; - } - } - - static class DistBlockFixerInputFormat - extends SequenceFileInputFormat { - - protected static final Log LOG = - LogFactory.getLog(DistBlockFixerMapper.class); - - /** - * splits the input files into tasks handled by a single node - * we have to read the input files to do this based on a number of - * items in a sequence - */ - @Override - public List getSplits(JobContext job) - throws IOException { - long filesPerTask = DistBlockFixer.filesPerTask(job.getConfiguration()); - - Path[] inPaths = getInputPaths(job); - - List splits = new LinkedList(); - - long fileCounter = 0; - - for (Path inPath: inPaths) { - - FileSystem fs = inPath.getFileSystem(job.getConfiguration()); - - if (!fs.getFileStatus(inPath).isDir()) { - throw new IOException(inPath.toString() + " is not a directory"); - } - - FileStatus[] inFiles = fs.listStatus(inPath); - - for (FileStatus inFileStatus: inFiles) { - Path inFile = inFileStatus.getPath(); - - if (!inFileStatus.isDir() && - (inFile.getName().equals(job.getJobName() + IN_FILE_SUFFIX))) { - - fileCounter++; - SequenceFile.Reader inFileReader = - new SequenceFile.Reader(fs, inFile, job.getConfiguration()); - - long startPos = inFileReader.getPosition(); - long counter = 0; - - // create an input split every filesPerTask items in the sequence - LongWritable key = new LongWritable(); - Text value = new Text(); - try { - while (inFileReader.next(key, value)) { - if (counter % filesPerTask == filesPerTask - 1L) { - splits.add(new FileSplit(inFile, startPos, - inFileReader.getPosition() - - startPos, - null)); - startPos = inFileReader.getPosition(); - } - counter++; - } - - // create input split for remaining items if necessary - // this includes the case where no splits were created by the loop - if (startPos != inFileReader.getPosition()) { - splits.add(new FileSplit(inFile, startPos, - inFileReader.getPosition() - startPos, - null)); - } - } finally { - inFileReader.close(); - } - } - } - } - - LOG.info("created " + splits.size() + " input splits from " + - fileCounter + " files"); - - return splits; - } - - /** - * indicates that input file can be split - */ - @Override - public boolean isSplitable (JobContext job, Path file) { - return true; - } - } - - - /** - * mapper for fixing stripes with corrupt blocks - */ - static class DistBlockFixerMapper - extends Mapper { - - protected static final Log LOG = - LogFactory.getLog(DistBlockFixerMapper.class); - - /** - * fix a stripe - */ - @Override - public void map(LongWritable key, Text fileText, Context context) - throws IOException, InterruptedException { - - BlockFixerHelper helper = - new BlockFixerHelper(context.getConfiguration()); - - String fileStr = fileText.toString(); - LOG.info("fixing " + fileStr); - - Path file = new Path(fileStr); - boolean success = false; - - try { - boolean fixed = helper.fixFile(file, context); - - if (fixed) { - context.getCounter(Counter.FILES_SUCCEEDED).increment(1L); - } else { - context.getCounter(Counter.FILES_NOACTION).increment(1L); - } - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - - // report file as failed - context.getCounter(Counter.FILES_FAILED).increment(1L); - String outkey = fileStr; - String outval = "failed"; - context.write(new Text(outkey), new Text(outval)); - } - - context.progress(); - } - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java deleted file mode 100644 index 5c6e5cc79dc..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java +++ /dev/null @@ -1,374 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.Date; -import java.text.SimpleDateFormat; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.SequenceFile.Writer; -import org.apache.hadoop.io.SequenceFile.Reader; - -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.FileSplit; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.raid.RaidNode.Statistics; -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.util.StringUtils; - -public class DistRaid extends Configured { - - protected static final Log LOG = LogFactory.getLog(DistRaid.class); - - static final String NAME = "distRaid"; - static final String JOB_DIR_LABEL = NAME + ".job.dir"; - static final int OP_LIST_BLOCK_SIZE = 32 * 1024 * 1024; // block size of control file - static final short OP_LIST_REPLICATION = 10; // replication factor of control file - - public static final String OPS_PER_TASK = "raid.distraid.opspertask"; - private static final int DEFAULT_OPS_PER_TASK = 100; - private static final int SYNC_FILE_MAX = 10; - private static final SimpleDateFormat dateForm = new SimpleDateFormat("yyyy-MM-dd HH:mm"); - - static enum Counter { - FILES_SUCCEEDED, FILES_FAILED, PROCESSED_BLOCKS, PROCESSED_SIZE, META_BLOCKS, META_SIZE - } - - public DistRaid(Configuration conf) { - super(conf); - } - - private static final Random RANDOM = new Random(); - - protected static String getRandomId() { - return Integer.toString(RANDOM.nextInt(Integer.MAX_VALUE), 36); - } - - /** - * - * helper class which holds the policy and paths - * - */ - public static class RaidPolicyPathPair { - public PolicyInfo policy; - public List srcPaths; - - RaidPolicyPathPair(PolicyInfo policy, List srcPaths) { - this.policy = policy; - this.srcPaths = srcPaths; - } - } - - List raidPolicyPathPairList = new ArrayList(); - - private Job runningJob; - private String lastReport = null; - - /** Responsible for generating splits of the src file list. */ - static class DistRaidInputFormat extends - SequenceFileInputFormat { - /** - * Produce splits such that each is no greater than the quotient of the - * total size and the number of splits requested. - * - * @param job - * The handle to the Configuration object - * @param numSplits - * Number of splits requested - */ - public List getSplits(JobContext job) throws IOException { - Configuration conf = job.getConfiguration(); - - // We create only one input file. So just get the first file in the first - // input directory. - Path inDir = getInputPaths(job)[0]; - FileSystem fs = inDir.getFileSystem(conf); - FileStatus[] inputFiles = fs.listStatus(inDir); - Path inputFile = inputFiles[0].getPath(); - - List splits = new ArrayList(); - SequenceFile.Reader in = - new SequenceFile.Reader(conf, Reader.file(inputFile)); - long prev = 0L; - final int opsPerTask = conf.getInt(OPS_PER_TASK, DEFAULT_OPS_PER_TASK); - try { - Text key = new Text(); - PolicyInfo value = new PolicyInfo(); - int count = 0; // count src - while (in.next(key, value)) { - long curr = in.getPosition(); - long delta = curr - prev; - if (++count > opsPerTask) { - count = 0; - splits.add(new FileSplit(inputFile, prev, delta, (String[]) null)); - prev = curr; - } - } - } finally { - in.close(); - } - long remaining = fs.getFileStatus(inputFile).getLen() - prev; - if (remaining != 0) { - splits.add(new FileSplit(inputFile, prev, remaining, (String[]) null)); - } - return splits; - } - } - - /** The mapper for raiding files. */ - static class DistRaidMapper extends Mapper { - private boolean ignoreFailures = false; - - private int failcount = 0; - private int succeedcount = 0; - private Statistics st = new Statistics(); - - private String getCountString() { - return "Succeeded: " + succeedcount + " Failed: " + failcount; - } - - /** Run a FileOperation - * @throws IOException - * @throws InterruptedException */ - public void map(Text key, PolicyInfo policy, Context context) - throws IOException, InterruptedException { - try { - Configuration jobConf = context.getConfiguration(); - LOG.info("Raiding file=" + key.toString() + " policy=" + policy); - Path p = new Path(key.toString()); - FileStatus fs = p.getFileSystem(jobConf).getFileStatus(p); - st.clear(); - RaidNode.doRaid(jobConf, policy, fs, st, context); - - ++succeedcount; - - context.getCounter(Counter.PROCESSED_BLOCKS).increment(st.numProcessedBlocks); - context.getCounter(Counter.PROCESSED_SIZE).increment(st.processedSize); - context.getCounter(Counter.META_BLOCKS).increment(st.numMetaBlocks); - context.getCounter(Counter.META_SIZE).increment(st.metaSize); - context.getCounter(Counter.FILES_SUCCEEDED).increment(1); - } catch (IOException e) { - ++failcount; - context.getCounter(Counter.FILES_FAILED).increment(1); - - String s = "FAIL: " + policy + ", " + key + " " - + StringUtils.stringifyException(e); - context.write(new Text(key), new Text(s)); - LOG.error(s); - } finally { - context.setStatus(getCountString()); - } - } - - /** {@inheritDoc} */ - public void close() throws IOException { - if (failcount == 0 || ignoreFailures) { - return; - } - throw new IOException(getCountString()); - } - } - - /** - * Set options specified in raid.scheduleroption. - * The string should be formatted as key:value[,key:value]* - */ - static void setSchedulerOption(Configuration conf) { - String schedulerOption = conf.get("raid.scheduleroption"); - if (schedulerOption != null) { - // Parse the scheduler option to get key:value pairs. - String[] keyValues = schedulerOption.trim().split(","); - for (String keyValue: keyValues) { - String[] fields = keyValue.trim().split(":"); - String key = fields[0].trim(); - String value = fields[1].trim(); - conf.set(key, value); - } - } - } - - /** - * Creates a new Job object. - * @param conf - * @return a Job object - * @throws IOException - */ - static Job createJob(Configuration jobConf) throws IOException { - String jobName = NAME + " " + dateForm.format(new Date(RaidNode.now())); - - setSchedulerOption(jobConf); - - Job job = Job.getInstance(jobConf, jobName); - job.setSpeculativeExecution(false); - job.setJarByClass(DistRaid.class); - job.setInputFormatClass(DistRaidInputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Text.class); - - job.setMapperClass(DistRaidMapper.class); - job.setNumReduceTasks(0); - - return job; - } - - /** Add paths to be raided */ - public void addRaidPaths(PolicyInfo info, List paths) { - raidPolicyPathPairList.add(new RaidPolicyPathPair(info, paths)); - } - - /** Invokes a map-reduce job do parallel raiding. - * @return true if the job was started, false otherwise - * @throws InterruptedException - */ - public boolean startDistRaid() throws IOException { - assert(raidPolicyPathPairList.size() > 0); - Job job = createJob(getConf()); - createInputFile(job); - try { - job.submit(); - this.runningJob = job; - LOG.info("Job Started: " + runningJob.getJobID()); - return true; - } catch (ClassNotFoundException e) { - throw new IOException(e); - } catch (InterruptedException e) { - return false; - } - } - - /** Checks if the map-reduce job has completed. - * - * @return true if the job completed, false otherwise. - * @throws IOException - */ - public boolean checkComplete() throws IOException { - JobID jobID = runningJob.getJobID(); - LOG.info("Checking job " + jobID); - try { - if (runningJob.isComplete()) { - // delete job directory - Configuration jobConf = runningJob.getConfiguration(); - final String jobdir = jobConf.get(JOB_DIR_LABEL); - if (jobdir != null) { - final Path jobpath = new Path(jobdir); - jobpath.getFileSystem(jobConf).delete(jobpath, true); - } - if (runningJob.isSuccessful()) { - LOG.info("Job Complete(Succeeded): " + jobID); - } else { - LOG.error("Job Complete(Failed): " + jobID); - } - raidPolicyPathPairList.clear(); - return true; - } else { - String report = (" job " + jobID + - " map " + StringUtils.formatPercent(runningJob.mapProgress(), 0)+ - " reduce " + StringUtils.formatPercent(runningJob.reduceProgress(), 0)); - if (!report.equals(lastReport)) { - LOG.info(report); - lastReport = report; - } - return false; - } - } catch (InterruptedException e) { - return false; - } - } - - public boolean successful() throws IOException { - try { - return runningJob.isSuccessful(); - } catch (InterruptedException e) { - return false; - } - } - - /** - * set up input file which has the list of input files. - * - * @return boolean - * @throws IOException - */ - private void createInputFile(Job job) throws IOException { - Configuration jobConf = job.getConfiguration(); - Path jobDir = new Path(JOB_DIR_LABEL + getRandomId()); - Path inDir = new Path(jobDir, "in"); - Path outDir = new Path(jobDir, "out"); - FileInputFormat.setInputPaths(job, inDir); - FileOutputFormat.setOutputPath(job, outDir); - Path opList = new Path(inDir, NAME); - - Configuration tmp = new Configuration(jobConf); - // The control file should have small size blocks. This helps - // in spreading out the load from mappers that will be spawned. - tmp.setInt("dfs.blocks.size", OP_LIST_BLOCK_SIZE); - FileSystem fs = opList.getFileSystem(tmp); - - int opCount = 0, synCount = 0; - SequenceFile.Writer opWriter = null; - try { - opWriter = SequenceFile.createWriter( - jobConf, Writer.file(opList), Writer.keyClass(Text.class), - Writer.valueClass(PolicyInfo.class), - Writer.compression(SequenceFile.CompressionType.NONE)); - for (RaidPolicyPathPair p : raidPolicyPathPairList) { - // If a large set of files are Raided for the first time, files - // in the same directory that tend to have the same size will end up - // with the same map. This shuffle mixes things up, allowing a better - // mix of files. - java.util.Collections.shuffle(p.srcPaths); - for (FileStatus st : p.srcPaths) { - opWriter.append(new Text(st.getPath().toString()), p.policy); - opCount++; - if (++synCount > SYNC_FILE_MAX) { - opWriter.sync(); - synCount = 0; - } - } - } - - } finally { - if (opWriter != null) { - opWriter.close(); - } - // increase replication for control file - fs.setReplication(opList, OP_LIST_REPLICATION); - } - raidPolicyPathPairList.clear(); - LOG.info("Number of files=" + opCount); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java deleted file mode 100644 index 43be2a725d6..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.util.Daemon; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; - -import org.apache.hadoop.raid.protocol.PolicyInfo; - -/** - * Implementation of {@link RaidNode} that uses map reduce jobs to raid files. - */ -public class DistRaidNode extends RaidNode { - - public static final Log LOG = LogFactory.getLog(DistRaidNode.class); - - /** Daemon thread to monitor raid job progress */ - JobMonitor jobMonitor = null; - Daemon jobMonitorThread = null; - - public DistRaidNode(Configuration conf) throws IOException { - super(conf); - this.jobMonitor = new JobMonitor(conf); - this.jobMonitorThread = new Daemon(this.jobMonitor); - this.jobMonitorThread.start(); - } - - /** - * {@inheritDoc} - */ - @Override - public void join() { - super.join(); - try { - if (jobMonitorThread != null) jobMonitorThread.join(); - } catch (InterruptedException ie) { - // do nothing - } - } - - /** - * {@inheritDoc} - */ - @Override - public void stop() { - if (stopRequested) { - return; - } - super.stop(); - if (jobMonitor != null) jobMonitor.running = false; - if (jobMonitorThread != null) jobMonitorThread.interrupt(); - } - - - /** - * {@inheritDoc} - */ - @Override - void raidFiles(PolicyInfo info, List paths) throws IOException { - // We already checked that no job for this policy is running - // So we can start a new job. - DistRaid dr = new DistRaid(conf); - //add paths for distributed raiding - dr.addRaidPaths(info, paths); - boolean started = dr.startDistRaid(); - if (started) { - jobMonitor.monitorJob(info.getName(), dr); - } - } - - /** - * {@inheritDoc} - */ - @Override - int getRunningJobsForPolicy(String policyName) { - return jobMonitor.runningJobsCount(policyName); - } - - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java deleted file mode 100644 index b87bad4f807..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Progressable; - -/** - * Represents a generic encoder that can generate a parity file for a source - * file. - * This is an abstract class, concrete subclasses need to implement - * encodeFileImpl. - */ -public abstract class Encoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.Encoder"); - protected Configuration conf; - protected int stripeSize; - protected int paritySize; - protected Random rand; - protected int bufSize; - protected byte[][] readBufs; - protected byte[][] writeBufs; - - /** - * A class that acts as a sink for data, similar to /dev/null. - */ - static class NullOutputStream extends OutputStream { - public void write(byte[] b) throws IOException {} - public void write(int b) throws IOException {} - public void write(byte[] b, int off, int len) throws IOException {} - } - - Encoder( - Configuration conf, int stripeSize, int paritySize) { - this.conf = conf; - this.stripeSize = stripeSize; - this.paritySize = paritySize; - this.rand = new Random(); - this.bufSize = conf.getInt("raid.encoder.bufsize", 1024 * 1024); - this.readBufs = new byte[stripeSize][]; - this.writeBufs = new byte[paritySize][]; - allocateBuffers(); - } - - private void allocateBuffers() { - for (int i = 0; i < stripeSize; i++) { - readBufs[i] = new byte[bufSize]; - } - for (int i = 0; i < paritySize; i++) { - writeBufs[i] = new byte[bufSize]; - } - } - - private void configureBuffers(long blockSize) { - if ((long)bufSize > blockSize) { - bufSize = (int)blockSize; - allocateBuffers(); - } else if (blockSize % bufSize != 0) { - bufSize = (int)(blockSize / 256L); // heuristic. - if (bufSize == 0) { - bufSize = 1024; - } - bufSize = Math.min(bufSize, 1024 * 1024); - allocateBuffers(); - } - } - - /** - * The interface to use to generate a parity file. - * This method can be called multiple times with the same Encoder object, - * thus allowing reuse of the buffers allocated by the Encoder object. - * - * @param fs The filesystem containing the source file. - * @param srcFile The source file. - * @param parityFile The parity file to be generated. - */ - public void encodeFile( - FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, - short parityRepl, Progressable reporter) throws IOException { - FileStatus srcStat = fs.getFileStatus(srcFile); - long srcSize = srcStat.getLen(); - long blockSize = srcStat.getBlockSize(); - - configureBuffers(blockSize); - - // Create a tmp file to which we will write first. - Path tmpDir = getParityTempPath(); - if (!parityFs.mkdirs(tmpDir)) { - throw new IOException("Could not create tmp dir " + tmpDir); - } - Path parityTmp = new Path(tmpDir, - parityFile.getName() + rand.nextLong()); - FSDataOutputStream out = parityFs.create( - parityTmp, - true, - conf.getInt("io.file.buffer.size", 64 * 1024), - parityRepl, - blockSize); - - try { - encodeFileToStream(fs, srcFile, srcSize, blockSize, out, reporter); - out.close(); - out = null; - LOG.info("Wrote temp parity file " + parityTmp); - - // delete destination if exists - if (parityFs.exists(parityFile)){ - parityFs.delete(parityFile, false); - } - parityFs.mkdirs(parityFile.getParent()); - if (!parityFs.rename(parityTmp, parityFile)) { - String msg = "Unable to rename file " + parityTmp + " to " + parityFile; - throw new IOException (msg); - } - LOG.info("Wrote parity file " + parityFile); - } finally { - if (out != null) { - out.close(); - } - parityFs.delete(parityTmp, false); - } - } - - /** - * Recovers a corrupt block in a parity file to a local file. - * - * The encoder generates paritySize parity blocks for a source file stripe. - * Since we want only one of the parity blocks, this function creates - * null outputs for the blocks to be discarded. - * - * @param fs The filesystem in which both srcFile and parityFile reside. - * @param srcFile The source file. - * @param srcSize The size of the source file. - * @param blockSize The block size for the source/parity files. - * @param corruptOffset The location of corruption in the parity file. - * @param localBlockFile The destination for the reovered block. - */ - public void recoverParityBlockToFile( - FileSystem fs, - Path srcFile, long srcSize, long blockSize, - Path parityFile, long corruptOffset, - File localBlockFile) throws IOException { - OutputStream out = new FileOutputStream(localBlockFile); - try { - recoverParityBlockToStream(fs, srcFile, srcSize, blockSize, parityFile, - corruptOffset, out); - } finally { - out.close(); - } - } - - /** - * Recovers a corrupt block in a parity file to a local file. - * - * The encoder generates paritySize parity blocks for a source file stripe. - * Since we want only one of the parity blocks, this function creates - * null outputs for the blocks to be discarded. - * - * @param fs The filesystem in which both srcFile and parityFile reside. - * @param srcFile The source file. - * @param srcSize The size of the source file. - * @param blockSize The block size for the source/parity files. - * @param corruptOffset The location of corruption in the parity file. - * @param out The destination for the reovered block. - */ - public void recoverParityBlockToStream( - FileSystem fs, - Path srcFile, long srcSize, long blockSize, - Path parityFile, long corruptOffset, - OutputStream out) throws IOException { - LOG.info("Recovering parity block" + parityFile + ":" + corruptOffset); - // Get the start offset of the corrupt block. - corruptOffset = (corruptOffset / blockSize) * blockSize; - // Output streams to each block in the parity file stripe. - OutputStream[] outs = new OutputStream[paritySize]; - long indexOfCorruptBlockInParityStripe = - (corruptOffset / blockSize) % paritySize; - LOG.info("Index of corrupt block in parity stripe: " + - indexOfCorruptBlockInParityStripe); - // Create a real output stream for the block we want to recover, - // and create null streams for the rest. - for (int i = 0; i < paritySize; i++) { - if (indexOfCorruptBlockInParityStripe == i) { - outs[i] = out; - } else { - outs[i] = new NullOutputStream(); - } - } - // Get the stripe index and start offset of stripe. - long stripeIdx = corruptOffset / (paritySize * blockSize); - long stripeStart = stripeIdx * blockSize * stripeSize; - - // Get input streams to each block in the source file stripe. - InputStream[] blocks = stripeInputs(fs, srcFile, stripeStart, - srcSize, blockSize); - LOG.info("Starting recovery by using source stripe " + - srcFile + ":" + stripeStart); - // Read the data from the blocks and write to the parity file. - encodeStripe(blocks, stripeStart, blockSize, outs, - new RaidUtils.DummyProgressable()); - } - - /** - * Recovers a corrupt block in a parity file to an output stream. - * - * The encoder generates paritySize parity blocks for a source file stripe. - * Since there is only one output provided, some blocks are written out to - * files before being written out to the output. - * - * @param fs The filesystem in which both srcFile and parityFile reside. - * @param srcFile The source file. - * @param srcSize The size of the source file. - * @param blockSize The block size for the source/parity files. - * @param out The destination for the reovered block. - */ - private void encodeFileToStream(FileSystem fs, Path srcFile, long srcSize, - long blockSize, OutputStream out, Progressable reporter) throws IOException { - OutputStream[] tmpOuts = new OutputStream[paritySize]; - // One parity block can be written directly to out, rest to local files. - tmpOuts[0] = out; - File[] tmpFiles = new File[paritySize - 1]; - for (int i = 0; i < paritySize - 1; i++) { - tmpFiles[i] = File.createTempFile("parity", "_" + i); - LOG.info("Created tmp file " + tmpFiles[i]); - tmpFiles[i].deleteOnExit(); - } - try { - // Loop over stripes in the file. - for (long stripeStart = 0; stripeStart < srcSize; - stripeStart += blockSize * stripeSize) { - reporter.progress(); - LOG.info("Starting encoding of stripe " + srcFile + ":" + stripeStart); - // Create input streams for blocks in the stripe. - InputStream[] blocks = stripeInputs(fs, srcFile, stripeStart, - srcSize, blockSize); - // Create output streams to the temp files. - for (int i = 0; i < paritySize - 1; i++) { - tmpOuts[i + 1] = new FileOutputStream(tmpFiles[i]); - } - // Call the implementation of encoding. - encodeStripe(blocks, stripeStart, blockSize, tmpOuts, reporter); - // Close output streams to the temp files and write the temp files - // to the output provided. - for (int i = 0; i < paritySize - 1; i++) { - tmpOuts[i + 1].close(); - tmpOuts[i + 1] = null; - InputStream in = new FileInputStream(tmpFiles[i]); - RaidUtils.copyBytes(in, out, writeBufs[i], blockSize); - reporter.progress(); - } - } - } finally { - for (int i = 0; i < paritySize - 1; i++) { - if (tmpOuts[i + 1] != null) { - tmpOuts[i + 1].close(); - } - tmpFiles[i].delete(); - LOG.info("Deleted tmp file " + tmpFiles[i]); - } - } - } - - /** - * Return input streams for each block in a source file's stripe. - * @param fs The filesystem where the file resides. - * @param srcFile The source file. - * @param stripeStartOffset The start offset of the stripe. - * @param srcSize The size of the source file. - * @param blockSize The block size for the source file. - */ - protected InputStream[] stripeInputs( - FileSystem fs, - Path srcFile, - long stripeStartOffset, - long srcSize, - long blockSize - ) throws IOException { - InputStream[] blocks = new InputStream[stripeSize]; - for (int i = 0; i < stripeSize; i++) { - long seekOffset = stripeStartOffset + i * blockSize; - if (seekOffset < srcSize) { - FSDataInputStream in = fs.open( - srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); - in.seek(seekOffset); - LOG.info("Opening stream at " + srcFile + ":" + seekOffset); - blocks[i] = in; - } else { - LOG.info("Using zeros at offset " + seekOffset); - // We have no src data at this offset. - blocks[i] = new RaidUtils.ZeroInputStream( - seekOffset + blockSize); - } - } - return blocks; - } - - /** - * The implementation of generating parity data for a stripe. - * - * @param blocks The streams to blocks in the stripe. - * @param stripeStartOffset The start offset of the stripe - * @param blockSize The maximum size of a block. - * @param outs output streams to the parity blocks. - * @param reporter progress indicator. - */ - protected abstract void encodeStripe( - InputStream[] blocks, - long stripeStartOffset, - long blockSize, - OutputStream[] outs, - Progressable reporter) throws IOException; - - /** - * Return the temp path for the parity file - */ - protected abstract Path getParityTempPath(); -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java deleted file mode 100644 index 0747394f440..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -public interface ErasureCode { - /** - * Encodes the given message. - * @param message The data of the message. The data is present in the least - * significant bits of each int. The number of data bits is - * symbolSize(). The number of elements of message is - * stripeSize(). - * @param parity (out) The information is present in the least - * significant bits of each int. The number of parity bits is - * symbolSize(). The number of elements in the code is - * paritySize(). - */ - public void encode(int[] message, int[] parity); - - /** - * Generates missing portions of data. - * @param data The message and parity. The parity should be placed in the - * first part of the array. In each integer, the relevant portion - * is present in the least significant bits of each int. - * The number of elements in data is stripeSize() + paritySize(). - * @param erasedLocations The indexes in data which are not available. - * @param erasedValues (out)The decoded values corresponding to erasedLocations. - */ - public void decode(int[] data, int[] erasedLocations, int[] erasedValues); - - /** - * The number of elements in the message. - */ - public int stripeSize(); - - /** - * The number of elements in the code. - */ - public int paritySize(); - - /** - * Number of bits for each symbol. - */ - public int symbolSize(); -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java deleted file mode 100644 index a10f5d79eaf..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.util.HashMap; -import java.util.Map; - -/** - * Implementation of Galois field arithmetics with 2^p elements. - * The input must be unsigned integers. - */ -public class GaloisField { - - private final int[] logTable; - private final int[] powTable; - private final int[][] mulTable; - private final int[][] divTable; - private final int fieldSize; - private final int primitivePeriod; - private final int primitivePolynomial; - - // Field size 256 is good for byte based system - private static final int DEFAULT_FIELD_SIZE = 256; - // primitive polynomial 1 + X^2 + X^3 + X^4 + X^8 - private static final int DEFAULT_PRIMITIVE_POLYNOMIAL = 285; - - static private final Map instances = - new HashMap(); - - /** - * Get the object performs Galois field arithmetics - * @param fieldSize size of the field - * @param primitivePolynomial a primitive polynomial corresponds to the size - */ - public static GaloisField getInstance(int fieldSize, - int primitivePolynomial) { - int key = ((fieldSize << 16) & 0xFFFF0000) + (primitivePolynomial & 0x0000FFFF); - GaloisField gf; - synchronized (instances) { - gf = instances.get(key); - if (gf == null) { - gf = new GaloisField(fieldSize, primitivePolynomial); - instances.put(key, gf); - } - } - return gf; - } - - /** - * Get the object performs Galois field arithmetics with default setting - */ - public static GaloisField getInstance() { - return getInstance(DEFAULT_FIELD_SIZE, DEFAULT_PRIMITIVE_POLYNOMIAL); - } - - private GaloisField(int fieldSize, int primitivePolynomial) { - assert fieldSize > 0; - assert primitivePolynomial > 0; - - this.fieldSize = fieldSize; - this.primitivePeriod = fieldSize - 1; - this.primitivePolynomial = primitivePolynomial; - logTable = new int[fieldSize]; - powTable = new int[fieldSize]; - mulTable = new int[fieldSize][fieldSize]; - divTable = new int[fieldSize][fieldSize]; - int value = 1; - for (int pow = 0; pow < fieldSize - 1; pow++) { - powTable[pow] = value; - logTable[value] = pow; - value = value * 2; - if (value >= fieldSize) { - value = value ^ primitivePolynomial; - } - } - // building multiplication table - for (int i = 0; i < fieldSize; i++) { - for (int j = 0; j < fieldSize; j++) { - if (i == 0 || j == 0) { - mulTable[i][j] = 0; - continue; - } - int z = logTable[i] + logTable[j]; - z = z >= primitivePeriod ? z - primitivePeriod : z; - z = powTable[z]; - mulTable[i][j] = z; - } - } - // building division table - for (int i = 0; i < fieldSize; i++) { - for (int j = 1; j < fieldSize; j++) { - if (i == 0) { - divTable[i][j] = 0; - continue; - } - int z = logTable[i] - logTable[j]; - z = z < 0 ? z + primitivePeriod : z; - z = powTable[z]; - divTable[i][j] = z; - } - } - } - - /** - * Return number of elements in the field - * @return number of elements in the field - */ - public int getFieldSize() { - return fieldSize; - } - - /** - * Return the primitive polynomial in GF(2) - * @return primitive polynomial as a integer - */ - public int getPrimitivePolynomial() { - return primitivePolynomial; - } - - /** - * Compute the sum of two fields - * @param x input field - * @param y input field - * @return result of addition - */ - public int add(int x, int y) { - assert(x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize()); - return x ^ y; - } - - /** - * Compute the multiplication of two fields - * @param x input field - * @param y input field - * @return result of multiplication - */ - public int multiply(int x, int y) { - assert(x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize()); - return mulTable[x][y]; - } - - /** - * Compute the division of two fields - * @param x input field - * @param y input field - * @return x/y - */ - public int divide(int x, int y) { - assert(x >= 0 && x < getFieldSize() && y > 0 && y < getFieldSize()); - return divTable[x][y]; - } - - /** - * Compute power n of a field - * @param x input field - * @param n power - * @return x^n - */ - public int power(int x, int n) { - assert(x >= 0 && x < getFieldSize()); - if (n == 0) { - return 1; - } - if (x == 0) { - return 0; - } - x = logTable[x] * n; - if (x < primitivePeriod) { - return powTable[x]; - } - x = x % primitivePeriod; - return powTable[x]; - } - - /** - * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such - * that Vz=y. The output z will be placed in y. - * @param x the vector which describe the Vandermonde matrix - * @param y right-hand side of the Vandermonde system equation. - * will be replaced the output in this vector - */ - public void solveVandermondeSystem(int[] x, int[] y) { - solveVandermondeSystem(x, y, x.length); - } - - /** - * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such - * that Vz=y. The output z will be placed in y. - * @param x the vector which describe the Vandermonde matrix - * @param y right-hand side of the Vandermonde system equation. - * will be replaced the output in this vector - * @param len consider x and y only from 0...len-1 - */ - public void solveVandermondeSystem(int[] x, int[] y, int len) { - assert(y.length <= len); - for (int i = 0; i < len - 1; i++) { - for (int j = len - 1; j > i; j--) { - y[j] = y[j] ^ mulTable[x[i]][y[j - 1]]; - } - } - for (int i = len - 1; i >= 0; i--) { - for (int j = i + 1; j < len; j++) { - y[j] = divTable[y[j]][x[j] ^ x[j - i - 1]]; - } - for (int j = i; j < len - 1; j++) { - y[j] = y[j] ^ y[j + 1]; - } - } - } - - /** - * Compute the multiplication of two polynomials. The index in the - * array corresponds to the power of the entry. For example p[0] is the - * constant term of the polynomial p. - * @param p input polynomial - * @param q input polynomial - * @return polynomial represents p*q - */ - public int[] multiply(int[] p, int[] q) { - int len = p.length + q.length - 1; - int[] result = new int[len]; - for (int i = 0; i < len; i++) { - result[i] = 0; - } - for (int i = 0; i < p.length; i++) { - for (int j = 0; j < q.length; j++) { - result[i + j] = add(result[i + j], multiply(p[i], q[j])); - } - } - return result; - } - - /** - * Compute the remainder of a dividend and divisor pair. The index in the - * array corresponds to the power of the entry. For example p[0] is the - * constant term of the polynomial p. - * @param dividend dividend polynomial, the remainder will be placed here when return - * @param divisor divisor polynomial - */ - public void remainder(int[] dividend, int[] divisor) { - for (int i = dividend.length - divisor.length; i >= 0; i--) { - int ratio = - divTable[dividend[i + divisor.length - 1]][divisor[divisor.length - 1]]; - for (int j = 0; j < divisor.length; j++) { - int k = j + i; - dividend[k] = dividend[k] ^ mulTable[ratio][divisor[j]]; - } - } - } - - /** - * Compute the sum of two polynomials. The index in the - * array corresponds to the power of the entry. For example p[0] is the - * constant term of the polynomial p. - * @param p input polynomial - * @param q input polynomial - * @return polynomial represents p+q - */ - public int[] add(int[] p, int[] q) { - int len = Math.max(p.length, q.length); - int[] result = new int[len]; - for (int i = 0; i < len; i++) { - if (i < p.length && i < q.length) { - result[i] = add(p[i], q[i]); - } else if (i < p.length) { - result[i] = p[i]; - } else { - result[i] = q[i]; - } - } - return result; - } - - /** - * Substitute x into polynomial p(x). - * @param p input polynomial - * @param x input field - * @return p(x) - */ - public int substitute(int[] p, int x) { - int result = 0; - int y = 1; - for (int i = 0; i < p.length; i++) { - result = result ^ mulTable[p[i]][y]; - y = mulTable[x][y]; - } - return result; - } - - /** - * Perform Gaussian elimination on the given matrix. This matrix has to be a - * fat matrix (number of rows > number of columns). - */ - public void gaussianElimination(int[][] matrix) { - assert(matrix != null && matrix.length > 0 && matrix[0].length > 0 - && matrix.length < matrix[0].length); - int height = matrix.length; - int width = matrix[0].length; - for (int i = 0; i < height; i++) { - boolean pivotFound = false; - // scan the column for a nonzero pivot and swap it to the diagonal - for (int j = i; j < height; j++) { - if (matrix[i][j] != 0) { - int[] tmp = matrix[i]; - matrix[i] = matrix[j]; - matrix[j] = tmp; - pivotFound = true; - break; - } - } - if (!pivotFound) { - continue; - } - int pivot = matrix[i][i]; - for (int j = i; j < width; j++) { - matrix[i][j] = divide(matrix[i][j], pivot); - } - for (int j = i + 1; j < height; j++) { - int lead = matrix[j][i]; - for (int k = i; k < width; k++) { - matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k])); - } - } - } - for (int i = height - 1; i >=0; i--) { - for (int j = 0; j < i; j++) { - int lead = matrix[j][i]; - for (int k = i; k < width; k++) { - matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k])); - } - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java deleted file mode 100644 index 5cb73c91394..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.InputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.List; -import java.util.LinkedList; - -import org.apache.hadoop.util.LineReader; -import org.apache.hadoop.io.Text; - -/** - * Represents the contents of a HAR Index file. The HAR is assumed to be - * comprising of RAID parity files only and no directories. - */ -public class HarIndex { - public static final String indexFileName = "_index"; - private List entries = new LinkedList(); - - /** - * Represents information in a single line of the HAR index file. - */ - public static class IndexEntry { - String fileName; // Name of the file in the part file. - long startOffset; // Start offset within the part file. - long length; // Length of this file within the part file. - long mtime; // Modification time of the file. - String partFileName; // Name of the part file. - - IndexEntry(String fileName, long startOffset, long length, - long mtime, String partFileName) { - this.fileName = fileName; - this.startOffset = startOffset; - this.length = length; - this.mtime = mtime; - this.partFileName = partFileName; - } - - public String toString() { - return "fileName=" + fileName + - ", startOffset=" + startOffset + - ", length=" + length + - ", mtime=" + mtime + - ", partFileName=" + partFileName; - } - } - - /** - * Constructor that reads the contents of the index file. - * @param in An input stream to the index file. - * @param max The size of the index file. - * @throws IOException - */ - public HarIndex(InputStream in, long max) throws IOException { - LineReader lineReader = new LineReader(in); - Text text = new Text(); - long nread = 0; - while (nread < max) { - int n = lineReader.readLine(text); - nread += n; - String line = text.toString(); - try { - parseLine(line); - } catch (UnsupportedEncodingException e) { - throw new IOException("UnsupportedEncodingException after reading " + - nread + "bytes"); - } - } - } - - /** - * Parses each line and extracts relevant information. - * @param line - * @throws UnsupportedEncodingException - */ - void parseLine(String line) throws UnsupportedEncodingException { - String[] splits = line.split(" "); - - boolean isDir = "dir".equals(splits[1]) ? true: false; - if (!isDir && splits.length >= 6) { - String name = URLDecoder.decode(splits[0], "UTF-8"); - String partName = URLDecoder.decode(splits[2], "UTF-8"); - long startIndex = Long.parseLong(splits[3]); - long length = Long.parseLong(splits[4]); - String[] newsplits = URLDecoder.decode(splits[5],"UTF-8").split(" "); - if (newsplits != null && newsplits.length >= 4) { - long mtime = Long.parseLong(newsplits[0]); - IndexEntry entry = new IndexEntry( - name, startIndex, length, mtime, partName); - entries.add(entry); - } - } - } - - /** - * Finds the index entry corresponding to a HAR partFile at an offset. - * @param partName The name of the part file (part-*). - * @param partFileOffset The offset into the part file. - * @return The entry corresponding to partName:partFileOffset. - */ - public IndexEntry findEntry(String partName, long partFileOffset) { - for (IndexEntry e: entries) { - boolean nameMatch = partName.equals(e.partFileName); - boolean inRange = (partFileOffset >= e.startOffset) && - (partFileOffset < e.startOffset + e.length); - if (nameMatch && inRange) { - return e; - } - } - return null; - } - - /** - * Finds the index entry corresponding to a file in the archive - */ - public IndexEntry findEntryByFileName(String fileName) { - for (IndexEntry e: entries) { - if (fileName.equals(e.fileName)) { - return e; - } - } - return null; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java deleted file mode 100644 index cda295c65a9..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.LinkedList; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; - -/** - * Periodically monitors the status of jobs registered with it. - * - * Jobs that are submitted for the same policy name are kept in the same list, - * and the list itself is kept in a map that has the policy name as the key and - * the list as value. - */ -class JobMonitor implements Runnable { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.JobMonitor"); - - volatile boolean running = true; - - private Map> jobs; - public static final String JOBMONITOR_INTERVAL_KEY = "raid.jobmonitor.interval"; - private long jobMonitorInterval; - private volatile long jobsMonitored = 0; - private volatile long jobsSucceeded = 0; - - public JobMonitor(Configuration conf) { - jobMonitorInterval = conf.getLong(JOBMONITOR_INTERVAL_KEY, 60000); - jobs = new java.util.HashMap>(); - } - - public void run() { - while (running) { - try { - LOG.info("JobMonitor thread continuing to run..."); - doMonitor(); - } catch (Throwable e) { - LOG.error("JobMonitor encountered exception " + - StringUtils.stringifyException(e)); - // All expected exceptions are caught by doMonitor(). It is better - // to exit now, this will prevent RaidNode from submitting more jobs - // since the number of running jobs will never decrease. - return; - } - } - } - - /** - * Periodically checks status of running map-reduce jobs. - */ - public void doMonitor() { - while (running) { - String[] keys = null; - // Make a copy of the names of the current jobs. - synchronized(jobs) { - keys = jobs.keySet().toArray(new String[0]); - } - - // Check all the jobs. We do not want to block access to `jobs` - // because that will prevent new jobs from being added. - // This is safe because JobMonitor.run is the only code that can - // remove a job from `jobs`. Thus all elements in `keys` will have - // valid values. - Map> finishedJobs = - new HashMap>(); - - for (String key: keys) { - // For each policy being monitored, get the list of jobs running. - DistRaid[] jobListCopy = null; - synchronized(jobs) { - List jobList = jobs.get(key); - synchronized(jobList) { - jobListCopy = jobList.toArray(new DistRaid[jobList.size()]); - } - } - // The code that actually contacts the JobTracker is not synchronized, - // it uses copies of the list of jobs. - for (DistRaid job: jobListCopy) { - // Check each running job. - try { - boolean complete = job.checkComplete(); - if (complete) { - addJob(finishedJobs, key, job); - if (job.successful()) { - jobsSucceeded++; - } - } - } catch (IOException ioe) { - // If there was an error, consider the job finished. - addJob(finishedJobs, key, job); - LOG.error("JobMonitor exception", ioe); - } - } - } - - if (finishedJobs.size() > 0) { - for (String key: finishedJobs.keySet()) { - List finishedJobList = finishedJobs.get(key); - // Iterate through finished jobs and remove from jobs. - // removeJob takes care of locking. - for (DistRaid job: finishedJobList) { - removeJob(jobs, key, job); - } - } - } - - try { - Thread.sleep(jobMonitorInterval); - } catch (InterruptedException ie) { - } - } - } - - public int runningJobsCount(String key) { - int count = 0; - synchronized(jobs) { - if (jobs.containsKey(key)) { - List jobList = jobs.get(key); - synchronized(jobList) { - count = jobList.size(); - } - } - } - return count; - } - - public void monitorJob(String key, DistRaid job) { - addJob(jobs, key, job); - jobsMonitored++; - } - - public long jobsMonitored() { - return this.jobsMonitored; - } - - public long jobsSucceeded() { - return this.jobsSucceeded; - } - - // For test code - int runningJobsCount() { - int total = 0; - synchronized(jobs) { - for (String key: jobs.keySet()) { - total += jobs.get(key).size(); - } - } - return total; - } - - private static void addJob(Map> jobsMap, - String jobName, DistRaid job) { - synchronized(jobsMap) { - List list = null; - if (jobsMap.containsKey(jobName)) { - list = jobsMap.get(jobName); - } else { - list = new LinkedList(); - jobsMap.put(jobName, list); - } - synchronized(list) { - list.add(job); - } - } - } - - private static void removeJob(Map> jobsMap, - String jobName, DistRaid job) { - synchronized(jobsMap) { - if (jobsMap.containsKey(jobName)) { - List list = jobsMap.get(jobName); - synchronized(list) { - for (Iterator it = list.iterator(); it.hasNext(); ) { - DistRaid val = it.next(); - if (val == job) { - it.remove(); - } - } - if (list.size() == 0) { - jobsMap.remove(jobName); - } - } - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java deleted file mode 100644 index 0bf4ab00cd1..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.RaidDFSUtil; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; - -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; - -import org.apache.hadoop.net.NetUtils; - -import org.apache.hadoop.hdfs.DistributedFileSystem; - -import org.apache.hadoop.raid.RaidNode; -import org.apache.hadoop.raid.RaidUtils; -import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType; - -/** - * This class fixes source file blocks using the parity file, - * and parity file blocks using the source file. - * It periodically fetches the list of corrupt files from the namenode, - * and figures out the location of the bad block by reading through - * the corrupt file. - */ -public class LocalBlockFixer extends BlockFixer { - public static final Log LOG = LogFactory.getLog(LocalBlockFixer.class); - - private java.util.HashMap history; - - private BlockFixerHelper helper; - - public LocalBlockFixer(Configuration conf) throws IOException { - super(conf); - history = new java.util.HashMap(); - helper = new BlockFixerHelper(getConf()); - } - - public void run() { - while (running) { - try { - LOG.info("LocalBlockFixer continuing to run..."); - doFix(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } catch (Error err) { - LOG.error("Exiting after encountering " + - StringUtils.stringifyException(err)); - throw err; - } - } - } - - void doFix() throws InterruptedException, IOException { - while (running) { - // Sleep before proceeding to fix files. - Thread.sleep(blockFixInterval); - - // Purge history older than the history interval. - purgeHistory(); - - List corruptFiles = getCorruptFiles(); - - filterUnfixableSourceFiles(corruptFiles.iterator()); - - if (corruptFiles.isEmpty()) { - // If there are no corrupt files, retry after some time. - continue; - } - LOG.info("Found " + corruptFiles.size() + " corrupt files."); - - helper.sortCorruptFiles(corruptFiles); - - for (Path srcPath: corruptFiles) { - if (!running) break; - try { - boolean fixed = helper.fixFile(srcPath); - LOG.info("Adding " + srcPath + " to history"); - history.put(srcPath.toString(), new java.util.Date()); - if (fixed) { - incrFilesFixed(); - } - } catch (IOException ie) { - LOG.error("Hit error while processing " + srcPath + - ": " + StringUtils.stringifyException(ie)); - // Do nothing, move on to the next file. - } - } - } - } - - - /** - * We maintain history of fixed files because a fixed file may appear in - * the list of corrupt files if we loop around too quickly. - * This function removes the old items in the history so that we can - * recognize files that have actually become corrupt since being fixed. - */ - void purgeHistory() { - java.util.Date cutOff = new java.util.Date(Time.now() - - historyInterval); - List toRemove = new java.util.ArrayList(); - - for (String key: history.keySet()) { - java.util.Date item = history.get(key); - if (item.before(cutOff)) { - toRemove.add(key); - } - } - for (String key: toRemove) { - LOG.info("Removing " + key + " from history"); - history.remove(key); - } - } - - /** - * @return A list of corrupt files as obtained from the namenode - */ - List getCorruptFiles() throws IOException { - DistributedFileSystem dfs = helper.getDFS(new Path("/")); - - String[] files = RaidDFSUtil.getCorruptFiles(dfs); - List corruptFiles = new LinkedList(); - for (String f: files) { - Path p = new Path(f); - if (!history.containsKey(p.toString())) { - corruptFiles.add(p); - } - } - RaidUtils.filterTrash(getConf(), corruptFiles); - return corruptFiles; - } - -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java deleted file mode 100644 index 5d2444d8af2..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; - -import org.apache.hadoop.raid.protocol.PolicyInfo; - -/** - * Implementation of {@link RaidNode} that performs raiding locally. - */ -public class LocalRaidNode extends RaidNode { - - public static final Log LOG = LogFactory.getLog(LocalRaidNode.class); - - public LocalRaidNode(Configuration conf) throws IOException { - super(conf); - } - - /** - * {@inheritDocs} - */ - @Override - void raidFiles(PolicyInfo info, List paths) throws IOException { - doRaid(conf, info, paths); - } - - /** - * {@inheritDocs} - */ - @Override - int getRunningJobsForPolicy(String policyName) { - return 0; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java deleted file mode 100644 index b30cfcfe2a5..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.hadoop.util.Progressable; - -/** - * Wraps over multiple input streams and provides an input stream that is - * an XOR of the streams. - */ -class ParityInputStream extends InputStream { - private static final int DEFAULT_BUFSIZE = 5*1024*1024; - private InputStream[] streams; - private byte[] xor; - private byte[] buf; - private int bufSize; - private long remaining; - private int available = 0; - private int readPos = 0; - - public ParityInputStream( - InputStream[] streams, long parityBlockSize, byte[] buf, byte[] xor) { - assert buf.length == xor.length; - bufSize = buf.length; - this.streams = streams; - remaining = parityBlockSize; - this.buf = buf; - this.xor = xor; - } - - @Override - public int read() throws IOException { - makeAvailable(); - if (available == 0) { - return -1; - } - int ret = xor[readPos]; - readPos++; - available--; - return ret; - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - makeAvailable(); - if (available == 0) { - return -1; - } - int ret = Math.min(len, available); - for (int i = 0; i < ret; ++i) { - b[off+i] = xor[readPos+i]; - } - readPos += ret; - available -= ret; - return ret; - } - - public void close() throws IOException { - for (InputStream i: streams) { - i.close(); - } - } - - /** - * Send the contents of the stream to the sink. - * @param sink - * @param reporter - * @throws IOException - */ - public void drain(OutputStream sink, Progressable reporter) - throws IOException { - - while (true) { - makeAvailable(); - if (available == 0) { - break; - } - sink.write(xor, readPos, available); - available = 0; - if (reporter != null) { - reporter.progress(); - } - } - } - - /** - * Make some bytes available for reading in the internal buffer. - * @throws IOException - */ - private void makeAvailable() throws IOException { - if (available > 0 || remaining <= 0) { - return; - } - // Read some bytes from the first stream. - int xorlen = (int)Math.min(remaining, bufSize); - readExact(streams[0], xor, xorlen); - - // Read bytes from all the other streams and xor them. - for (int i = 1; i < streams.length; i++) { - readExact(streams[i], buf, xorlen); - - for (int j = 0; j < xorlen; j++) { - xor[j] ^= buf[j]; - } - } - - remaining -= xorlen; - available = xorlen; - readPos = 0; - readPos = 0; - } - - private static void readExact(InputStream in, byte[] bufs, int toRead) - throws IOException { - int tread = 0; - while (tread < toRead) { - int read = in.read(bufs, tread, toRead - tread); - if (read == -1) { - // If the stream ends, fill in zeros. - Arrays.fill(bufs, tread, toRead, (byte)0); - tread = toRead; - } else { - tread += read; - } - } - assert tread == toRead; - } - -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java deleted file mode 100644 index 891863ae927..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -/** - * Thrown when the config file for {@link RaidNode} is malformed. - */ -public class RaidConfigurationException extends Exception { - private static final long serialVersionUID = 4046516718965587999L; - - public RaidConfigurationException(String message) { - super(message); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java deleted file mode 100644 index 2af4ed9980b..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.raid.protocol.PolicyInfo; - -public class RaidFilter { - static class Statistics { - long numRaided = 0; - long numTooNew = 0; - long sizeTooNew = 0; - long numTooSmall = 0; - long sizeTooSmall = 0; - - public void aggregate(Statistics other) { - this.numRaided += other.numRaided; - this.numTooNew += other.numTooNew; - this.sizeTooNew += other.sizeTooNew; - this.numTooSmall += other.numTooSmall; - this.sizeTooSmall += other.sizeTooSmall; - } - - public String toString() { - return "numRaided = " + numRaided + - ", numTooNew = " + numTooNew + - ", sizeTooNew = " + sizeTooNew + - ", numTooSmall = " + numTooSmall + - ", sizeTooSmall = " + sizeTooSmall; - } - } - - static class TimeBasedFilter extends Configured - implements DirectoryTraversal.FileFilter { - int targetRepl; - Path raidDestPrefix; - long modTimePeriod; - long startTime; - Statistics stats = new Statistics(); - String currentSrcPath = null; - long[] modTimePeriods = new long[0]; - String[] otherSrcPaths = new String[0]; - - TimeBasedFilter(Configuration conf, Path destPrefix, int targetRepl, - long startTime, long modTimePeriod) { - super(conf); - this.raidDestPrefix = destPrefix; - this.targetRepl = targetRepl; - this.startTime = startTime; - this.modTimePeriod = modTimePeriod; - } - - TimeBasedFilter(Configuration conf, - Path destPrefix, PolicyInfo info, - List allPolicies, long startTime, Statistics stats) { - super(conf); - this.raidDestPrefix = destPrefix; - this.targetRepl = Integer.parseInt(info.getProperty("targetReplication")); - this.modTimePeriod = Long.parseLong(info.getProperty("modTimePeriod")); - this.startTime = startTime; - this.stats = stats; - this.currentSrcPath = info.getSrcPath().toUri().getPath(); - initializeOtherPaths(allPolicies); - } - - private void initializeOtherPaths(List allPolicies) { - ArrayList tmp = new ArrayList(allPolicies); - // Remove all policies where srcPath <= currentSrcPath or - // matchingPrefixLength is < length(currentSrcPath) - // The policies remaining are the only ones that could better - // select a file chosen by the current policy. - for (Iterator it = tmp.iterator(); it.hasNext(); ) { - String src = it.next().getSrcPath().toUri().getPath(); - if (src.compareTo(currentSrcPath) <= 0) { - it.remove(); - continue; - } - int matchLen = matchingPrefixLength(src, currentSrcPath); - if (matchLen < currentSrcPath.length()) { - it.remove(); - } - } - // Sort in reverse lexicographic order. - Collections.sort(tmp, new Comparator() { - public int compare(Object o1, Object o2) { - return 0 - - ((PolicyInfo)o1).getSrcPath().toUri().getPath().compareTo( - ((PolicyInfo)o1).getSrcPath().toUri().getPath()); - } - }); - otherSrcPaths = new String[tmp.size()]; - modTimePeriods = new long[otherSrcPaths.length]; - for (int i = 0; i < otherSrcPaths.length; i++) { - otherSrcPaths[i] = tmp.get(i).getSrcPath().toUri().getPath(); - modTimePeriods[i] = Long.parseLong( - tmp.get(i).getProperty("modTimePeriod")); - } - } - - public boolean check(FileStatus f) throws IOException { - if (!canChooseForCurrentPolicy(f)) { - return false; - } - - // If the source file has fewer than or equal to 2 blocks, then skip it. - long blockSize = f.getBlockSize(); - if (2 * blockSize >= f.getLen()) { - stats.numTooSmall++; - stats.sizeTooSmall += f.getLen(); - return false; - } - - boolean select = false; - try { - Object ppair = RaidNode.getParityFile( - raidDestPrefix, f.getPath(), getConf()); - // Is there is a valid parity file? - if (ppair != null) { - // Is the source at the target replication? - if (f.getReplication() != targetRepl) { - // Select the file so that its replication can be set. - select = true; - } else { - stats.numRaided++; - // Nothing to do, don't select the file. - select = false; - } - } else { - // No parity file. - if (f.getModificationTime() + modTimePeriod < startTime) { - // If the file is not too new, choose it for raiding. - select = true; - } else { - select = false; - stats.numTooNew++; - stats.sizeTooNew += f.getLen(); - } - } - } catch (java.io.FileNotFoundException e) { - select = true; // destination file does not exist - } catch (java.io.IOException e) { - // If there is a problem with the har path, this will let us continue. - DirectoryTraversal.LOG.error( - "Error while selecting " + StringUtils.stringifyException(e)); - } - return select; - } - - /** - * Checks if a file can be chosen for the current policy. - */ - boolean canChooseForCurrentPolicy(FileStatus stat) { - boolean choose = true; - if (otherSrcPaths.length > 0) { - String fileStr = stat.getPath().toUri().getPath(); - - // For a given string, find the best matching srcPath. - int matchWithCurrent = matchingPrefixLength(fileStr, currentSrcPath); - for (int i = 0; i < otherSrcPaths.length; i++) { - // If the file is too new, move to the next. - if (stat.getModificationTime() > startTime - modTimePeriods[i]) { - continue; - } - int matchLen = matchingPrefixLength(fileStr, otherSrcPaths[i]); - if (matchLen > 0 && - fileStr.charAt(matchLen - 1) == Path.SEPARATOR_CHAR) { - matchLen--; - } - if (matchLen > matchWithCurrent) { - choose = false; - break; - } - } - } - return choose; - } - - int matchingPrefixLength(final String s1, final String s2) { - int len = 0; - for (int j = 0; j < s1.length() && j < s2.length(); j++) { - if (s1.charAt(j) == s2.charAt(j)) { - len++; - } else { - break; - } - } - return len; - } - } - - static class PreferenceFilter extends Configured - implements DirectoryTraversal.FileFilter { - Path firstChoicePrefix; - DirectoryTraversal.FileFilter secondChoiceFilter; - - PreferenceFilter(Configuration conf, - Path firstChoicePrefix, Path secondChoicePrefix, - int targetRepl, long startTime, long modTimePeriod) { - super(conf); - this.firstChoicePrefix = firstChoicePrefix; - this.secondChoiceFilter = new TimeBasedFilter(conf, - secondChoicePrefix, targetRepl, startTime, modTimePeriod); - } - - PreferenceFilter(Configuration conf, - Path firstChoicePrefix, Path secondChoicePrefix, - PolicyInfo info, List allPolicies, long startTime, - Statistics stats) { - super(conf); - this.firstChoicePrefix = firstChoicePrefix; - this.secondChoiceFilter = new TimeBasedFilter( - conf, secondChoicePrefix, info, allPolicies, startTime, stats); - } - - public boolean check(FileStatus f) throws IOException { - Object firstChoicePPair = - RaidNode.getParityFile(firstChoicePrefix, f.getPath(), getConf()); - if (firstChoicePPair == null) { - // The decision is upto the the second choice filter. - return secondChoiceFilter.check(f); - } else { - // There is already a parity file under the first choice path. - // We dont want to choose this file. - return false; - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java deleted file mode 100644 index 27ecc0b4f74..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java +++ /dev/null @@ -1,1630 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.ArrayList; -import java.util.List; -import java.util.LinkedList; -import java.util.Arrays; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.HashSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.lang.Thread; -import java.net.InetSocketAddress; -import java.net.URI; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; - -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; - -import org.apache.hadoop.ipc.*; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.tools.HadoopArchives; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Daemon; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.fs.HarFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.util.Progressable; - -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.raid.protocol.PolicyList; -import org.apache.hadoop.raid.protocol.RaidProtocol; -import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType; - -/** - * A base class that implements {@link RaidProtocol}. - * - * use raid.classname to specify which implementation to use - */ -public abstract class RaidNode implements RaidProtocol { - - static{ - Configuration.addDefaultResource("hdfs-default.xml"); - Configuration.addDefaultResource("hdfs-site.xml"); - Configuration.addDefaultResource("mapred-default.xml"); - Configuration.addDefaultResource("mapred-site.xml"); - } - public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.RaidNode"); - public static final long SLEEP_TIME = 10000L; // 10 seconds - public static final String TRIGGER_MONITOR_SLEEP_TIME_KEY = - "hdfs.raid.trigger.monitor.sleep.time"; - public static final int DEFAULT_PORT = 60000; - // Default stripe length = 5, parity length for RS code = 3 - public static final int DEFAULT_STRIPE_LENGTH = 5; - public static final int RS_PARITY_LENGTH_DEFAULT = 3; - - public static final String RS_PARITY_LENGTH_KEY = "hdfs.raidrs.paritylength"; - public static final String STRIPE_LENGTH_KEY = "hdfs.raid.stripeLength"; - - public static final String DEFAULT_RAID_LOCATION = "/raid"; - public static final String RAID_LOCATION_KEY = "hdfs.raid.locations"; - public static final String DEFAULT_RAID_TMP_LOCATION = "/tmp/raid"; - public static final String RAID_TMP_LOCATION_KEY = "fs.raid.tmpdir"; - public static final String DEFAULT_RAID_HAR_TMP_LOCATION = "/tmp/raid_har"; - public static final String RAID_HAR_TMP_LOCATION_KEY = "fs.raid.hartmpdir"; - - public static final String DEFAULT_RAIDRS_LOCATION = "/raidrs"; - public static final String RAIDRS_LOCATION_KEY = "hdfs.raidrs.locations"; - public static final String DEFAULT_RAIDRS_TMP_LOCATION = "/tmp/raidrs"; - public static final String RAIDRS_TMP_LOCATION_KEY = "fs.raidrs.tmpdir"; - public static final String DEFAULT_RAIDRS_HAR_TMP_LOCATION = "/tmp/raidrs_har"; - public static final String RAIDRS_HAR_TMP_LOCATION_KEY = "fs.raidrs.hartmpdir"; - - public static final String HAR_SUFFIX = "_raid.har"; - public static final Pattern PARITY_HAR_PARTFILE_PATTERN = - Pattern.compile(".*" + HAR_SUFFIX + "/part-.*"); - - public static final String RAIDNODE_CLASSNAME_KEY = "raid.classname"; - - /** RPC server */ - private Server server; - /** RPC server address */ - private InetSocketAddress serverAddress = null; - /** only used for testing purposes */ - protected boolean stopRequested = false; - - /** Configuration Manager */ - private ConfigManager configMgr; - - /** hadoop configuration */ - protected Configuration conf; - - protected boolean initialized; // Are we initialized? - protected volatile boolean running; // Are we running? - - /** Deamon thread to trigger policies */ - Daemon triggerThread = null; - public static long triggerMonitorSleepTime = SLEEP_TIME; - - /** Deamon thread to delete obsolete parity files */ - PurgeMonitor purgeMonitor = null; - Daemon purgeThread = null; - - /** Deamon thread to har raid directories */ - Daemon harThread = null; - - /** Daemon thread to fix corrupt files */ - BlockFixer blockFixer = null; - Daemon blockFixerThread = null; - - // statistics about RAW hdfs blocks. This counts all replicas of a block. - public static class Statistics { - long numProcessedBlocks; // total blocks encountered in namespace - long processedSize; // disk space occupied by all blocks - long remainingSize; // total disk space post RAID - - long numMetaBlocks; // total blocks in metafile - long metaSize; // total disk space for meta files - - public void clear() { - numProcessedBlocks = 0; - processedSize = 0; - remainingSize = 0; - numMetaBlocks = 0; - metaSize = 0; - } - public String toString() { - long save = processedSize - (remainingSize + metaSize); - long savep = 0; - if (processedSize > 0) { - savep = (save * 100)/processedSize; - } - String msg = " numProcessedBlocks = " + numProcessedBlocks + - " processedSize = " + processedSize + - " postRaidSize = " + remainingSize + - " numMetaBlocks = " + numMetaBlocks + - " metaSize = " + metaSize + - " %save in raw disk space = " + savep; - return msg; - } - } - - // Startup options - static public enum StartupOption{ - TEST ("-test"), - REGULAR ("-regular"); - - private String name = null; - private StartupOption(String arg) {this.name = arg;} - public String getName() {return name;} - } - - /** - * Start RaidNode. - *

- * The raid-node can be started with one of the following startup options: - *

    - *
  • {@link StartupOption#REGULAR REGULAR} - normal raid node startup
    • - *
- * The option is passed via configuration field: - * fs.raidnode.startup - * - * The conf will be modified to reflect the actual ports on which - * the RaidNode is up and running if the user passes the port as - * zero in the conf. - * - * @param conf confirguration - * @throws IOException - */ - - RaidNode(Configuration conf) throws IOException { - try { - initialize(conf); - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - this.stop(); - throw e; - } catch (Exception e) { - this.stop(); - throw new IOException(e); - } - } - - public long getProtocolVersion(String protocol, - long clientVersion) throws IOException { - if (protocol.equals(RaidProtocol.class.getName())) { - return RaidProtocol.versionID; - } else { - throw new IOException("Unknown protocol to name node: " + protocol); - } - } - - @Override - public ProtocolSignature getProtocolSignature(String protocol, - long clientVersion, int clientMethodsHash) throws IOException { - return ProtocolSignature.getProtocolSignature( - this, protocol, clientVersion, clientMethodsHash); - } - - /** - * Wait for service to finish. - * (Normally, it runs forever.) - */ - public void join() { - try { - if (server != null) server.join(); - if (triggerThread != null) triggerThread.join(); - if (blockFixerThread != null) blockFixerThread.join(); - if (purgeThread != null) purgeThread.join(); - } catch (InterruptedException ie) { - // do nothing - } - } - - /** - * Stop all RaidNode threads and wait for all to finish. - */ - public void stop() { - if (stopRequested) { - return; - } - stopRequested = true; - running = false; - if (server != null) server.stop(); - if (triggerThread != null) triggerThread.interrupt(); - if (blockFixer != null) blockFixer.running = false; - if (blockFixerThread != null) blockFixerThread.interrupt(); - if (purgeThread != null) purgeThread.interrupt(); - } - - private static InetSocketAddress getAddress(String address) { - return NetUtils.createSocketAddr(address); - } - - public static InetSocketAddress getAddress(Configuration conf) { - String nodeport = conf.get("raid.server.address"); - if (nodeport == null) { - nodeport = "localhost:" + DEFAULT_PORT; - } - return getAddress(nodeport); - } - - public InetSocketAddress getListenerAddress() { - return server.getListenerAddress(); - } - - private void initialize(Configuration conf) - throws IOException, SAXException, InterruptedException, RaidConfigurationException, - ClassNotFoundException, ParserConfigurationException { - this.conf = conf; - InetSocketAddress socAddr = RaidNode.getAddress(conf); - int handlerCount = conf.getInt("fs.raidnode.handler.count", 10); - - // read in the configuration - configMgr = new ConfigManager(conf); - - // create rpc server - this.server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(), - handlerCount, false, conf); - - // The rpc-server port can be ephemeral... ensure we have the correct info - this.serverAddress = this.server.getListenerAddress(); - LOG.info("RaidNode up at: " + this.serverAddress); - - initialized = true; - running = true; - this.server.start(); // start RPC server - - this.blockFixer = BlockFixer.createBlockFixer(conf); - this.blockFixerThread = new Daemon(this.blockFixer); - this.blockFixerThread.start(); - // start the deamon thread to fire polcies appropriately - RaidNode.triggerMonitorSleepTime = conf.getLong( - TRIGGER_MONITOR_SLEEP_TIME_KEY, - SLEEP_TIME); - - // start the deamon thread to fire polcies appropriately - this.triggerThread = new Daemon(new TriggerMonitor()); - this.triggerThread.start(); - - // start the thread that deletes obsolete parity files - this.purgeMonitor = new PurgeMonitor(); - this.purgeThread = new Daemon(purgeMonitor); - this.purgeThread.start(); - - // start the thread that creates HAR files - this.harThread = new Daemon(new HarMonitor()); - this.harThread.start(); - } - - - /** - * Implement RaidProtocol methods - */ - - /** {@inheritDoc} */ - public PolicyList[] getAllPolicies() throws IOException { - Collection list = configMgr.getAllPolicies(); - return list.toArray(new PolicyList[list.size()]); - } - - /** {@inheritDoc} */ - public String recoverFile(String inStr, long corruptOffset) throws IOException { - - LOG.info("Recover File for " + inStr + " for corrupt offset " + corruptOffset); - Path inputPath = new Path(inStr); - Path srcPath = inputPath.makeQualified(inputPath.getFileSystem(conf)); - // find stripe length from config - int stripeLength = getStripeLength(conf); - - // first try decode using XOR code - Path destPref = xorDestinationPath(conf); - Decoder decoder = new XORDecoder(conf, RaidNode.getStripeLength(conf)); - Path unraided = unRaid(conf, srcPath, destPref, decoder, - stripeLength, corruptOffset); - if (unraided != null) { - return unraided.toString(); - } - - // try decode using ReedSolomon code - destPref = rsDestinationPath(conf); - decoder = new ReedSolomonDecoder(conf, RaidNode.getStripeLength(conf), - RaidNode.rsParityLength(conf)); - unraided = unRaid(conf, srcPath, destPref, decoder, - stripeLength, corruptOffset); - if (unraided != null) { - return unraided.toString(); - } - return null; - } - - /** - * returns the number of raid jobs running for a particular policy - */ - abstract int getRunningJobsForPolicy(String policyName); - - /** - * Periodically checks to see which policies should be fired. - */ - class TriggerMonitor implements Runnable { - class ScanState { - long fullScanStartTime; - DirectoryTraversal pendingTraversal; - RaidFilter.Statistics stats; - ScanState() { - fullScanStartTime = 0; - pendingTraversal = null; - stats = new RaidFilter.Statistics(); - } - } - - private Map scanStateMap = - new HashMap(); - - /** - */ - public void run() { - while (running) { - try { - doProcess(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - LOG.info("Trigger thread continuing to run..."); - } - } - } - - /** - * Should we select more files for a policy. - */ - private boolean shouldSelectFiles(PolicyInfo info) { - String policyName = info.getName(); - ScanState scanState = scanStateMap.get(policyName); - int runningJobsCount = getRunningJobsForPolicy(policyName); - // Is there a scan in progress for this policy? - if (scanState.pendingTraversal != null) { - int maxJobsPerPolicy = configMgr.getMaxJobsPerPolicy(); - - // If there is a scan in progress for this policy, we can have - // upto maxJobsPerPolicy running jobs. - return (runningJobsCount < maxJobsPerPolicy); - } else { - // If there isn't a scan in progress for this policy, we don't - // want to start a fresh scan if there is even one running job. - if (runningJobsCount >= 1) { - return false; - } - // Check the time of the last full traversal before starting a fresh - // traversal. - long lastScan = scanState.fullScanStartTime; - return (now() > lastScan + configMgr.getPeriodicity()); - } - } - - /** - * Returns a list of pathnames that needs raiding. - * The list of paths could be obtained by resuming a previously suspended - * traversal. - * The number of paths returned is limited by raid.distraid.max.jobs. - */ - private List selectFiles( - PolicyInfo info, ArrayList allPolicies) throws IOException { - Path destPrefix = getDestinationPath(info.getErasureCode(), conf); - String policyName = info.getName(); - Path srcPath = info.getSrcPath(); - long modTimePeriod = Long.parseLong(info.getProperty("modTimePeriod")); - - // Max number of files returned. - int selectLimit = configMgr.getMaxFilesPerJob(); - int targetRepl = Integer.parseInt(info.getProperty("targetReplication")); - - long selectStartTime = Time.now(); - - ScanState scanState = scanStateMap.get(policyName); - // If we have a pending traversal, resume it. - if (scanState.pendingTraversal != null) { - DirectoryTraversal dt = scanState.pendingTraversal; - LOG.info("Resuming traversal for policy " + policyName); - DirectoryTraversal.FileFilter filter = - filterForPolicy(selectStartTime, info, allPolicies, scanState.stats); - List returnSet = dt.getFilteredFiles(filter, selectLimit); - if (dt.doneTraversal()) { - scanState.pendingTraversal = null; - } - return returnSet; - } - - // Expand destination prefix path. - String destpstr = destPrefix.toString(); - if (!destpstr.endsWith(Path.SEPARATOR)) { - destpstr += Path.SEPARATOR; - } - - List returnSet = new LinkedList(); - - FileSystem fs = srcPath.getFileSystem(conf); - FileStatus[] gpaths = fs.globStatus(srcPath); - if (gpaths != null) { - List selectedPaths = new LinkedList(); - for (FileStatus onepath: gpaths) { - String pathstr = onepath.getPath().makeQualified(fs).toString(); - if (!pathstr.endsWith(Path.SEPARATOR)) { - pathstr += Path.SEPARATOR; - } - if (pathstr.startsWith(destpstr) || destpstr.startsWith(pathstr)) { - LOG.info("Skipping source " + pathstr + - " because it conflicts with raid directory " + destpstr); - } else { - selectedPaths.add(onepath); - } - } - - // Set the time for a new traversal. - scanState.fullScanStartTime = now(); - DirectoryTraversal dt = new DirectoryTraversal(fs, selectedPaths, - conf.getInt("raid.directorytraversal.threads", 4)); - DirectoryTraversal.FileFilter filter = - filterForPolicy(selectStartTime, info, allPolicies, scanState.stats); - returnSet = dt.getFilteredFiles(filter, selectLimit); - if (!dt.doneTraversal()) { - scanState.pendingTraversal = dt; - } - } - return returnSet; - } - - /** - * Keep processing policies. - * If the config file has changed, then reload config file and start afresh. - */ - private void doProcess() throws IOException, InterruptedException { - ArrayList allPolicies = new ArrayList(); - for (PolicyList category : configMgr.getAllPolicies()) { - for (PolicyInfo info: category.getAll()) { - allPolicies.add(info); - } - } - while (running) { - Thread.sleep(RaidNode.triggerMonitorSleepTime); - - boolean reloaded = configMgr.reloadConfigsIfNecessary(); - if (reloaded) { - allPolicies.clear(); - for (PolicyList category : configMgr.getAllPolicies()) { - for (PolicyInfo info: category.getAll()) { - allPolicies.add(info); - } - } - } - for (PolicyInfo info: allPolicies) { - if (!scanStateMap.containsKey(info.getName())) { - scanStateMap.put(info.getName(), new ScanState()); - } - - if (!shouldSelectFiles(info)) { - continue; - } - - LOG.info("Triggering Policy Filter " + info.getName() + - " " + info.getSrcPath()); - List filteredPaths = null; - try { - filteredPaths = selectFiles(info, allPolicies); - } catch (Exception e) { - LOG.error("Exception while invoking filter on policy " + info.getName() + - " srcPath " + info.getSrcPath() + - " exception " + StringUtils.stringifyException(e)); - continue; - } - - if (filteredPaths == null || filteredPaths.size() == 0) { - LOG.info("No filtered paths for policy " + info.getName()); - continue; - } - - // Apply the action on accepted paths - LOG.info("Triggering Policy Action " + info.getName() + - " " + info.getSrcPath() + " raid " + filteredPaths.size() + " files"); - try { - raidFiles(info, filteredPaths); - } catch (Exception e) { - LOG.error("Exception while invoking action on policy " + info.getName() + - " srcPath " + info.getSrcPath() + - " exception " + StringUtils.stringifyException(e)); - continue; - } - } - } - } - - DirectoryTraversal.FileFilter filterForPolicy( - long startTime, PolicyInfo info, List allPolicies, - RaidFilter.Statistics stats) - throws IOException { - switch (info.getErasureCode()) { - case XOR: - // Return a preference-based filter that prefers RS parity files - // over XOR parity files. - return new RaidFilter.PreferenceFilter( - conf, rsDestinationPath(conf), xorDestinationPath(conf), - info, allPolicies, startTime, stats); - case RS: - return new RaidFilter.TimeBasedFilter(conf, rsDestinationPath(conf), - info, allPolicies, startTime, stats); - default: - return null; - } - } - } - - /** - * raid a list of files, this will be overridden by subclasses of RaidNode - */ - abstract void raidFiles(PolicyInfo info, List paths) - throws IOException; - - static private Path getOriginalParityFile(Path destPathPrefix, Path srcPath) { - return new Path(destPathPrefix, makeRelative(srcPath)); - } - - static class ParityFilePair { - private Path path; - private FileSystem fs; - - public ParityFilePair( Path path, FileSystem fs) { - this.path = path; - this.fs = fs; - } - - public Path getPath() { - return this.path; - } - - public FileSystem getFileSystem() { - return this.fs; - } - - } - - - /** - * Returns the Path to the parity file of a given file - * - * @param destPathPrefix Destination prefix defined by some policy - * @param srcPath Path to the original source file - * @param create Boolean value telling whether a new parity file should be created - * @return Path object representing the parity file of the source - * @throws IOException - */ - static ParityFilePair getParityFile(Path destPathPrefix, Path srcPath, Configuration conf) throws IOException { - Path srcParent = srcPath.getParent(); - - FileSystem fsDest = destPathPrefix.getFileSystem(conf); - FileSystem fsSrc = srcPath.getFileSystem(conf); - - FileStatus srcStatus = null; - try { - srcStatus = fsSrc.getFileStatus(srcPath); - } catch (java.io.FileNotFoundException e) { - return null; - } - - Path outDir = destPathPrefix; - if (srcParent != null) { - if (srcParent.getParent() == null) { - outDir = destPathPrefix; - } else { - outDir = new Path(destPathPrefix, makeRelative(srcParent)); - } - } - - - //CASE 1: CHECK HAR - Must be checked first because har is created after - // parity file and returning the parity file could result in error while - // reading it. - Path outPath = getOriginalParityFile(destPathPrefix, srcPath); - String harDirName = srcParent.getName() + HAR_SUFFIX; - Path HarPath = new Path(outDir,harDirName); - if (fsDest.exists(HarPath)) { - URI HarPathUri = HarPath.toUri(); - Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath()); - FileSystem fsHar = new HarFileSystem(fsDest); - fsHar.initialize(inHarPath.toUri(), conf); - if (fsHar.exists(inHarPath)) { - FileStatus inHar = fsHar.getFileStatus(inHarPath); - if (inHar.getModificationTime() == srcStatus.getModificationTime()) { - return new ParityFilePair(inHarPath,fsHar); - } - } - } - - //CASE 2: CHECK PARITY - try { - FileStatus outHar = fsDest.getFileStatus(outPath); - if (outHar.getModificationTime() == srcStatus.getModificationTime()) { - return new ParityFilePair(outPath,fsDest); - } - } catch (java.io.FileNotFoundException e) { - } - - return null; // NULL if no parity file - } - - static ParityFilePair xorParityForSource(Path srcPath, Configuration conf) - throws IOException { - try { - Path destPath = xorDestinationPath(conf); - return getParityFile(destPath, srcPath, conf); - } catch (FileNotFoundException e) { - } - return null; - } - - static ParityFilePair rsParityForSource(Path srcPath, Configuration conf) - throws IOException { - try { - Path destPath = rsDestinationPath(conf); - return getParityFile(destPath, srcPath, conf); - } catch (FileNotFoundException e) { - } - return null; - } - - private ParityFilePair getParityFile(Path destPathPrefix, Path srcPath) - throws IOException { - return getParityFile(destPathPrefix, srcPath, conf); - } - - /** - * RAID a list of files. - */ - void doRaid(Configuration conf, PolicyInfo info, List paths) - throws IOException { - int targetRepl = Integer.parseInt(info.getProperty("targetReplication")); - int metaRepl = Integer.parseInt(info.getProperty("metaReplication")); - int stripeLength = getStripeLength(conf); - Path destPref = getDestinationPath(info.getErasureCode(), conf); - String simulate = info.getProperty("simulate"); - boolean doSimulate = simulate == null ? false : Boolean - .parseBoolean(simulate); - - Statistics statistics = new Statistics(); - int count = 0; - - for (FileStatus s : paths) { - doRaid(conf, s, destPref, info.getErasureCode(), statistics, - new RaidUtils.DummyProgressable(), doSimulate, targetRepl, metaRepl, - stripeLength); - if (count % 1000 == 0) { - LOG.info("RAID statistics " + statistics.toString()); - } - count++; - } - LOG.info("RAID statistics " + statistics.toString()); - } - - - /** - * RAID an individual file - */ - - static public void doRaid(Configuration conf, PolicyInfo info, - FileStatus src, Statistics statistics, Progressable reporter) - throws IOException { - int targetRepl = Integer.parseInt(info.getProperty("targetReplication")); - int metaRepl = Integer.parseInt(info.getProperty("metaReplication")); - int stripeLength = getStripeLength(conf); - - Path destPref = getDestinationPath(info.getErasureCode(), conf); - String simulate = info.getProperty("simulate"); - boolean doSimulate = simulate == null ? false : Boolean - .parseBoolean(simulate); - - doRaid(conf, src, destPref, info.getErasureCode(), statistics, - reporter, doSimulate, targetRepl, metaRepl, stripeLength); - } - - /** - * RAID an individual file - */ - static public void doRaid(Configuration conf, FileStatus stat, Path destPath, - PolicyInfo.ErasureCodeType code, Statistics statistics, - Progressable reporter, boolean doSimulate, - int targetRepl, int metaRepl, int stripeLength) - throws IOException { - Path p = stat.getPath(); - FileSystem srcFs = p.getFileSystem(conf); - - // extract block locations from File system - BlockLocation[] locations = srcFs.getFileBlockLocations(stat, 0, stat.getLen()); - - // if the file has fewer than 2 blocks, then nothing to do - if (locations.length <= 2) { - return; - } - - // add up the raw disk space occupied by this file - long diskSpace = 0; - for (BlockLocation l: locations) { - diskSpace += (l.getLength() * stat.getReplication()); - } - statistics.numProcessedBlocks += locations.length; - statistics.processedSize += diskSpace; - - // generate parity file - generateParityFile(conf, stat, reporter, srcFs, destPath, code, - locations, metaRepl, stripeLength); - - // reduce the replication factor of the source file - if (!doSimulate) { - if (srcFs.setReplication(p, (short)targetRepl) == false) { - LOG.error("Error in reducing relication factor of file " + p + " to " + targetRepl); - statistics.remainingSize += diskSpace; // no change in disk space usage - return; - } - } - - diskSpace = 0; - for (BlockLocation l: locations) { - diskSpace += (l.getLength() * targetRepl); - } - statistics.remainingSize += diskSpace; - - // the metafile will have this many number of blocks - int numMeta = locations.length / stripeLength; - if (locations.length % stripeLength != 0) { - numMeta++; - } - - // we create numMeta for every file. This metablock has metaRepl # replicas. - // the last block of the metafile might not be completely filled up, but we - // ignore that for now. - statistics.numMetaBlocks += (numMeta * metaRepl); - statistics.metaSize += (numMeta * metaRepl * stat.getBlockSize()); - } - - /** - * Create the parity file. - */ - static private void generateParityFile(Configuration conf, FileStatus stat, - Progressable reporter, - FileSystem inFs, - Path destPathPrefix, - ErasureCodeType code, - BlockLocation[] locations, - int metaRepl, int stripeLength) throws IOException { - - Path inpath = stat.getPath(); - Path outpath = getOriginalParityFile(destPathPrefix, inpath); - FileSystem outFs = outpath.getFileSystem(conf); - - // If the parity file is already upto-date, then nothing to do - try { - FileStatus stmp = outFs.getFileStatus(outpath); - if (stmp.getModificationTime() == stat.getModificationTime()) { - LOG.info("Parity file for " + inpath + "(" + locations.length + - ") is " + outpath + " already upto-date. Nothing more to do."); - return; - } - } catch (IOException e) { - // ignore errors because the raid file might not exist yet. - } - - Encoder encoder = encoderForCode(conf, code); - encoder.encodeFile(inFs, inpath, outFs, outpath, (short)metaRepl, reporter); - - // set the modification time of the RAID file. This is done so that the modTime of the - // RAID file reflects that contents of the source file that it has RAIDed. This should - // also work for files that are being appended to. This is necessary because the time on - // on the destination namenode may not be synchronised with the timestamp of the - // source namenode. - outFs.setTimes(outpath, stat.getModificationTime(), -1); - inFs.setTimes(inpath, stat.getModificationTime(), stat.getAccessTime()); - - FileStatus outstat = outFs.getFileStatus(outpath); - FileStatus inStat = inFs.getFileStatus(inpath); - LOG.info("Source file " + inpath + " of size " + inStat.getLen() + - " Parity file " + outpath + " of size " + outstat.getLen() + - " src mtime " + stat.getModificationTime() + - " parity mtime " + outstat.getModificationTime()); - } - - /** - * Extract a good block from the parity block. This assumes that the - * corruption is in the main file and the parity file is always good. - */ - public static Path unRaid(Configuration conf, Path srcPath, - Path destPathPrefix, Decoder decoder, int stripeLength, - long corruptOffset) throws IOException { - - // Test if parity file exists - ParityFilePair ppair = getParityFile(destPathPrefix, srcPath, conf); - if (ppair == null) { - return null; - } - - final Path recoveryDestination = new Path( - RaidNode.unraidTmpDirectory(conf)); - FileSystem destFs = recoveryDestination.getFileSystem(conf); - final Path recoveredPrefix = - destFs.makeQualified(new Path(recoveryDestination, makeRelative(srcPath))); - final Path recoveredPath = - new Path(recoveredPrefix + "." + new Random().nextLong() + ".recovered"); - LOG.info("Creating recovered file " + recoveredPath); - - FileSystem srcFs = srcPath.getFileSystem(conf); - decoder.decodeFile(srcFs, srcPath, ppair.getFileSystem(), - ppair.getPath(), corruptOffset, recoveredPath); - - return recoveredPath; - } - - public static Path unRaidCorruptBlock(Configuration conf, Path srcPath, - Path destPathPrefix, Decoder decoder, int stripeLength, - long corruptOffset) throws IOException { - // Test if parity file exists - ParityFilePair ppair = getParityFile(destPathPrefix, srcPath, conf); - if (ppair == null) { - LOG.error("Could not find parity file for " + srcPath); - return null; - } - - final Path recoveryDestination = new Path(RaidNode.xorTempPrefix(conf)); - FileSystem destFs = recoveryDestination.getFileSystem(conf); - final Path recoveredPrefix = - destFs.makeQualified(new Path(recoveryDestination, makeRelative(srcPath))); - final Path recoveredBlock = - new Path(recoveredPrefix + "." + new Random().nextLong() + ".recovered"); - LOG.info("Creating recovered Block " + recoveredBlock); - - FileSystem srcFs = srcPath.getFileSystem(conf); - FileStatus stat = srcFs.getFileStatus(srcPath); - long limit = Math.min(stat.getBlockSize(), stat.getLen() - corruptOffset); - java.io.OutputStream out = ppair.getFileSystem().create(recoveredBlock); - decoder.fixErasedBlock(srcFs, srcPath, - ppair.getFileSystem(), ppair.getPath(), - stat.getBlockSize(), corruptOffset, 0, limit, out); - out.close(); - return recoveredBlock; - } - - /** - * Periodically delete orphaned parity files. - */ - class PurgeMonitor implements Runnable { - /** - */ - public void run() { - while (running) { - try { - doPurge(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - LOG.info("Purge parity files thread continuing to run..."); - } - } - } - - /** - * Traverse the parity destination directory, removing directories that - * no longer existing in the source. - * @throws IOException - */ - private void purgeDirectories(FileSystem fs, Path root) throws IOException { - String prefix = root.toUri().getPath(); - List startPaths = new LinkedList(); - try { - startPaths.add(fs.getFileStatus(root)); - } catch (FileNotFoundException e) { - return; - } - DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths); - FileStatus dir = dt.getNextDirectory(); - for (; dir != null; dir = dt.getNextDirectory()) { - Path dirPath = dir.getPath(); - if (dirPath.toUri().getPath().endsWith(HAR_SUFFIX)) { - continue; - } - String dirStr = dirPath.toUri().getPath(); - if (!dirStr.startsWith(prefix)) { - continue; - } - String src = dirStr.replaceFirst(prefix, ""); - if (src.length() == 0) continue; - Path srcPath = new Path(src); - if (!fs.exists(srcPath)) { - LOG.info("Purging directory " + dirPath); - boolean done = fs.delete(dirPath, true); - if (!done) { - LOG.error("Could not purge " + dirPath); - } - } - } - } - - /** - * Delete orphaned files. The reason this is done by a separate thread - * is to not burden the TriggerMonitor with scanning the - * destination directories. - */ - private void doPurge() throws IOException, InterruptedException { - long prevExec = 0; - while (running) { - - // The config may be reloaded by the TriggerMonitor. - // This thread uses whatever config is currently active. - while(now() < prevExec + configMgr.getPeriodicity()){ - Thread.sleep(SLEEP_TIME); - } - - LOG.info("Started purge scan"); - prevExec = now(); - - // expand destination prefix path - Path destPref = xorDestinationPath(conf); - FileSystem destFs = destPref.getFileSystem(conf); - purgeDirectories(destFs, destPref); - - destPref = rsDestinationPath(conf); - destFs = destPref.getFileSystem(conf); - purgeDirectories(destFs, destPref); - - // fetch all categories - for (PolicyList category : configMgr.getAllPolicies()) { - for (PolicyInfo info: category.getAll()) { - - try { - // expand destination prefix path - destPref = getDestinationPath(info.getErasureCode(), conf); - destFs = destPref.getFileSystem(conf); - - //get srcPaths - Path[] srcPaths = info.getSrcPathExpanded(); - - if (srcPaths != null) { - for (Path srcPath: srcPaths) { - // expand destination prefix - Path destPath = getOriginalParityFile(destPref, srcPath); - - FileSystem srcFs = info.getSrcPath().getFileSystem(conf); - FileStatus stat = null; - try { - stat = destFs.getFileStatus(destPath); - } catch (FileNotFoundException e) { - // do nothing, leave stat = null; - } - if (stat != null) { - LOG.info("Purging obsolete parity files for policy " + - info.getName() + " " + destPath); - recursePurge(info.getErasureCode(), srcFs, destFs, - destPref.toUri().getPath(), stat); - } - - } - } - } catch (Exception e) { - LOG.warn("Ignoring Exception while processing policy " + - info.getName() + " " + - StringUtils.stringifyException(e)); - } - } - } - } - } - - /** - * The destPrefix is the absolute pathname of the destinationPath - * specified in the policy (without the host:port) - */ - void recursePurge(ErasureCodeType code, - FileSystem srcFs, FileSystem destFs, - String destPrefix, FileStatus dest) - throws IOException { - - Path destPath = dest.getPath(); // pathname, no host:port - String destStr = destPath.toUri().getPath(); - LOG.debug("Checking " + destPath + " prefix " + destPrefix); - - // Verify if it is a har file - if (dest.isDirectory() && destStr.endsWith(HAR_SUFFIX)) { - try { - int harUsedPercent = - usefulHar(code, srcFs, destFs, destPath, destPrefix, conf); - LOG.info("Useful percentage of " + destStr + " " + harUsedPercent); - // Delete the har if its usefulness reaches a threshold. - if (harUsedPercent <= conf.getInt("raid.har.usage.threshold", 0)) { - LOG.info("Purging " + destStr + " at usage " + harUsedPercent); - boolean done = destFs.delete(destPath, true); - if (!done) { - LOG.error("Unable to purge directory " + destPath); - } - } - } catch (IOException e) { - LOG.warn("Error during purging " + destStr + " " + - StringUtils.stringifyException(e)); - } - return; - } - - // Verify the destPrefix is a prefix of the destPath - if (!destStr.startsWith(destPrefix)) { - LOG.error("Destination path " + destStr + " should have " + - destPrefix + " as its prefix."); - return; - } - - if (dest.isDirectory()) { - FileStatus[] files = null; - files = destFs.listStatus(destPath); - if (files == null || files.length == 0){ - boolean done = destFs.delete(destPath,true); // ideal is false, but - // DFSClient only deletes directories if it is recursive - if (done) { - LOG.info("Purged directory " + destPath ); - } - else { - LOG.error("Unable to purge directory " + destPath); - } - } - if (files != null) { - for (FileStatus one:files) { - recursePurge(code, srcFs, destFs, destPrefix, one); - } - } - // If the directory is empty now, it will be purged the next time this - // thread runs. - return; // the code below does the file checking - } - - String src = destStr.replaceFirst(destPrefix, ""); - - Path srcPath = new Path(src); - boolean shouldDelete = false; - - if (!srcFs.exists(srcPath)) { - shouldDelete = true; - } else { - try { - // If there is a RS parity file, the XOR parity can be deleted. - if (code == ErasureCodeType.XOR) { - ParityFilePair ppair = getParityFile( - getDestinationPath(ErasureCodeType.RS, conf), srcPath, conf); - if (ppair != null) { - shouldDelete = true; - } - } - if (!shouldDelete) { - Path dstPath = (new Path(destPrefix.trim())).makeQualified(destFs); - ParityFilePair ppair = getParityFile(dstPath,srcPath); - // If the parity file is not the appropriate one for the source or - // the parityFs is not the same as this file's filesystem - // (it is a HAR), this file can be deleted. - if ( ppair == null || - !destFs.equals(ppair.getFileSystem()) || - !destPath.equals(ppair.getPath())) { - shouldDelete = true; - } - } - } catch (IOException e) { - LOG.warn("Error during purging " + src + " " + - StringUtils.stringifyException(e)); - } - } - - if (shouldDelete) { - boolean done = destFs.delete(destPath, false); - if (done) { - LOG.info("Purged file " + destPath ); - } else { - LOG.error("Unable to purge file " + destPath ); - } - } - } - } - - // - // Returns the number of up-to-date files in the har as a percentage of the - // total number of files in the har. - // - protected static int usefulHar( - ErasureCodeType code, - FileSystem srcFs, FileSystem destFs, - Path harPath, String destPrefix, Configuration conf) throws IOException { - - FileSystem fsHar = new HarFileSystem(destFs); - String harURIPath = harPath.toUri().getPath(); - Path qualifiedPath = new Path("har://", harURIPath + - Path.SEPARATOR + harPath.getParent().toUri().getPath()); - fsHar.initialize(qualifiedPath.toUri(), conf); - FileStatus[] filesInHar = fsHar.listStatus(qualifiedPath); - if (filesInHar.length == 0) { - return 0; - } - int numUseless = 0; - for (FileStatus one: filesInHar) { - Path parityPath = one.getPath(); - String parityStr = parityPath.toUri().getPath(); - if (parityStr.startsWith("har:/")) { - LOG.error("Unexpected prefix har:/ for " + parityStr); - continue; - } - String prefixToReplace = harURIPath + destPrefix; - if (!parityStr.startsWith(prefixToReplace)) { - continue; - } - String src = parityStr.substring(prefixToReplace.length()); - if (code == ErasureCodeType.XOR) { - ParityFilePair ppair = getParityFile( - getDestinationPath(ErasureCodeType.RS, conf), new Path(src), conf); - if (ppair != null) { - // There is a valid RS parity file, so the XOR one is useless. - numUseless++; - continue; - } - } - try { - FileStatus srcStatus = srcFs.getFileStatus(new Path(src)); - if (srcStatus == null) { - numUseless++; - } else if (one.getModificationTime() != - srcStatus.getModificationTime()) { - numUseless++; - } - } catch (FileNotFoundException e) { - LOG.info("File not found: " + e); - numUseless++; - } - } - int uselessPercent = numUseless * 100 / filesInHar.length; - return 100 - uselessPercent; - } - - private void doHar() throws IOException, InterruptedException { - long prevExec = 0; - while (running) { - - // The config may be reloaded by the TriggerMonitor. - // This thread uses whatever config is currently active. - while(now() < prevExec + configMgr.getPeriodicity()){ - Thread.sleep(SLEEP_TIME); - } - - LOG.info("Started archive scan"); - prevExec = now(); - - // fetch all categories - for (PolicyList category : configMgr.getAllPolicies()) { - for (PolicyInfo info: category.getAll()) { - String tmpHarPath = tmpHarPathForCode(conf, info.getErasureCode()); - String str = info.getProperty("time_before_har"); - if (str != null) { - try { - long cutoff = now() - ( Long.parseLong(str) * 24L * 3600000L ); - - Path destPref = getDestinationPath(info.getErasureCode(), conf); - FileSystem destFs = destPref.getFileSystem(conf); - - //get srcPaths - Path[] srcPaths = info.getSrcPathExpanded(); - - if ( srcPaths != null ){ - for (Path srcPath: srcPaths) { - // expand destination prefix - Path destPath = getOriginalParityFile(destPref, srcPath); - - FileStatus stat = null; - try { - stat = destFs.getFileStatus(destPath); - } catch (FileNotFoundException e) { - // do nothing, leave stat = null; - } - if (stat != null) { - LOG.info("Haring parity files for policy " + - info.getName() + " " + destPath); - recurseHar(info, destFs, stat, destPref.toUri().getPath(), - srcPath.getFileSystem(conf), cutoff, tmpHarPath); - } - } - } - } catch (Exception e) { - LOG.warn("Ignoring Exception while processing policy " + - info.getName() + " " + - StringUtils.stringifyException(e)); - } - } - } - } - } - return; - } - - void recurseHar(PolicyInfo info, FileSystem destFs, FileStatus dest, - String destPrefix, FileSystem srcFs, long cutoff, String tmpHarPath) - throws IOException { - - if (dest.isFile()) { - return; - } - - Path destPath = dest.getPath(); // pathname, no host:port - String destStr = destPath.toUri().getPath(); - - // Verify if it already contains a HAR directory - if ( destFs.exists(new Path(destPath, destPath.getName()+HAR_SUFFIX)) ) { - return; - } - - FileStatus[] files = null; - files = destFs.listStatus(destPath); - boolean shouldHar = false; - if (files != null) { - shouldHar = files.length > 0; - for (FileStatus one: files) { - if (one.isDirectory()){ - recurseHar(info, destFs, one, destPrefix, srcFs, cutoff, tmpHarPath); - shouldHar = false; - } else if (one.getModificationTime() > cutoff ) { - if (shouldHar) { - LOG.info("Cannot archive " + destPath + - " because " + one.getPath() + " was modified after cutoff"); - shouldHar = false; - } - } - } - - if (shouldHar) { - String src = destStr.replaceFirst(destPrefix, ""); - Path srcPath = new Path(src); - FileStatus[] statuses = srcFs.listStatus(srcPath); - Path destPathPrefix = new Path(destPrefix).makeQualified(destFs); - if (statuses != null) { - for (FileStatus status : statuses) { - if (getParityFile(destPathPrefix, - status.getPath().makeQualified(srcFs)) == null ) { - LOG.info("Cannot archive " + destPath + - " because it doesn't contain parity file for " + - status.getPath().makeQualified(srcFs) + " on destination " + - destPathPrefix); - shouldHar = false; - break; - } - } - } - } - } - - if ( shouldHar ) { - LOG.info("Archiving " + dest.getPath() + " to " + tmpHarPath ); - singleHar(info, destFs, dest, tmpHarPath); - } - } - - - private void singleHar(PolicyInfo info, FileSystem destFs, FileStatus dest, - String tmpHarPath) throws IOException { - - Random rand = new Random(); - Path root = new Path("/"); - Path qualifiedPath = dest.getPath().makeQualified(destFs); - String harFileDst = qualifiedPath.getName() + HAR_SUFFIX; - String harFileSrc = qualifiedPath.getName() + "-" + - rand.nextLong() + "-" + HAR_SUFFIX; - short metaReplication = - (short) Integer.parseInt(info.getProperty("metaReplication")); - // HadoopArchives.HAR_PARTFILE_LABEL is private, so hard-coding the label. - conf.setLong("har.partfile.size", configMgr.getHarPartfileSize()); - HadoopArchives har = new HadoopArchives(conf); - String[] args = new String[7]; - args[0] = "-Ddfs.replication=" + metaReplication; - args[1] = "-archiveName"; - args[2] = harFileSrc; - args[3] = "-p"; - args[4] = root.makeQualified(destFs).toString(); - args[5] = qualifiedPath.toUri().getPath().substring(1); - args[6] = tmpHarPath.toString(); - int ret = 0; - try { - ret = ToolRunner.run(har, args); - if (ret == 0 && !destFs.rename(new Path(tmpHarPath+"/"+harFileSrc), - new Path(qualifiedPath, harFileDst))) { - LOG.error("HAR rename didn't succeed from " + tmpHarPath+"/"+harFileSrc + - " to " + qualifiedPath + "/" + harFileDst); - ret = -2; - } - } catch (Exception exc) { - throw new IOException("Error while creating archive " + ret, exc); - } - - if (ret != 0){ - throw new IOException("Error while creating archive " + ret); - } - return; - } - - /** - * Periodically generates HAR files - */ - class HarMonitor implements Runnable { - - public void run() { - while (running) { - try { - doHar(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - LOG.info("Har parity files thread continuing to run..."); - } - } - LOG.info("Leaving Har thread."); - } - - } - - /** - * Return the temp path for XOR parity files - */ - public static String unraidTmpDirectory(Configuration conf) { - return conf.get(RAID_TMP_LOCATION_KEY, DEFAULT_RAID_TMP_LOCATION); - } - - /** - * Return the temp path for ReedSolomonEncoder parity files - */ - public static String rsTempPrefix(Configuration conf) { - return conf.get(RAIDRS_TMP_LOCATION_KEY, DEFAULT_RAIDRS_TMP_LOCATION); - } - - /** - * Return the temp path for XOR parity files - */ - public static String xorTempPrefix(Configuration conf) { - return conf.get(RAID_TMP_LOCATION_KEY, DEFAULT_RAID_TMP_LOCATION); - } - - /** - * Return the temp path for XOR parity files - */ - public static String xorHarTempPrefix(Configuration conf) { - return conf.get(RAID_HAR_TMP_LOCATION_KEY, DEFAULT_RAID_HAR_TMP_LOCATION); - } - - /** - * Return the temp path for ReedSolomonEncoder parity files - */ - public static String rsHarTempPrefix(Configuration conf) { - return conf.get(RAIDRS_HAR_TMP_LOCATION_KEY, - DEFAULT_RAIDRS_HAR_TMP_LOCATION); - } - - /** - * Return the destination path for ReedSolomon parity files - */ - public static Path rsDestinationPath(Configuration conf, FileSystem fs) { - String loc = conf.get(RAIDRS_LOCATION_KEY, DEFAULT_RAIDRS_LOCATION); - Path p = new Path(loc.trim()); - p = p.makeQualified(fs); - return p; - } - - /** - * Return the destination path for ReedSolomon parity files - */ - public static Path rsDestinationPath(Configuration conf) - throws IOException { - String loc = conf.get(RAIDRS_LOCATION_KEY, DEFAULT_RAIDRS_LOCATION); - Path p = new Path(loc.trim()); - FileSystem fs = FileSystem.get(p.toUri(), conf); - p = p.makeQualified(fs); - return p; - } - - /** - * Return the destination path for XOR parity files - */ - public static Path xorDestinationPath(Configuration conf, FileSystem fs) { - String loc = conf.get(RAID_LOCATION_KEY, DEFAULT_RAID_LOCATION); - Path p = new Path(loc.trim()); - p = p.makeQualified(fs); - return p; - } - - /** - * Return the destination path for XOR parity files - */ - public static Path xorDestinationPath(Configuration conf) - throws IOException { - String loc = conf.get(RAID_LOCATION_KEY, DEFAULT_RAID_LOCATION); - Path p = new Path(loc.trim()); - FileSystem fs = FileSystem.get(p.toUri(), conf); - p = p.makeQualified(fs); - return p; - } - - /** - * Return the path prefix that stores the parity files - */ - static Path getDestinationPath(ErasureCodeType code, Configuration conf) - throws IOException { - switch (code) { - case XOR: - return xorDestinationPath(conf); - case RS: - return rsDestinationPath(conf); - default: - return null; - } - } - - static Encoder encoderForCode(Configuration conf, ErasureCodeType code) { - int stripeLength = getStripeLength(conf); - switch (code) { - case XOR: - return new XOREncoder(conf, stripeLength); - case RS: - return new ReedSolomonEncoder(conf, stripeLength, rsParityLength(conf)); - default: - return null; - } - } - - static String tmpHarPathForCode(Configuration conf, ErasureCodeType code) { - switch (code) { - case XOR: - return xorHarTempPrefix(conf); - case RS: - return rsHarTempPrefix(conf); - default: - return null; - } - } - - /** - * Obtain stripe length from configuration - */ - public static int getStripeLength(Configuration conf) { - return conf.getInt(STRIPE_LENGTH_KEY, DEFAULT_STRIPE_LENGTH); - } - - /** - * Obtain stripe length from configuration - */ - public static int rsParityLength(Configuration conf) { - return conf.getInt(RS_PARITY_LENGTH_KEY, RS_PARITY_LENGTH_DEFAULT); - } - - static boolean isParityHarPartFile(Path p) { - Matcher m = PARITY_HAR_PARTFILE_PATTERN.matcher(p.toUri().getPath()); - return m.matches(); - } - - /** - * Returns current time. - */ - static long now() { - return Time.now(); - } - - /** - * Make an absolute path relative by stripping the leading / - */ - static private Path makeRelative(Path path) { - if (!path.isAbsolute()) { - return path; - } - String p = path.toUri().getPath(); - String relative = p.substring(1, p.length()); - return new Path(relative); - } - - private static void printUsage() { - System.err.println("Usage: java RaidNode "); - } - - private static StartupOption parseArguments(String args[]) { - int argsLen = (args == null) ? 0 : args.length; - StartupOption startOpt = StartupOption.REGULAR; - for(int i=0; i < argsLen; i++) { - String cmd = args[i]; // We have to parse command line args in future. - } - return startOpt; - } - - - /** - * Convert command line options to configuration parameters - */ - private static void setStartupOption(Configuration conf, StartupOption opt) { - conf.set("fs.raidnode.startup", opt.toString()); - } - - /** - * Create an instance of the appropriate subclass of RaidNode - */ - public static RaidNode createRaidNode(Configuration conf) - throws ClassNotFoundException { - try { - // default to distributed raid node - Class raidNodeClass = - conf.getClass(RAIDNODE_CLASSNAME_KEY, DistRaidNode.class); - if (!RaidNode.class.isAssignableFrom(raidNodeClass)) { - throw new ClassNotFoundException("not an implementation of RaidNode"); - } - Constructor constructor = - raidNodeClass.getConstructor(new Class[] {Configuration.class} ); - return (RaidNode) constructor.newInstance(conf); - } catch (NoSuchMethodException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (InstantiationException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (IllegalAccessException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } catch (InvocationTargetException e) { - throw new ClassNotFoundException("cannot construct blockfixer", e); - } - } - - /** - * Create an instance of the RaidNode - */ - public static RaidNode createRaidNode(String argv[], Configuration conf) - throws IOException, ClassNotFoundException { - if (conf == null) { - conf = new Configuration(); - } - StartupOption startOpt = parseArguments(argv); - if (startOpt == null) { - printUsage(); - return null; - } - setStartupOption(conf, startOpt); - RaidNode node = createRaidNode(conf); - return node; - } - - - /** - */ - public static void main(String argv[]) throws Exception { - try { - StringUtils.startupShutdownMessage(RaidNode.class, argv, LOG); - RaidNode raid = createRaidNode(argv, null); - if (raid != null) { - raid.join(); - } - } catch (Throwable e) { - LOG.error(StringUtils.stringifyException(e)); - System.exit(-1); - } - } - - - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java deleted file mode 100644 index 625a6a8eefe..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java +++ /dev/null @@ -1,682 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.IOException; -import java.io.FileNotFoundException; -import java.util.Collection; -import java.util.Map; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.LinkedHashMap; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.net.InetSocketAddress; -import javax.security.auth.login.LoginException; - -import org.apache.hadoop.ipc.*; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.io.retry.RetryPolicy; -import org.apache.hadoop.io.retry.RetryPolicies; -import org.apache.hadoop.io.retry.RetryProxy; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.HarFileSystem; - -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.DistributedRaidFileSystem; -import org.apache.hadoop.hdfs.RaidDFSUtil; - -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.raid.protocol.PolicyList; -import org.apache.hadoop.raid.protocol.RaidProtocol; - -/** - * A {@link RaidShell} that allows browsing configured raid policies. - */ -public class RaidShell extends Configured implements Tool { - static { - Configuration.addDefaultResource("hdfs-default.xml"); - Configuration.addDefaultResource("hdfs-site.xml"); - } - public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.RaidShell"); - public RaidProtocol raidnode; - RaidProtocol rpcRaidnode; - private UserGroupInformation ugi; - volatile boolean clientRunning = true; - private Configuration conf; - - /** - * Start RaidShell. - *

- * The RaidShell connects to the specified RaidNode and performs basic - * configuration options. - * @throws IOException - */ - public RaidShell(Configuration conf) throws IOException { - super(conf); - this.conf = conf; - } - - void initializeRpc(Configuration conf, InetSocketAddress address) throws IOException { - this.ugi = UserGroupInformation.getCurrentUser(); - this.rpcRaidnode = createRPCRaidnode(address, conf, ugi); - this.raidnode = createRaidnode(rpcRaidnode); - } - - void initializeLocal(Configuration conf) throws IOException { - this.ugi = UserGroupInformation.getCurrentUser(); - } - - public static RaidProtocol createRaidnode(Configuration conf) throws IOException { - return createRaidnode(RaidNode.getAddress(conf), conf); - } - - public static RaidProtocol createRaidnode(InetSocketAddress raidNodeAddr, - Configuration conf) throws IOException { - return createRaidnode(createRPCRaidnode(raidNodeAddr, conf, - UserGroupInformation.getCurrentUser())); - } - - private static RaidProtocol createRPCRaidnode(InetSocketAddress raidNodeAddr, - Configuration conf, UserGroupInformation ugi) - throws IOException { - LOG.debug("RaidShell connecting to " + raidNodeAddr); - return (RaidProtocol)RPC.getProxy(RaidProtocol.class, - RaidProtocol.versionID, raidNodeAddr, ugi, conf, - NetUtils.getSocketFactory(conf, RaidProtocol.class)); - } - - private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode) - throws IOException { - RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( - 5, 5000, TimeUnit.MILLISECONDS); - - Map,RetryPolicy> remoteExceptionToPolicyMap = - new HashMap, RetryPolicy>(); - - Map,RetryPolicy> exceptionToPolicyMap = - new HashMap, RetryPolicy>(); - exceptionToPolicyMap.put(RemoteException.class, - RetryPolicies.retryByRemoteException( - RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); - RetryPolicy methodPolicy = RetryPolicies.retryByException( - RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); - Map methodNameToPolicyMap = new HashMap(); - - methodNameToPolicyMap.put("create", methodPolicy); - - return (RaidProtocol) RetryProxy.create(RaidProtocol.class, - rpcRaidnode, methodNameToPolicyMap); - } - - private void checkOpen() throws IOException { - if (!clientRunning) { - IOException result = new IOException("RaidNode closed"); - throw result; - } - } - - /** - * Close the connection to the raidNode. - */ - public synchronized void close() throws IOException { - if(clientRunning) { - clientRunning = false; - RPC.stopProxy(rpcRaidnode); - } - } - - /** - * Displays format of commands. - */ - private static void printUsage(String cmd) { - String prefix = "Usage: java " + RaidShell.class.getSimpleName(); - if ("-showConfig".equals(cmd)) { - System.err.println("Usage: java RaidShell" + - " [-showConfig]"); - } else if ("-recover".equals(cmd)) { - System.err.println("Usage: java RaidShell" + - " [-recover srcPath1 corruptOffset]"); - } else if ("-recoverBlocks".equals(cmd)) { - System.err.println("Usage: java RaidShell" + - " [-recoverBlocks path1 path2...]"); - } else { - System.err.println("Usage: java RaidShell"); - System.err.println(" [-showConfig ]"); - System.err.println(" [-help [cmd]]"); - System.err.println(" [-recover srcPath1 corruptOffset]"); - System.err.println(" [-recoverBlocks path1 path2...]"); - System.err.println(" [-fsck [path]]"); - System.err.println(); - ToolRunner.printGenericCommandUsage(System.err); - } - } - - /** - * run - */ - public int run(String argv[]) throws Exception { - - if (argv.length < 1) { - printUsage(""); - return -1; - } - - int exitCode = -1; - int i = 0; - String cmd = argv[i++]; - // - // verify that we have enough command line parameters - // - if ("-showConfig".equals(cmd)) { - if (argv.length < 1) { - printUsage(cmd); - return exitCode; - } - } else if ("-recover".equals(cmd)) { - if (argv.length < 3) { - printUsage(cmd); - return exitCode; - } - } else if ("-fsck".equals(cmd)) { - if ((argv.length < 1) || (argv.length > 2)) { - printUsage(cmd); - return exitCode; - } - } - - try { - if ("-showConfig".equals(cmd)) { - initializeRpc(conf, RaidNode.getAddress(conf)); - exitCode = showConfig(cmd, argv, i); - } else if ("-recover".equals(cmd)) { - initializeRpc(conf, RaidNode.getAddress(conf)); - exitCode = recoverAndPrint(cmd, argv, i); - } else if ("-recoverBlocks".equals(cmd)) { - initializeLocal(conf); - recoverBlocks(argv, i); - exitCode = 0; - } else if ("-fsck".equals(cmd)) { - if (argv.length == 1) { - // if there are no args, check the whole file system - exitCode = fsck("/"); - } else { - // argv.length == 2 - // otherwise, check the path passed - exitCode = fsck(argv[1]); - } - } else { - exitCode = -1; - System.err.println(cmd.substring(1) + ": Unknown command"); - printUsage(""); - } - } catch (IllegalArgumentException arge) { - exitCode = -1; - System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage()); - printUsage(cmd); - } catch (RemoteException e) { - // - // This is a error returned by raidnode server. Print - // out the first line of the error mesage, ignore the stack trace. - exitCode = -1; - try { - String[] content; - content = e.getLocalizedMessage().split("\n"); - System.err.println(cmd.substring(1) + ": " + - content[0]); - } catch (Exception ex) { - System.err.println(cmd.substring(1) + ": " + - ex.getLocalizedMessage()); - } - } catch (IOException e) { - // - // IO exception encountered locally. - // - exitCode = -1; - System.err.println(cmd.substring(1) + ": " + - e.getLocalizedMessage()); - } catch (Exception re) { - exitCode = -1; - System.err.println(cmd.substring(1) + ": " + re.getLocalizedMessage()); - } finally { - } - return exitCode; - } - - /** - * Apply operation specified by 'cmd' on all parameters - * starting from argv[startindex]. - */ - private int showConfig(String cmd, String argv[], int startindex) throws IOException { - int exitCode = 0; - int i = startindex; - PolicyList[] all = raidnode.getAllPolicies(); - for (PolicyList list: all) { - for (PolicyInfo p : list.getAll()) { - System.out.println(p); - } - } - return exitCode; - } - - /** - * Recovers the specified path from the parity file - */ - public Path[] recover(String cmd, String argv[], int startindex) - throws IOException { - Path[] paths = new Path[(argv.length - startindex) / 2]; - int j = 0; - for (int i = startindex; i < argv.length; i = i + 2) { - String path = argv[i]; - long corruptOffset = Long.parseLong(argv[i+1]); - LOG.info("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset); - Path recovered = new Path("/tmp/recovered." + Time.now()); - FileSystem fs = recovered.getFileSystem(conf); - DistributedFileSystem dfs = (DistributedFileSystem)fs; - Configuration raidConf = new Configuration(conf); - raidConf.set("fs.hdfs.impl", - "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); - raidConf.set("fs.raid.underlyingfs.impl", - "org.apache.hadoop.hdfs.DistributedFileSystem"); - raidConf.setBoolean("fs.hdfs.impl.disable.cache", true); - java.net.URI dfsUri = dfs.getUri(); - FileSystem raidFs = FileSystem.get(dfsUri, raidConf); - FileUtil.copy(raidFs, new Path(path), fs, recovered, false, conf); - - paths[j] = recovered; - LOG.info("Raidshell created recovery file " + paths[j]); - j++; - } - return paths; - } - - public int recoverAndPrint(String cmd, String argv[], int startindex) - throws IOException { - int exitCode = 0; - for (Path p : recover(cmd,argv,startindex)) { - System.out.println(p); - } - return exitCode; - } - - public void recoverBlocks(String[] args, int startIndex) - throws IOException { - LOG.debug("Recovering blocks for " + (args.length - startIndex) + " files"); - BlockFixer.BlockFixerHelper fixer = new BlockFixer.BlockFixerHelper(conf); - for (int i = startIndex; i < args.length; i++) { - String path = args[i]; - fixer.fixFile(new Path(path)); - } - } - - /** - * checks whether a file has more than the allowable number of - * corrupt blocks and must therefore be considered corrupt - */ - private boolean isFileCorrupt(final DistributedFileSystem dfs, - final Path filePath) - throws IOException { - // corruptBlocksPerStripe: - // map stripe # -> # of corrupt blocks in that stripe (data + parity) - HashMap corruptBlocksPerStripe = - new LinkedHashMap(); - - // read conf - final int stripeBlocks = RaidNode.getStripeLength(conf); - - // figure out which blocks are missing/corrupted - final FileStatus fileStatus = dfs.getFileStatus(filePath); - final long blockSize = fileStatus.getBlockSize(); - final long fileLength = fileStatus.getLen(); - final long fileLengthInBlocks = (fileLength / blockSize) + - (((fileLength % blockSize) == 0) ? 0L : 1L); - final long fileStripes = (fileLengthInBlocks / stripeBlocks) + - (((fileLengthInBlocks % stripeBlocks) == 0) ? 0L : 1L); - final BlockLocation[] fileBlocks = - dfs.getFileBlockLocations(fileStatus, 0, fileLength); - - // figure out which stripes these corrupted blocks belong to - for (BlockLocation fileBlock: fileBlocks) { - int blockNo = (int) (fileBlock.getOffset() / blockSize); - final int stripe = (int) (blockNo / stripeBlocks); - if (fileBlock.isCorrupt() || - (fileBlock.getNames().length == 0 && fileBlock.getLength() > 0)) { - if (corruptBlocksPerStripe.get(stripe) == null) { - corruptBlocksPerStripe.put(stripe, 1); - } else { - corruptBlocksPerStripe.put(stripe, corruptBlocksPerStripe. - get(stripe) + 1); - } - LOG.debug("file " + filePath.toString() + " corrupt in block " + - blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe + - "/" + fileStripes); - } else { - LOG.debug("file " + filePath.toString() + " OK in block " + blockNo + - "/" + fileLengthInBlocks + ", stripe " + stripe + "/" + - fileStripes); - } - } - - RaidInfo raidInfo = getFileRaidInfo(dfs, filePath); - - // now check parity blocks - if (raidInfo.raidType != RaidType.NONE) { - checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize, - fileStripes, raidInfo); - } - - final int maxCorruptBlocksPerStripe = raidInfo.parityBlocksPerStripe; - - for (int corruptBlocksInStripe: corruptBlocksPerStripe.values()) { - if (corruptBlocksInStripe > maxCorruptBlocksPerStripe) { - return true; - } - } - return false; - } - - /** - * holds the type of raid used for a particular file - */ - private enum RaidType { - XOR, - RS, - NONE - } - - /** - * holds raid type and parity file pair - */ - private class RaidInfo { - public RaidInfo(final RaidType raidType, - final RaidNode.ParityFilePair parityPair, - final int parityBlocksPerStripe) { - this.raidType = raidType; - this.parityPair = parityPair; - this.parityBlocksPerStripe = parityBlocksPerStripe; - } - public final RaidType raidType; - public final RaidNode.ParityFilePair parityPair; - public final int parityBlocksPerStripe; - } - - /** - * returns the raid for a given file - */ - private RaidInfo getFileRaidInfo(final DistributedFileSystem dfs, - final Path filePath) - throws IOException { - // now look for the parity file - Path destPath = null; - RaidNode.ParityFilePair ppair = null; - try { - // look for xor parity file first - destPath = RaidNode.xorDestinationPath(conf); - ppair = RaidNode.getParityFile(destPath, filePath, conf); - } catch (FileNotFoundException ignore) { - } - if (ppair != null) { - return new RaidInfo(RaidType.XOR, ppair, 1); - } else { - // failing that, look for rs parity file - try { - destPath = RaidNode.rsDestinationPath(conf); - ppair = RaidNode.getParityFile(destPath, filePath, conf); - } catch (FileNotFoundException ignore) { - } - if (ppair != null) { - return new RaidInfo(RaidType.RS, ppair, RaidNode.rsParityLength(conf)); - } else { - return new RaidInfo(RaidType.NONE, null, 0); - } - } - } - - /** - * gets the parity blocks corresponding to file - * returns the parity blocks in case of DFS - * and the part blocks containing parity blocks - * in case of HAR FS - */ - private BlockLocation[] getParityBlocks(final Path filePath, - final long blockSize, - final long fileStripes, - final RaidInfo raidInfo) - throws IOException { - - - final String parityPathStr = raidInfo.parityPair.getPath().toUri(). - getPath(); - FileSystem parityFS = raidInfo.parityPair.getFileSystem(); - - // get parity file metadata - FileStatus parityFileStatus = parityFS. - getFileStatus(new Path(parityPathStr)); - long parityFileLength = parityFileStatus.getLen(); - - if (parityFileLength != fileStripes * raidInfo.parityBlocksPerStripe * - blockSize) { - throw new IOException("expected parity file of length" + - (fileStripes * raidInfo.parityBlocksPerStripe * - blockSize) + - " but got parity file of length " + - parityFileLength); - } - - BlockLocation[] parityBlocks = - parityFS.getFileBlockLocations(parityFileStatus, 0L, parityFileLength); - - if (parityFS instanceof DistributedFileSystem || - parityFS instanceof DistributedRaidFileSystem) { - long parityBlockSize = parityFileStatus.getBlockSize(); - if (parityBlockSize != blockSize) { - throw new IOException("file block size is " + blockSize + - " but parity file block size is " + - parityBlockSize); - } - } else if (parityFS instanceof HarFileSystem) { - LOG.debug("HAR FS found"); - } else { - LOG.warn("parity file system is not of a supported type"); - } - - return parityBlocks; - } - - /** - * checks the parity blocks for a given file and modifies - * corruptBlocksPerStripe accordingly - */ - private void checkParityBlocks(final Path filePath, - final HashMap - corruptBlocksPerStripe, - final long blockSize, - final long fileStripes, - final RaidInfo raidInfo) - throws IOException { - - // get the blocks of the parity file - // because of har, multiple blocks may be returned as one container block - BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize, - fileStripes, raidInfo); - - long parityStripeLength = blockSize * - ((long) raidInfo.parityBlocksPerStripe); - - long parityFileLength = parityStripeLength * fileStripes; - - long parityBlocksFound = 0L; - - for (BlockLocation cb: containerBlocks) { - if (cb.getLength() % blockSize != 0) { - throw new IOException("container block size is not " + - "multiple of parity block size"); - } - int blocksInContainer = (int) (cb.getLength() / blockSize); - LOG.debug("found container with offset " + cb.getOffset() + - ", length " + cb.getLength()); - - for (long offset = cb.getOffset(); - offset < cb.getOffset() + cb.getLength(); - offset += blockSize) { - long block = offset / blockSize; - - int stripe = (int) (offset / parityStripeLength); - - if (stripe < 0) { - // before the beginning of the parity file - continue; - } - if (stripe >= fileStripes) { - // past the end of the parity file - break; - } - - parityBlocksFound++; - - if (cb.isCorrupt() || - (cb.getNames().length == 0 && cb.getLength() > 0)) { - LOG.debug("parity file for " + filePath.toString() + - " corrupt in block " + block + - ", stripe " + stripe + "/" + fileStripes); - - if (corruptBlocksPerStripe.get(stripe) == null) { - corruptBlocksPerStripe.put(stripe, 1); - } else { - corruptBlocksPerStripe.put(stripe, - corruptBlocksPerStripe.get(stripe) + - 1); - } - } else { - LOG.debug("parity file for " + filePath.toString() + - " OK in block " + block + - ", stripe " + stripe + "/" + fileStripes); - } - } - } - - long parityBlocksExpected = raidInfo.parityBlocksPerStripe * fileStripes; - if (parityBlocksFound != parityBlocksExpected ) { - throw new IOException("expected " + parityBlocksExpected + - " parity blocks but got " + parityBlocksFound); - } - } - - - /** - * checks the raided file system, prints a list of corrupt files to - * System.out and returns the number of corrupt files - */ - public int fsck(final String path) throws IOException { - - FileSystem fs = (new Path(path)).getFileSystem(conf); - - // if we got a raid fs, get the underlying fs - if (fs instanceof DistributedRaidFileSystem) { - fs = ((DistributedRaidFileSystem) fs).getFileSystem(); - } - - // check that we have a distributed fs - if (!(fs instanceof DistributedFileSystem)) { - throw new IOException("expected DistributedFileSystem but got " + - fs.getClass().getName()); - } - final DistributedFileSystem dfs = (DistributedFileSystem) fs; - - // get conf settings - String xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath(); - String rsPrefix = RaidNode.rsDestinationPath(conf).toUri().getPath(); - if (!xorPrefix.endsWith("/")) { - xorPrefix = xorPrefix + "/"; - } - if (!rsPrefix.endsWith("/")) { - rsPrefix = rsPrefix + "/"; - } - LOG.debug("prefixes: " + xorPrefix + ", " + rsPrefix); - - // get a list of corrupted files (not considering parity blocks just yet) - // from the name node - // these are the only files we need to consider: - // if a file has no corrupted data blocks, it is OK even if some - // of its parity blocks are corrupted, so no further checking is - // necessary - final String[] files = RaidDFSUtil.getCorruptFiles(dfs); - final List corruptFileCandidates = new LinkedList(); - for (final String f: files) { - final Path p = new Path(f); - // if this file is a parity file - // or if it does not start with the specified path, - // ignore it - if (!p.toString().startsWith(xorPrefix) && - !p.toString().startsWith(rsPrefix) && - p.toString().startsWith(path)) { - corruptFileCandidates.add(p); - } - } - // filter files marked for deletion - RaidUtils.filterTrash(conf, corruptFileCandidates); - - int numberOfCorruptFiles = 0; - - for (final Path corruptFileCandidate: corruptFileCandidates) { - if (isFileCorrupt(dfs, corruptFileCandidate)) { - System.out.println(corruptFileCandidate.toString()); - numberOfCorruptFiles++; - } - } - - return numberOfCorruptFiles; - } - - /** - * main() has some simple utility methods - */ - public static void main(String argv[]) throws Exception { - RaidShell shell = null; - try { - shell = new RaidShell(new Configuration()); - int res = ToolRunner.run(shell, argv); - System.exit(res); - } catch (RPC.VersionMismatch v) { - System.err.println("Version Mismatch between client and server" + - "... command aborted."); - System.exit(-1); - } catch (IOException e) { - System.err. - println("Bad connection to RaidNode or NameNode. command aborted."); - System.err.println(e.getMessage()); - System.exit(-1); - } finally { - shell.close(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java deleted file mode 100644 index 7ce0c828039..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Pattern; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.Progressable; - -public class RaidUtils { - /** - * A {@link Progressable} that does nothing. - * - * We could have used Reporter.NULL here but that would introduce - * a dependency on mapreduce. - */ - public static class DummyProgressable implements Progressable { - /** - * Do nothing. - */ - @Override - public void progress() { - } - } - - /** - * Removes files matching the trash file pattern. - */ - public static void filterTrash(Configuration conf, List files) { - // Remove files under Trash. - String trashPattern = conf.get("raid.blockfixer.trash.pattern", - "^/user/.*/\\.Trash.*"); - for (Iterator it = files.iterator(); it.hasNext(); ) { - String pathStr = it.next().toString(); - if (Pattern.matches(trashPattern, pathStr)) { - it.remove(); - } - } - } - - public static void readTillEnd(InputStream in, byte[] buf, boolean eofOK) - throws IOException { - int toRead = buf.length; - int numRead = 0; - while (numRead < toRead) { - int nread = in.read(buf, numRead, toRead - numRead); - if (nread < 0) { - if (eofOK) { - // EOF hit, fill with zeros - Arrays.fill(buf, numRead, toRead, (byte)0); - numRead = toRead; - } else { - // EOF hit, throw. - throw new IOException("Premature EOF"); - } - } else { - numRead += nread; - } - } - } - - public static void copyBytes( - InputStream in, OutputStream out, byte[] buf, long count) - throws IOException { - for (long bytesRead = 0; bytesRead < count; ) { - int toRead = Math.min(buf.length, (int)(count - bytesRead)); - IOUtils.readFully(in, buf, 0, toRead); - bytesRead += toRead; - out.write(buf, 0, toRead); - } - } - - public static class ZeroInputStream extends InputStream - implements Seekable, PositionedReadable { - private long endOffset; - private long pos; - - public ZeroInputStream(long endOffset) { - this.endOffset = endOffset; - this.pos = 0; - } - - @Override - public int read() throws IOException { - if (pos < endOffset) { - pos++; - return 0; - } - return -1; - } - - @Override - public int available() throws IOException { - return (int)(endOffset - pos); - } - - @Override - public long getPos() throws IOException { - return pos; - } - - @Override - public void seek(long seekOffset) throws IOException { - if (seekOffset < endOffset) { - pos = seekOffset; - } else { - throw new IOException("Illegal Offset" + pos); - } - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - return false; - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - int count = 0; - for (; position < endOffset && count < length; position++) { - buffer[offset + count] = 0; - count++; - } - return count; - } - - @Override - public void readFully(long position, byte[] buffer, int offset, int length) - throws IOException { - int count = 0; - for (; position < endOffset && count < length; position++) { - buffer[offset + count] = 0; - count++; - } - if (count < length) { - throw new IOException("Premature EOF"); - } - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - readFully(position, buffer, 0, buffer.length); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java deleted file mode 100644 index 0a5d91bf4d7..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; -import java.util.Set; - - -public class ReedSolomonCode implements ErasureCode { - - private final int stripeSize; - private final int paritySize; - private final int[] generatingPolynomial; - private final int PRIMITIVE_ROOT = 2; - private final int[] primitivePower; - private final GaloisField GF = GaloisField.getInstance(); - private int[] errSignature; - private final int[] paritySymbolLocations; - private final int[] dataBuff; - - public ReedSolomonCode(int stripeSize, int paritySize) { - assert(stripeSize + paritySize < GF.getFieldSize()); - this.stripeSize = stripeSize; - this.paritySize = paritySize; - this.errSignature = new int[paritySize]; - this.paritySymbolLocations = new int[paritySize]; - this.dataBuff = new int[paritySize + stripeSize]; - for (int i = 0; i < paritySize; i++) { - paritySymbolLocations[i] = i; - } - - this.primitivePower = new int[stripeSize + paritySize]; - // compute powers of the primitive root - for (int i = 0; i < stripeSize + paritySize; i++) { - primitivePower[i] = GF.power(PRIMITIVE_ROOT, i); - } - // compute generating polynomial - int[] gen = {1}; - int[] poly = new int[2]; - for (int i = 0; i < paritySize; i++) { - poly[0] = primitivePower[i]; - poly[1] = 1; - gen = GF.multiply(gen, poly); - } - // generating polynomial has all generating roots - generatingPolynomial = gen; - } - - @Override - public void encode(int[] message, int[] parity) { - assert(message.length == stripeSize && parity.length == paritySize); - for (int i = 0; i < paritySize; i++) { - dataBuff[i] = 0; - } - for (int i = 0; i < stripeSize; i++) { - dataBuff[i + paritySize] = message[i]; - } - GF.remainder(dataBuff, generatingPolynomial); - for (int i = 0; i < paritySize; i++) { - parity[i] = dataBuff[i]; - } - } - - @Override - public void decode(int[] data, int[] erasedLocation, int[] erasedValue) { - if (erasedLocation.length == 0) { - return; - } - assert(erasedLocation.length == erasedValue.length); - for (int i = 0; i < erasedLocation.length; i++) { - data[erasedLocation[i]] = 0; - } - for (int i = 0; i < erasedLocation.length; i++) { - errSignature[i] = primitivePower[erasedLocation[i]]; - erasedValue[i] = GF.substitute(data, primitivePower[i]); - } - GF.solveVandermondeSystem(errSignature, erasedValue, erasedLocation.length); - } - - @Override - public int stripeSize() { - return this.stripeSize; - } - - @Override - public int paritySize() { - return this.paritySize; - } - - @Override - public int symbolSize() { - return (int) Math.round(Math.log(GF.getFieldSize()) / Math.log(2)); - } - - /** - * Given parity symbols followed by message symbols, return the locations of - * symbols that are corrupted. Can resolve up to (parity length / 2) error - * locations. - * @param data The message and parity. The parity should be placed in the - * first part of the array. In each integer, the relevant portion - * is present in the least significant bits of each int. - * The number of elements in data is stripeSize() + paritySize(). - * Note that data may be changed after calling this method. - * @param errorLocations The set to put the error location results - * @return true If the locations can be resolved, return true. - */ - public boolean computeErrorLocations(int[] data, - Set errorLocations) { - assert(data.length == paritySize + stripeSize && errorLocations != null); - errorLocations.clear(); - int maxError = paritySize / 2; - int[][] syndromeMatrix = new int[maxError][]; - for (int i = 0; i < syndromeMatrix.length; ++i) { - syndromeMatrix[i] = new int[maxError + 1]; - } - int[] syndrome = new int[paritySize]; - - if (computeSyndrome(data, syndrome)) { - // Parity check OK. No error location added. - return true; - } - for (int i = 0; i < maxError; ++i) { - for (int j = 0; j < maxError + 1; ++j) { - syndromeMatrix[i][j] = syndrome[i + j]; - } - } - GF.gaussianElimination(syndromeMatrix); - int[] polynomial = new int[maxError + 1]; - polynomial[0] = 1; - for (int i = 0; i < maxError; ++i) { - polynomial[i + 1] = syndromeMatrix[maxError - 1 - i][maxError]; - } - for (int i = 0; i < paritySize + stripeSize; ++i) { - int possibleRoot = GF.divide(1, primitivePower[i]); - if (GF.substitute(polynomial, possibleRoot) == 0) { - errorLocations.add(i); - } - } - // Now recover with error locations and check the syndrome again - int[] locations = new int[errorLocations.size()]; - int k = 0; - for (int loc : errorLocations) { - locations[k++] = loc; - } - int [] erasedValue = new int[locations.length]; - decode(data, locations, erasedValue); - for (int i = 0; i < locations.length; ++i) { - data[locations[i]] = erasedValue[i]; - } - return computeSyndrome(data, syndrome); - } - - /** - * Compute the syndrome of the input [parity, message] - * @param data [parity, message] - * @param syndrome The syndromes (checksums) of the data - * @return true If syndromes are all zeros - */ - private boolean computeSyndrome(int[] data, int [] syndrome) { - boolean corruptionFound = false; - for (int i = 0; i < paritySize; i++) { - syndrome[i] = GF.substitute(data, primitivePower[i]); - if (syndrome[i] != 0) { - corruptionFound = true; - } - } - return !corruptionFound; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java deleted file mode 100644 index 35eab7d6af3..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.OutputStream; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.BlockMissingException; - -public class ReedSolomonDecoder extends Decoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.ReedSolomonDecoder"); - private ErasureCode reedSolomonCode; - - public ReedSolomonDecoder( - Configuration conf, int stripeSize, int paritySize) { - super(conf, stripeSize, paritySize); - this.reedSolomonCode = new ReedSolomonCode(stripeSize, paritySize); - } - - @Override - protected void fixErasedBlock( - FileSystem fs, Path srcFile, - FileSystem parityFs, Path parityFile, - long blockSize, long errorOffset, long bytesToSkip, long limit, - OutputStream out) throws IOException { - FSDataInputStream[] inputs = new FSDataInputStream[stripeSize + paritySize]; - int[] erasedLocations = buildInputs(fs, srcFile, parityFs, parityFile, - errorOffset, inputs); - int blockIdxInStripe = ((int)(errorOffset/blockSize)) % stripeSize; - int erasedLocationToFix = paritySize + blockIdxInStripe; - writeFixedBlock(inputs, erasedLocations, erasedLocationToFix, - bytesToSkip, limit, out); - } - - protected int[] buildInputs(FileSystem fs, Path srcFile, - FileSystem parityFs, Path parityFile, - long errorOffset, FSDataInputStream[] inputs) - throws IOException { - LOG.info("Building inputs to recover block starting at " + errorOffset); - FileStatus srcStat = fs.getFileStatus(srcFile); - long blockSize = srcStat.getBlockSize(); - long blockIdx = (int)(errorOffset / blockSize); - long stripeIdx = blockIdx / stripeSize; - LOG.info("FileSize = " + srcStat.getLen() + ", blockSize = " + blockSize + - ", blockIdx = " + blockIdx + ", stripeIdx = " + stripeIdx); - ArrayList erasedLocations = new ArrayList(); - // First open streams to the parity blocks. - for (int i = 0; i < paritySize; i++) { - long offset = blockSize * (stripeIdx * paritySize + i); - FSDataInputStream in = parityFs.open( - parityFile, conf.getInt("io.file.buffer.size", 64 * 1024)); - in.seek(offset); - LOG.info("Adding " + parityFile + ":" + offset + " as input " + i); - inputs[i] = in; - } - // Now open streams to the data blocks. - for (int i = paritySize; i < paritySize + stripeSize; i++) { - long offset = blockSize * (stripeIdx * stripeSize + i - paritySize); - if (offset == errorOffset) { - LOG.info(srcFile + ":" + offset + - " is known to have error, adding zeros as input " + i); - inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream( - offset + blockSize)); - erasedLocations.add(i); - } else if (offset > srcStat.getLen()) { - LOG.info(srcFile + ":" + offset + - " is past file size, adding zeros as input " + i); - inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream( - offset + blockSize)); - } else { - FSDataInputStream in = fs.open( - srcFile, conf.getInt("io.file.buffer.size", 64 * 1024)); - in.seek(offset); - LOG.info("Adding " + srcFile + ":" + offset + " as input " + i); - inputs[i] = in; - } - } - if (erasedLocations.size() > paritySize) { - String msg = "Too many erased locations: " + erasedLocations.size(); - LOG.error(msg); - throw new IOException(msg); - } - int[] locs = new int[erasedLocations.size()]; - for (int i = 0; i < locs.length; i++) { - locs[i] = erasedLocations.get(i); - } - return locs; - } - - /** - * Decode the inputs provided and write to the output. - * @param inputs array of inputs. - * @param erasedLocations indexes in the inputs which are known to be erased. - * @param erasedLocationToFix index in the inputs which needs to be fixed. - * @param skipBytes number of bytes to skip before writing to output. - * @param limit maximum number of bytes to be written/skipped. - * @param out the output. - * @throws IOException - */ - void writeFixedBlock( - FSDataInputStream[] inputs, - int[] erasedLocations, - int erasedLocationToFix, - long skipBytes, - long limit, - OutputStream out) throws IOException { - - LOG.info("Need to write " + (limit - skipBytes) + - " bytes for erased location index " + erasedLocationToFix); - int[] tmp = new int[inputs.length]; - int[] decoded = new int[erasedLocations.length]; - long toDiscard = skipBytes; - // Loop while the number of skipped + written bytes is less than the max. - for (long written = 0; skipBytes + written < limit; ) { - erasedLocations = readFromInputs(inputs, erasedLocations, limit); - if (decoded.length != erasedLocations.length) { - decoded = new int[erasedLocations.length]; - } - - int toWrite = (int)Math.min((long)bufSize, limit - (skipBytes + written)); - if (toDiscard >= toWrite) { - toDiscard -= toWrite; - continue; - } - - // Decoded bufSize amount of data. - for (int i = 0; i < bufSize; i++) { - performDecode(readBufs, writeBufs, i, tmp, erasedLocations, decoded); - } - - for (int i = 0; i < erasedLocations.length; i++) { - if (erasedLocations[i] == erasedLocationToFix) { - toWrite -= toDiscard; - out.write(writeBufs[i], (int)toDiscard, toWrite); - toDiscard = 0; - written += toWrite; - LOG.debug("Wrote " + toWrite + " bytes for erased location index " + - erasedLocationToFix); - break; - } - } - } - } - - int[] readFromInputs( - FSDataInputStream[] inputs, - int[] erasedLocations, - long limit) throws IOException { - // For every input, read some data = bufSize. - for (int i = 0; i < inputs.length; i++) { - long curPos = inputs[i].getPos(); - try { - RaidUtils.readTillEnd(inputs[i], readBufs[i], true); - continue; - } catch (BlockMissingException e) { - LOG.error("Encountered BlockMissingException in stream " + i); - } catch (ChecksumException e) { - LOG.error("Encountered ChecksumException in stream " + i); - } - - // Found a new erased location. - if (erasedLocations.length == paritySize) { - String msg = "Too many read errors"; - LOG.error(msg); - throw new IOException(msg); - } - - // Add this stream to the set of erased locations. - int[] newErasedLocations = new int[erasedLocations.length + 1]; - for (int j = 0; j < erasedLocations.length; j++) { - newErasedLocations[j] = erasedLocations[j]; - } - newErasedLocations[newErasedLocations.length - 1] = i; - erasedLocations = newErasedLocations; - - LOG.info("Using zeros for stream " + i); - inputs[i] = new FSDataInputStream( - new RaidUtils.ZeroInputStream(curPos + limit)); - inputs[i].seek(curPos); - RaidUtils.readTillEnd(inputs[i], readBufs[i], true); - } - return erasedLocations; - } - - void performDecode(byte[][] readBufs, byte[][] writeBufs, - int idx, int[] inputs, - int[] erasedLocations, int[] decoded) { - for (int i = 0; i < decoded.length; i++) { - decoded[i] = 0; - } - for (int i = 0; i < inputs.length; i++) { - inputs[i] = readBufs[i][idx] & 0x000000FF; - } - reedSolomonCode.decode(inputs, erasedLocations, decoded); - for (int i = 0; i < decoded.length; i++) { - writeBufs[i][idx] = (byte)decoded[i]; - } - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java deleted file mode 100644 index 411978449d9..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Progressable; - -public class ReedSolomonEncoder extends Encoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.ReedSolomonEncoder"); - private ErasureCode reedSolomonCode; - - public ReedSolomonEncoder( - Configuration conf, int stripeSize, int paritySize) { - super(conf, stripeSize, paritySize); - this.reedSolomonCode = new ReedSolomonCode(stripeSize, paritySize); - } - - protected void encodeStripe( - InputStream[] blocks, - long stripeStartOffset, - long blockSize, - OutputStream[] outs, - Progressable reporter) throws IOException { - - int[] data = new int[stripeSize]; - int[] code = new int[paritySize]; - - for (long encoded = 0; encoded < blockSize; encoded += bufSize) { - // Read some data from each block = bufSize. - for (int i = 0; i < blocks.length; i++) { - RaidUtils.readTillEnd(blocks[i], readBufs[i], true); - } - - // Encode the data read. - for (int j = 0; j < bufSize; j++) { - performEncode(readBufs, writeBufs, j, data, code); - } - - // Now that we have some data to write, send it to the temp files. - for (int i = 0; i < paritySize; i++) { - outs[i].write(writeBufs[i], 0, bufSize); - } - - if (reporter != null) { - reporter.progress(); - } - } - } - - void performEncode(byte[][] readBufs, byte[][] writeBufs, int idx, - int[] data, int[] code) { - for (int i = 0; i < paritySize; i++) { - code[i] = 0; - } - for (int i = 0; i < stripeSize; i++) { - data[i] = readBufs[i][idx] & 0x000000FF; - } - reedSolomonCode.encode(data, code); - for (int i = 0; i < paritySize; i++) { - writeBufs[i][idx] = (byte)code[i]; - } - } - - @Override - public Path getParityTempPath() { - return new Path(RaidNode.rsTempPrefix(conf)); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java deleted file mode 100644 index 3dfe592f8ad..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.OutputStream; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; - -public class XORDecoder extends Decoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.XORDecoder"); - - public XORDecoder( - Configuration conf, int stripeSize) { - super(conf, stripeSize, 1); - } - - @Override - protected void fixErasedBlock( - FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile, - long blockSize, long errorOffset, long bytesToSkip, long limit, - OutputStream out) throws IOException { - LOG.info("Fixing block at " + srcFile + ":" + errorOffset + - ", skipping " + bytesToSkip + ", limit " + limit); - FileStatus srcStat = fs.getFileStatus(srcFile); - ArrayList xorinputs = new ArrayList(); - - FSDataInputStream parityFileIn = parityFs.open(parityFile); - parityFileIn.seek(parityOffset(errorOffset, blockSize)); - xorinputs.add(parityFileIn); - - long errorBlockOffset = (errorOffset / blockSize) * blockSize; - long[] srcOffsets = stripeOffsets(errorOffset, blockSize); - for (int i = 0; i < srcOffsets.length; i++) { - if (srcOffsets[i] == errorBlockOffset) { - LOG.info("Skipping block at " + srcFile + ":" + errorBlockOffset); - continue; - } - if (srcOffsets[i] < srcStat.getLen()) { - FSDataInputStream in = fs.open(srcFile); - in.seek(srcOffsets[i]); - xorinputs.add(in); - } - } - FSDataInputStream[] inputs = xorinputs.toArray( - new FSDataInputStream[]{null}); - ParityInputStream recovered = - new ParityInputStream(inputs, limit, readBufs[0], writeBufs[0]); - recovered.skip(bytesToSkip); - recovered.drain(out, null); - } - - protected long[] stripeOffsets(long errorOffset, long blockSize) { - long[] offsets = new long[stripeSize]; - long stripeIdx = errorOffset / (blockSize * stripeSize); - long startOffsetOfStripe = stripeIdx * stripeSize * blockSize; - for (int i = 0; i < stripeSize; i++) { - offsets[i] = startOffsetOfStripe + i * blockSize; - } - return offsets; - } - - protected long parityOffset(long errorOffset, long blockSize) { - long stripeIdx = errorOffset / (blockSize * stripeSize); - return stripeIdx * blockSize; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java deleted file mode 100644 index 469a1716e02..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import java.io.OutputStream; -import java.io.InputStream; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Progressable; - -public class XOREncoder extends Encoder { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.XOREncoder"); - public XOREncoder( - Configuration conf, int stripeSize) { - super(conf, stripeSize, 1); - } - - @Override - protected void encodeStripe( - InputStream[] blocks, - long stripeStartOffset, - long blockSize, - OutputStream[] outs, - Progressable reporter) throws IOException { - LOG.info("Peforming XOR "); - ParityInputStream parityIn = - new ParityInputStream(blocks, blockSize, readBufs[0], writeBufs[0]); - try { - parityIn.drain(outs[0], reporter); - } finally { - parityIn.close(); - } - } - - @Override - public Path getParityTempPath() { - return new Path(RaidNode.unraidTmpDirectory(conf)); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java deleted file mode 100644 index 2df302a905d..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid.protocol; - -import java.io.IOException; -import java.io.DataInput; -import java.io.DataOutput; -import java.util.Properties; -import java.util.Enumeration; -import java.lang.Math; -import java.text.SimpleDateFormat; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileStatus; - -/** - * Maintains information about one policy - */ -public class PolicyInfo implements Writable { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.protocol.PolicyInfo"); - protected static final SimpleDateFormat dateFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - - private Path srcPath; // the specified src path - private String policyName; // name of policy - private ErasureCodeType codeType;// the erasure code used - private String description; // A verbose description of this policy - private Configuration conf; // Hadoop configuration - - private Properties properties; // Policy-dependent properties - - private ReentrantReadWriteLock plock; // protects policy operations. - public static enum ErasureCodeType { - XOR, RS; - public static ErasureCodeType fromString(String s) { - if (XOR.toString().equalsIgnoreCase(s)) { - return XOR; - } - if (RS.toString().equalsIgnoreCase(s)) { - return RS; - } - return null; - } - } - - /** - * Create the empty object - */ - public PolicyInfo() { - this.conf = null; - this.policyName = ""; - this.description = ""; - this.srcPath = null; - this.properties = new Properties(); - this.plock = new ReentrantReadWriteLock(); - } - - /** - * Create the metadata that describes a policy - */ - public PolicyInfo(String policyName, Configuration conf) { - this.conf = conf; - this.policyName = policyName; - this.description = ""; - this.srcPath = null; - this.properties = new Properties(); - this.plock = new ReentrantReadWriteLock(); - } - - /** - * Copy fields from another PolicyInfo - */ - public void copyFrom(PolicyInfo other) { - if (other.conf != null) { - this.conf = other.conf; - } - if (other.policyName != null && other.policyName.length() > 0) { - this.policyName = other.policyName; - } - if (other.description != null && other.description.length() > 0) { - this.description = other.description; - } - if (other.codeType != null) { - this.codeType = other.codeType; - } - if (other.srcPath != null) { - this.srcPath = other.srcPath; - } - for (Object key : other.properties.keySet()) { - String skey = (String) key; - this.properties.setProperty(skey, other.properties.getProperty(skey)); - } - } - - /** - * Sets the input path on which this policy has to be applied - */ - public void setSrcPath(String in) throws IOException { - srcPath = new Path(in); - srcPath = srcPath.makeQualified(srcPath.getFileSystem(conf)); - } - - /** - * Set the erasure code type used in this policy - */ - public void setErasureCode(String code) { - this.codeType = ErasureCodeType.fromString(code); - } - - /** - * Set the description of this policy. - */ - public void setDescription(String des) { - this.description = des; - } - - /** - * Sets an internal property. - * @param name property name. - * @param value property value. - */ - public void setProperty(String name, String value) { - properties.setProperty(name, value); - } - - /** - * Returns the value of an internal property. - * @param name property name. - */ - public String getProperty(String name) { - return properties.getProperty(name); - } - - /** - * Get the name of this policy. - */ - public String getName() { - return this.policyName; - } - - /** - * Get the destination path of this policy. - */ - public ErasureCodeType getErasureCode() { - return this.codeType; - } - - /** - * Get the srcPath - */ - public Path getSrcPath() { - return srcPath; - } - - /** - * Get the expanded (unglobbed) forms of the srcPaths - */ - public Path[] getSrcPathExpanded() throws IOException { - FileSystem fs = srcPath.getFileSystem(conf); - - // globbing on srcPath - FileStatus[] gpaths = fs.globStatus(srcPath); - if (gpaths == null) { - return null; - } - Path[] values = new Path[gpaths.length]; - for (int i = 0; i < gpaths.length; i++) { - Path p = gpaths[i].getPath(); - values[i] = p.makeQualified(fs); - } - return values; - } - - /** - * Convert this policy into a printable form - */ - public String toString() { - StringBuffer buff = new StringBuffer(); - buff.append("Policy Name:\t" + policyName + " --------------------\n"); - buff.append("Source Path:\t" + srcPath + "\n"); - buff.append("Erasure Code:\t" + codeType + "\n"); - for (Enumeration e = properties.propertyNames(); e.hasMoreElements();) { - String name = (String) e.nextElement(); - buff.append( name + ":\t" + properties.getProperty(name) + "\n"); - } - if (description.length() > 0) { - int len = Math.min(description.length(), 80); - String sub = description.substring(0, len).trim(); - sub = sub.replaceAll("\n", " "); - buff.append("Description:\t" + sub + "...\n"); - } - return buff.toString(); - } - - ////////////////////////////////////////////////// - // Writable - ////////////////////////////////////////////////// - static { // register a ctor - WritableFactories.setFactory - (PolicyInfo.class, - new WritableFactory() { - public Writable newInstance() { return new PolicyInfo(); } - }); - } - - public void write(DataOutput out) throws IOException { - Text.writeString(out, srcPath.toString()); - Text.writeString(out, policyName); - Text.writeString(out, codeType.toString()); - Text.writeString(out, description); - out.writeInt(properties.size()); - for (Enumeration e = properties.propertyNames(); e.hasMoreElements();) { - String name = (String) e.nextElement(); - Text.writeString(out, name); - Text.writeString(out, properties.getProperty(name)); - } - } - - public void readFields(DataInput in) throws IOException { - this.srcPath = new Path(Text.readString(in)); - this.policyName = Text.readString(in); - this.codeType = ErasureCodeType.fromString(Text.readString(in)); - this.description = Text.readString(in); - for (int n = in.readInt(); n>0; n--) { - String name = Text.readString(in); - String value = Text.readString(in); - properties.setProperty(name,value); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java deleted file mode 100644 index eb3dc6e7449..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid.protocol; - -import java.io.IOException; -import java.io.DataInput; -import java.io.DataOutput; -import java.util.Collection; -import java.util.List; -import java.util.LinkedList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.conf.Configuration; - -/** - * Maintains informatiom about all policies that belong to a category. - * These policies have to be applied one-at-a-time and cannot be run - * simultaneously. - */ -public class PolicyList implements Writable { - public static final Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.protocol.PolicyList"); - - private List category; // list of policies - private Path srcPath; - - /** - * Create a new category of policies. - */ - public PolicyList() { - this.category = new LinkedList(); - this.srcPath = null; - } - - /** - * Add a new policy to this category. - */ - public void add(PolicyInfo info) { - category.add(info); - } - - public void setSrcPath(Configuration conf, String src) throws IOException { - srcPath = new Path(src); - srcPath = srcPath.makeQualified(srcPath.getFileSystem(conf)); - } - - public Path getSrcPath() { - return srcPath; - } - - /** - * Returns the policies in this category - */ - public Collection getAll() { - return category; - } - - ////////////////////////////////////////////////// - // Writable - ////////////////////////////////////////////////// - static { // register a ctor - WritableFactories.setFactory - (PolicyList.class, - new WritableFactory() { - public Writable newInstance() { return new PolicyList(); } - }); - } - - public void write(DataOutput out) throws IOException { - out.writeInt(category.size()); - for (PolicyInfo p : category) { - p.write(out); - } - } - - public void readFields(DataInput in) throws IOException { - int count = in.readInt(); - for (int i = 0; i < count; i++) { - PolicyInfo p = new PolicyInfo(); - p.readFields(in); - add(p); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java deleted file mode 100644 index 0dac296e188..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid.protocol; - -import java.util.Collection; -import java.io.IOException; - -import org.apache.hadoop.ipc.VersionedProtocol; -import org.apache.hadoop.fs.Path; - -/********************************************************************** - * RaidProtocol is used by user code - * {@link org.apache.hadoop.raid.RaidShell} class to communicate - * with the RaidNode. User code can manipulate the configured policies. - * - **********************************************************************/ -public interface RaidProtocol extends VersionedProtocol { - - /** - * Compared to the previous version the following changes have been introduced: - * Only the latest change is reflected. - * 1: new protocol introduced - */ - public static final long versionID = 1L; - - /** - * Get a listing of all configured policies - * @throws IOException - * return all categories of configured policies - */ - public PolicyList[] getAllPolicies() throws IOException; - - /** - * Unraid the specified input path. This is called when the specified file - * is corrupted. This call will move the specified file to file.old - * and then recover it from the RAID subsystem. - * - * @param inputPath The absolute pathname of the file to be recovered. - * @param corruptOffset The offset that has the corruption - */ - public String recoverFile(String inputPath, long corruptOffset) throws IOException; - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh deleted file mode 100644 index 3e9405eb72f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Start hadoop RaidNode process on machine specified on file conf/raidnode - -usage="Usage: start-raidnode-remote.sh" - -params=$# -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin" -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hadoop-config.sh - -# get arguments -if [ $# -ge 1 ]; then - echo $usage - exit -fi - -if [ -f "${HADOOP_CONF_DIR}/raidnode" ]; then - export HADOOP_SLAVES="${HADOOP_CONF_DIR}/raidnode" - echo "Starting raidnode at "`cat ${HADOOP_SLAVES}` - "$bin"/slaves.sh --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/start-raidnode.sh" -else - echo "No raidnode file in ${HADOOP_CONF_DIR}/raidnode" -fi - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh deleted file mode 100644 index d193bb1aeb6..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Start hadoop RaidNode process -# Run this on RaidNode machine - -usage="Usage: start-raidnode.sh" - -params=$# -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin" -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hadoop-config.sh - -# get arguments -if [ $# -ge 1 ]; then - echo $usage -fi - -if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then - . "${HADOOP_CONF_DIR}/hadoop-env.sh" -fi -export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_RAIDNODE_OPTS" - -"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start org.apache.hadoop.raid.RaidNode diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh deleted file mode 100644 index d185629b23b..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Stop hadoop RaidNode process on machine specified on file conf/raidnode - -usage="Usage: stop-raidnode-remote.sh" - -params=$# -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin" -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hadoop-config.sh - -# get arguments -if [ $# -ge 1 ]; then - echo $usage -fi - -if [ -f "${HADOOP_CONF_DIR}/raidnode" ]; then - export HADOOP_SLAVES="${HADOOP_CONF_DIR}/raidnode" - echo "Stopping raidnode at "`cat ${HADOOP_SLAVES}` - "$bin"/slaves.sh --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/stop-raidnode.sh" -else - echo "No raidnode file in ${HADOOP_CONF_DIR}/raidnode" -fi diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh b/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh deleted file mode 100644 index 02a61a87f7c..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Stop hadoop RaidNode process -# Run this on RaidNode machine. - -usage="Usage: stop-raidnode.sh" - -params=$# -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -DEFAULT_LIBEXEC_DIR="$bin" -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hadoop-config.sh - -# get arguments -if [ $# -ge 1 ]; then - echo $usage -fi - -export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_RAIDNODE_OPTS" - -"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop org.apache.hadoop.raid.RaidNode diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java deleted file mode 100644 index ae7d0293a7c..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java +++ /dev/null @@ -1,501 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.RandomAccessFile; -import java.net.URI; -import java.util.Random; -import java.util.regex.Pattern; -import java.util.zip.CRC32; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.raid.RaidNode; -import org.apache.hadoop.raid.RaidUtils; -import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType; -import org.apache.hadoop.util.StringUtils; -import org.junit.Test; - -public class TestRaidDfs { - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static String LOG_DIR = "target/raidlog"; - final static long RELOAD_INTERVAL = 1000; - final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidDfs"); - final static int NUM_DATANODES = 3; - - Configuration conf; - String namenode = null; - String hftp = null; - MiniDFSCluster dfs = null; - FileSystem fileSys = null; - String jobTrackerName = null; - ErasureCodeType code; - int stripeLength; - - private void mySetup( - String erasureCode, int rsParityLength) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - conf.set("fs.raid.recoverylogdir", LOG_DIR); - conf.setInt(RaidNode.RS_PARITY_LENGTH_KEY, rsParityLength); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // do not use map-reduce cluster for Raiding - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - - conf.set("raid.server.address", "localhost:0"); - conf.setInt("hdfs.raid.stripeLength", stripeLength); - conf.set("xor".equals(erasureCode) ? RaidNode.RAID_LOCATION_KEY : - RaidNode.RAIDRS_LOCATION_KEY, "/destraid"); - - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - } - - private void myTearDown() throws Exception { - if (dfs != null) { dfs.shutdown(); } - } - - private LocatedBlocks getBlockLocations(Path file, long length) - throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem) fileSys; - return RaidDFSUtil.getBlockLocations( - dfs, file.toUri().getPath(), 0, length); - } - - private LocatedBlocks getBlockLocations(Path file) - throws IOException { - FileStatus stat = fileSys.getFileStatus(file); - return getBlockLocations(file, stat.getLen()); - } - - private DistributedRaidFileSystem getRaidFS() throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - Configuration clientConf = new Configuration(conf); - clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); - clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); - clientConf.setBoolean("fs.hdfs.impl.disable.cache", true); - URI dfsUri = dfs.getUri(); - return (DistributedRaidFileSystem)FileSystem.get(dfsUri, clientConf); - } - - public static void waitForFileRaided( - Log logger, FileSystem fileSys, Path file, Path destPath) - throws IOException, InterruptedException { - FileStatus parityStat = null; - String fileName = file.getName().toString(); - // wait till file is raided - while (parityStat == null) { - logger.info("Waiting for files to be raided."); - try { - FileStatus[] listPaths = fileSys.listStatus(destPath); - if (listPaths != null) { - for (FileStatus f : listPaths) { - logger.info("File raided so far : " + f.getPath()); - String found = f.getPath().getName().toString(); - if (fileName.equals(found)) { - parityStat = f; - break; - } - } - } - } catch (FileNotFoundException e) { - //ignore - } - Thread.sleep(1000); // keep waiting - } - - while (true) { - LocatedBlocks locations = null; - DistributedFileSystem dfs = (DistributedFileSystem) fileSys; - locations = RaidDFSUtil.getBlockLocations( - dfs, file.toUri().getPath(), 0, parityStat.getLen()); - if (!locations.isUnderConstruction()) { - break; - } - Thread.sleep(1000); - } - - while (true) { - FileStatus stat = fileSys.getFileStatus(file); - if (stat.getReplication() == 1) break; - Thread.sleep(1000); - } - } - - private void corruptBlockAndValidate(Path srcFile, Path destPath, - int[] listBlockNumToCorrupt, long blockSize, int numBlocks) - throws IOException, InterruptedException { - int repl = 1; - long crc = createTestFilePartialLastBlock(fileSys, srcFile, repl, - numBlocks, blockSize); - long length = fileSys.getFileStatus(srcFile).getLen(); - - RaidNode.doRaid(conf, fileSys.getFileStatus(srcFile), - destPath, code, new RaidNode.Statistics(), new RaidUtils.DummyProgressable(), - false, repl, repl, stripeLength); - - // Delete first block of file - for (int blockNumToCorrupt : listBlockNumToCorrupt) { - LOG.info("Corrupt block " + blockNumToCorrupt + " of file " + srcFile); - LocatedBlocks locations = getBlockLocations(srcFile); - corruptBlock(dfs, srcFile, locations.get(blockNumToCorrupt).getBlock(), - NUM_DATANODES, true); - } - - // Validate - DistributedRaidFileSystem raidfs = getRaidFS(); - assertTrue(validateFile(raidfs, srcFile, length, crc)); - validateLogFile(getRaidFS(), new Path(LOG_DIR)); - } - - /** - * Create a file, corrupt several blocks in it and ensure that the file can be - * read through DistributedRaidFileSystem by ReedSolomon coding. - */ - @Test - public void testRaidDfsRs() throws Exception { - LOG.info("Test testRaidDfs started."); - - code = ErasureCodeType.RS; - long blockSize = 8192L; - int numBlocks = 8; - stripeLength = 3; - mySetup("rs", 3); - - int[][] corrupt = {{1, 2, 3}, {1, 4, 7}, {3, 6, 7}}; - try { - for (int i = 0; i < corrupt.length; i++) { - Path file = new Path("/user/dhruba/raidtest/file" + i); - corruptBlockAndValidate( - file, new Path("/destraid"), corrupt[i], blockSize, numBlocks); - } - } catch (Exception e) { - LOG.info("testRaidDfs Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test testRaidDfs completed."); - } - - /** - * Test DistributedRaidFileSystem.readFully() - */ - @Test - public void testReadFully() throws Exception { - code = ErasureCodeType.XOR; - stripeLength = 3; - mySetup("xor", 1); - - try { - Path file = new Path("/user/raid/raidtest/file1"); - long crc = createTestFile(fileSys, file, 1, 8, 8192L); - FileStatus stat = fileSys.getFileStatus(file); - LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); - - byte[] filebytes = new byte[(int)stat.getLen()]; - // Test that readFully returns the correct CRC when there are no errors. - DistributedRaidFileSystem raidfs = getRaidFS(); - FSDataInputStream stm = raidfs.open(file); - stm.readFully(0, filebytes); - assertEquals(crc, bufferCRC(filebytes)); - stm.close(); - - // Generate parity. - RaidNode.doRaid(conf, fileSys.getFileStatus(file), - new Path("/destraid"), code, new RaidNode.Statistics(), - new RaidUtils.DummyProgressable(), - false, 1, 1, stripeLength); - int[] corrupt = {0, 4, 7}; // first, last and middle block - for (int blockIdx : corrupt) { - LOG.info("Corrupt block " + blockIdx + " of file " + file); - LocatedBlocks locations = getBlockLocations(file); - corruptBlock(dfs, file, locations.get(blockIdx).getBlock(), - NUM_DATANODES, true); - } - // Test that readFully returns the correct CRC when there are errors. - stm = raidfs.open(file); - stm.readFully(0, filebytes); - assertEquals(crc, bufferCRC(filebytes)); - } finally { - myTearDown(); - } - } - - /** - * Test that access time and mtime of a source file do not change after - * raiding. - */ - @Test - public void testAccessTime() throws Exception { - LOG.info("Test testAccessTime started."); - - code = ErasureCodeType.XOR; - long blockSize = 8192L; - int numBlocks = 8; - int repl = 1; - stripeLength = 3; - mySetup("xor", 1); - - Path file = new Path("/user/dhruba/raidtest/file"); - createTestFilePartialLastBlock(fileSys, file, repl, numBlocks, blockSize); - FileStatus stat = fileSys.getFileStatus(file); - - try { - RaidNode.doRaid(conf, fileSys.getFileStatus(file), - new Path("/destraid"), code, new RaidNode.Statistics(), - new RaidUtils.DummyProgressable(), false, repl, repl, stripeLength); - - FileStatus newStat = fileSys.getFileStatus(file); - - assertEquals(stat.getModificationTime(), newStat.getModificationTime()); - assertEquals(stat.getAccessTime(), newStat.getAccessTime()); - } finally { - myTearDown(); - } - } - - /** - * Create a file, corrupt a block in it and ensure that the file can be - * read through DistributedRaidFileSystem by XOR code. - */ - @Test - public void testRaidDfsXor() throws Exception { - LOG.info("Test testRaidDfs started."); - - code = ErasureCodeType.XOR; - long blockSize = 8192L; - int numBlocks = 8; - stripeLength = 3; - mySetup("xor", 1); - - int[][] corrupt = {{0}, {4}, {7}}; // first, last and middle block - try { - for (int i = 0; i < corrupt.length; i++) { - Path file = new Path("/user/dhruba/raidtest/" + i); - corruptBlockAndValidate( - file, new Path("/destraid"), corrupt[i], blockSize, numBlocks); - } - } catch (Exception e) { - LOG.info("testRaidDfs Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test testRaidDfs completed."); - } - - // - // creates a file and populate it with random data. Returns its crc. - // - public static long createTestFile(FileSystem fileSys, Path name, int repl, - int numBlocks, long blocksize) - throws IOException { - CRC32 crc = new CRC32(); - Random rand = new Random(); - FSDataOutputStream stm = fileSys.create(name, true, - fileSys.getConf().getInt("io.file.buffer.size", 4096), - (short)repl, blocksize); - // fill random data into file - final byte[] b = new byte[(int)blocksize]; - for (int i = 0; i < numBlocks; i++) { - rand.nextBytes(b); - stm.write(b); - crc.update(b); - } - stm.close(); - return crc.getValue(); - } - - // - // Creates a file with partially full last block. Populate it with random - // data. Returns its crc. - // - public static long createTestFilePartialLastBlock( - FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) - throws IOException { - CRC32 crc = new CRC32(); - Random rand = new Random(); - FSDataOutputStream stm = fileSys.create(name, true, - fileSys.getConf().getInt("io.file.buffer.size", 4096), - (short)repl, blocksize); - // Write whole blocks. - byte[] b = new byte[(int)blocksize]; - for (int i = 1; i < numBlocks; i++) { - rand.nextBytes(b); - stm.write(b); - crc.update(b); - } - // Write partial block. - b = new byte[(int)blocksize/2 - 1]; - rand.nextBytes(b); - stm.write(b); - crc.update(b); - - stm.close(); - return crc.getValue(); - } - - static long bufferCRC(byte[] buf) { - CRC32 crc = new CRC32(); - crc.update(buf, 0, buf.length); - return crc.getValue(); - } - - // - // validates that file matches the crc. - // - public static boolean validateFile(FileSystem fileSys, Path name, long length, - long crc) - throws IOException { - - long numRead = 0; - CRC32 newcrc = new CRC32(); - FSDataInputStream stm = fileSys.open(name); - final byte[] b = new byte[4192]; - int num = 0; - while (num >= 0) { - num = stm.read(b); - if (num < 0) { - break; - } - numRead += num; - newcrc.update(b, 0, num); - } - stm.close(); - - if (numRead != length) { - LOG.info("Number of bytes read " + numRead + - " does not match file size " + length); - return false; - } - - LOG.info(" Newcrc " + newcrc.getValue() + " old crc " + crc); - if (newcrc.getValue() != crc) { - LOG.info("CRC mismatch of file " + name + ": " + newcrc + " vs. " + crc); - return false; - } - return true; - } - // - // validates the contents of raid recovery log file - // - public static void validateLogFile(FileSystem fileSys, Path logDir) - throws IOException { - FileStatus f = fileSys.listStatus(logDir)[0]; - FSDataInputStream stm = fileSys.open(f.getPath()); - try { - BufferedReader reader = new BufferedReader(new InputStreamReader(stm)); - assertEquals("Recovery attempt log", reader.readLine()); - assertTrue(Pattern.matches("Source path : /user/dhruba/raidtest/.*", - reader.readLine())); - assertTrue(Pattern.matches("Alternate path : .*/destraid", - reader.readLine())); - assertEquals("Stripe lentgh : 3", reader.readLine()); - assertTrue(Pattern.matches("Corrupt offset : \\d*", reader.readLine())); - assertTrue(Pattern.matches("Output from unRaid : " + - "hdfs://.*/tmp/raid/user/dhruba/raidtest/.*recovered", - reader.readLine())); - } finally { - stm.close(); - } - LOG.info("Raid HDFS Recovery log verified"); - } - - // - // Delete/Corrupt specified block of file - // - public static void corruptBlock(MiniDFSCluster dfs, Path file, ExtendedBlock blockNum, - int numDataNodes, boolean delete) throws IOException { - // Now deliberately remove/truncate replicas of blocks - int numDeleted = 0; - int numCorrupted = 0; - for (int i = 0; i < numDataNodes; i++) { - File block = MiniDFSCluster.getBlockFile(i, blockNum); - if (block == null || !block.exists()) { - continue; - } - if (delete) { - block.delete(); - LOG.info("Deleted block " + block); - numDeleted++; - } else { - // Corrupt - long seekPos = block.length()/2; - RandomAccessFile raf = new RandomAccessFile(block, "rw"); - raf.seek(seekPos); - int data = raf.readInt(); - raf.seek(seekPos); - raf.writeInt(data+1); - LOG.info("Corrupted block " + block); - numCorrupted++; - } - } - assertTrue("Nothing corrupted or deleted", - (numCorrupted + numDeleted) > 0); - } - - public static void corruptBlock(Path file, ExtendedBlock blockNum, - int numDataNodes, long offset) throws IOException { - // Now deliberately corrupt replicas of the the block. - for (int i = 0; i < numDataNodes; i++) { - File block = MiniDFSCluster.getBlockFile(i, blockNum); - if (block == null || !block.exists()) { - continue; - } - RandomAccessFile raf = new RandomAccessFile(block, "rw"); - raf.seek(offset); - int data = raf.readInt(); - raf.seek(offset); - raf.writeInt(data+1); - LOG.info("Corrupted block " + block); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java deleted file mode 100644 index 84cc534d215..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java +++ /dev/null @@ -1,518 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs.server.blockmanagement; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.CachedFullPathNames; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.CachedLocatedBlocks; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.FileType; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.INodeFile; -import org.apache.hadoop.hdfs.server.namenode.NameNodeRaidTestUtil; -import org.apache.hadoop.hdfs.server.namenode.NameNodeRaidUtil; -import org.apache.hadoop.net.NetworkTopology; -import org.apache.hadoop.raid.RaidNode; -import org.junit.Assert; -import org.junit.Test; - -public class TestBlockPlacementPolicyRaid { - private Configuration conf = null; - private MiniDFSCluster cluster = null; - private FSNamesystem namesystem = null; - private BlockManager blockManager; - private NetworkTopology networktopology; - private BlockPlacementPolicyRaid policy = null; - private FileSystem fs = null; - String[] rack1 = {"/rack1"}; - String[] rack2 = {"/rack2"}; - String[] host1 = {"host1.rack1.com"}; - String[] host2 = {"host2.rack2.com"}; - String xorPrefix = null; - String raidTempPrefix = null; - String raidrsTempPrefix = null; - String raidrsHarTempPrefix = null; - - final static Log LOG = - LogFactory.getLog(TestBlockPlacementPolicyRaid.class); - - protected void setupCluster() throws IOException { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); - conf.set("dfs.replication.pending.timeout.sec", "2"); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1L); - conf.set("dfs.block.replicator.classname", - BlockPlacementPolicyRaid.class.getName()); - conf.set(RaidNode.STRIPE_LENGTH_KEY, "2"); - conf.set(RaidNode.RS_PARITY_LENGTH_KEY, "3"); - conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); - // start the cluster with one datanode first - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1). - format(true).racks(rack1).hosts(host1).build(); - cluster.waitActive(); - namesystem = cluster.getNameNode().getNamesystem(); - blockManager = namesystem.getBlockManager(); - networktopology = blockManager.getDatanodeManager().getNetworkTopology(); - - Assert.assertTrue("BlockPlacementPolicy type is not correct.", - blockManager.getBlockPlacementPolicy() instanceof BlockPlacementPolicyRaid); - policy = (BlockPlacementPolicyRaid)blockManager.getBlockPlacementPolicy(); - fs = cluster.getFileSystem(); - xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath(); - raidTempPrefix = RaidNode.xorTempPrefix(conf); - raidrsTempPrefix = RaidNode.rsTempPrefix(conf); - raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf); - } - - /** - * Test that the parity files will be placed at the good locations when we - * create them. - */ - @Test - public void testChooseTargetForRaidFile() throws IOException { - setupCluster(); - try { - String src = "/dir/file"; - String parity = raidrsTempPrefix + src; - DFSTestUtil.createFile(fs, new Path(src), 4, (short)1, 0L); - DFSTestUtil.waitReplication(fs, new Path(src), (short)1); - refreshPolicy(); - setBlockPlacementPolicy(namesystem, policy); - // start 3 more datanodes - String[] racks = {"/rack2", "/rack2", "/rack2", - "/rack2", "/rack2", "/rack2"}; - String[] hosts = - {"host2.rack2.com", "host3.rack2.com", "host4.rack2.com", - "host5.rack2.com", "host6.rack2.com", "host7.rack2.com"}; - cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); - int numBlocks = 6; - DFSTestUtil.createFile(fs, new Path(parity), numBlocks, (short)2, 0L); - DFSTestUtil.waitReplication(fs, new Path(parity), (short)2); - FileStatus srcStat = fs.getFileStatus(new Path(src)); - BlockLocation[] srcLoc = - fs.getFileBlockLocations(srcStat, 0, srcStat.getLen()); - FileStatus parityStat = fs.getFileStatus(new Path(parity)); - BlockLocation[] parityLoc = - fs.getFileBlockLocations(parityStat, 0, parityStat.getLen()); - int parityLen = RaidNode.rsParityLength(conf); - for (int i = 0; i < numBlocks / parityLen; i++) { - Set locations = new HashSet(); - for (int j = 0; j < srcLoc.length; j++) { - String [] names = srcLoc[j].getNames(); - for (int k = 0; k < names.length; k++) { - LOG.info("Source block location: " + names[k]); - locations.add(names[k]); - } - } - for (int j = 0 ; j < parityLen; j++) { - String[] names = parityLoc[j + i * parityLen].getNames(); - for (int k = 0; k < names.length; k++) { - LOG.info("Parity block location: " + names[k]); - Assert.assertTrue(locations.add(names[k])); - } - } - } - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Test that the har parity files will be placed at the good locations when we - * create them. - */ - @Test - public void testChooseTargetForHarRaidFile() throws IOException { - setupCluster(); - try { - String[] racks = {"/rack2", "/rack2", "/rack2", - "/rack2", "/rack2", "/rack2"}; - String[] hosts = - {"host2.rack2.com", "host3.rack2.com", "host4.rack2.com", - "host5.rack2.com", "host6.rack2.com", "host7.rack2.com"}; - cluster.startDataNodes(conf, 6, true, null, racks, hosts, null); - String harParity = raidrsHarTempPrefix + "/dir/file"; - int numBlocks = 11; - DFSTestUtil.createFile(fs, new Path(harParity), numBlocks, (short)1, 0L); - DFSTestUtil.waitReplication(fs, new Path(harParity), (short)1); - FileStatus stat = fs.getFileStatus(new Path(harParity)); - BlockLocation[] loc = fs.getFileBlockLocations(stat, 0, stat.getLen()); - int rsParityLength = RaidNode.rsParityLength(conf); - for (int i = 0; i < numBlocks - rsParityLength; i++) { - Set locations = new HashSet(); - for (int j = 0; j < rsParityLength; j++) { - for (int k = 0; k < loc[i + j].getNames().length; k++) { - // verify that every adjacent 4 blocks are on differnt nodes - String name = loc[i + j].getNames()[k]; - LOG.info("Har Raid block location: " + name); - Assert.assertTrue(locations.add(name)); - } - } - } - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Test BlockPlacementPolicyRaid.CachedLocatedBlocks - * Verify that the results obtained from cache is the same as - * the results obtained directly - */ - @Test - public void testCachedBlocks() throws IOException { - setupCluster(); - try { - String file1 = "/dir/file1"; - String file2 = "/dir/file2"; - DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L); - DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L); - // test blocks cache - CachedLocatedBlocks cachedBlocks = new CachedLocatedBlocks(namesystem); - verifyCachedBlocksResult(cachedBlocks, namesystem, file1); - verifyCachedBlocksResult(cachedBlocks, namesystem, file1); - verifyCachedBlocksResult(cachedBlocks, namesystem, file2); - verifyCachedBlocksResult(cachedBlocks, namesystem, file2); - try { - Thread.sleep(1200L); - } catch (InterruptedException e) { - } - verifyCachedBlocksResult(cachedBlocks, namesystem, file2); - verifyCachedBlocksResult(cachedBlocks, namesystem, file1); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Test BlockPlacementPolicyRaid.CachedFullPathNames - * Verify that the results obtained from cache is the same as - * the results obtained directly - */ - @Test - public void testCachedPathNames() throws IOException { - setupCluster(); - try { - String file1 = "/dir/file1"; - String file2 = "/dir/file2"; - DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L); - DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L); - // test full path cache - CachedFullPathNames cachedFullPathNames = - new CachedFullPathNames(namesystem); - final BlockCollection[] bcs = NameNodeRaidTestUtil.getBlockCollections( - namesystem, file1, file2); - - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]); - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]); - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]); - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]); - try { - Thread.sleep(1200L); - } catch (InterruptedException e) { - } - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]); - verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - /** - * Test the result of getCompanionBlocks() on the unraided files - */ - @Test - public void testGetCompanionBLocks() throws IOException { - setupCluster(); - try { - String file1 = "/dir/file1"; - String file2 = "/raid/dir/file2"; - String file3 = "/raidrs/dir/file3"; - // Set the policy to default policy to place the block in the default way - setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( - conf, namesystem, networktopology)); - DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L); - DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L); - DFSTestUtil.createFile(fs, new Path(file3), 8, (short)1, 0L); - Collection companionBlocks; - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock()); - Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock()); - Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0); - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock()); - Assert.assertEquals(1, companionBlocks.size()); - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock()); - Assert.assertEquals(1, companionBlocks.size()); - - int rsParityLength = RaidNode.rsParityLength(conf); - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock()); - Assert.assertEquals(rsParityLength, companionBlocks.size()); - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock()); - Assert.assertEquals(rsParityLength, companionBlocks.size()); - - companionBlocks = getCompanionBlocks( - namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock()); - Assert.assertEquals(2, companionBlocks.size()); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - static void setBlockPlacementPolicy( - FSNamesystem namesystem, BlockPlacementPolicy policy) { - namesystem.writeLock(); - try { - namesystem.getBlockManager().setBlockPlacementPolicy(policy); - } finally { - namesystem.writeUnlock(); - } - } - - /** - * Test BlockPlacementPolicyRaid actually deletes the correct replica. - * Start 2 datanodes and create 1 source file and its parity file. - * 1) Start host1, create the parity file with replication 1 - * 2) Start host2, create the source file with replication 2 - * 3) Set repliation of source file to 1 - * Verify that the policy should delete the block with more companion blocks. - */ - @Test - public void testDeleteReplica() throws IOException { - setupCluster(); - try { - // Set the policy to default policy to place the block in the default way - setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault( - conf, namesystem, networktopology)); - DatanodeDescriptor datanode1 = blockManager.getDatanodeManager( - ).getDatanodeCyclicIteration("").iterator().next().getValue(); - String source = "/dir/file"; - String parity = xorPrefix + source; - - final Path parityPath = new Path(parity); - DFSTestUtil.createFile(fs, parityPath, 3, (short)1, 0L); - DFSTestUtil.waitReplication(fs, parityPath, (short)1); - - // start one more datanode - cluster.startDataNodes(conf, 1, true, null, rack2, host2, null); - DatanodeDescriptor datanode2 = null; - for(Map.Entry e : blockManager.getDatanodeManager( - ).getDatanodeCyclicIteration("")) { - final DatanodeDescriptor d = e.getValue(); - if (!d.getName().equals(datanode1.getName())) { - datanode2 = d; - } - } - Assert.assertTrue(datanode2 != null); - cluster.waitActive(); - final Path sourcePath = new Path(source); - DFSTestUtil.createFile(fs, sourcePath, 5, (short)2, 0L); - DFSTestUtil.waitReplication(fs, sourcePath, (short)2); - - refreshPolicy(); - Assert.assertEquals(parity, - policy.getParityFile(source)); - Assert.assertEquals(source, - policy.getSourceFile(parity, xorPrefix)); - - List sourceBlocks = getBlocks(namesystem, source); - List parityBlocks = getBlocks(namesystem, parity); - Assert.assertEquals(5, sourceBlocks.size()); - Assert.assertEquals(3, parityBlocks.size()); - - // verify the result of getCompanionBlocks() - Collection companionBlocks; - companionBlocks = getCompanionBlocks( - namesystem, policy, sourceBlocks.get(0).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{0, 1}, new int[]{0}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, sourceBlocks.get(1).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{0, 1}, new int[]{0}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, sourceBlocks.get(2).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{2, 3}, new int[]{1}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, sourceBlocks.get(3).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{2, 3}, new int[]{1}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, sourceBlocks.get(4).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{4}, new int[]{2}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, parityBlocks.get(0).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{0, 1}, new int[]{0}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, parityBlocks.get(1).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{2, 3}, new int[]{1}); - - companionBlocks = getCompanionBlocks( - namesystem, policy, parityBlocks.get(2).getBlock()); - verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks, - new int[]{4}, new int[]{2}); - - // Set the policy back to raid policy. We have to create a new object - // here to clear the block location cache - refreshPolicy(); - setBlockPlacementPolicy(namesystem, policy); - // verify policy deletes the correct blocks. companion blocks should be - // evenly distributed. - fs.setReplication(sourcePath, (short)1); - DFSTestUtil.waitReplication(fs, sourcePath, (short)1); - Map counters = new HashMap(); - refreshPolicy(); - for (int i = 0; i < parityBlocks.size(); i++) { - companionBlocks = getCompanionBlocks( - namesystem, policy, parityBlocks.get(i).getBlock()); - - counters = BlockPlacementPolicyRaid.countCompanionBlocks( - companionBlocks, false); - Assert.assertTrue(counters.get(datanode1.getName()) >= 1 && - counters.get(datanode1.getName()) <= 2); - Assert.assertTrue(counters.get(datanode1.getName()) + - counters.get(datanode2.getName()) == - companionBlocks.size()); - - counters = BlockPlacementPolicyRaid.countCompanionBlocks( - companionBlocks, true); - Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 && - counters.get(datanode1.getParent().getName()) <= 2); - Assert.assertTrue(counters.get(datanode1.getParent().getName()) + - counters.get(datanode2.getParent().getName()) == - companionBlocks.size()); - } - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - // create a new BlockPlacementPolicyRaid to clear the cache - private void refreshPolicy() { - policy = new BlockPlacementPolicyRaid(); - policy.initialize(conf, namesystem, networktopology); - } - - private void verifyCompanionBlocks(Collection companionBlocks, - List sourceBlocks, List parityBlocks, - int[] sourceBlockIndexes, int[] parityBlockIndexes) { - Set blockSet = new HashSet(); - for (LocatedBlock b : companionBlocks) { - blockSet.add(b.getBlock()); - } - Assert.assertEquals(sourceBlockIndexes.length + parityBlockIndexes.length, - blockSet.size()); - for (int index : sourceBlockIndexes) { - Assert.assertTrue(blockSet.contains(sourceBlocks.get(index).getBlock())); - } - for (int index : parityBlockIndexes) { - Assert.assertTrue(blockSet.contains(parityBlocks.get(index).getBlock())); - } - } - - private void verifyCachedFullPathNameResult( - CachedFullPathNames cachedFullPathNames, BlockCollection bc) - throws IOException { - String res1 = bc.getName(); - String res2 = cachedFullPathNames.get(bc); - LOG.info("Actual path name: " + res1); - LOG.info("Cached path name: " + res2); - Assert.assertEquals(cachedFullPathNames.get(bc), - bc.getName()); - } - - private void verifyCachedBlocksResult(CachedLocatedBlocks cachedBlocks, - FSNamesystem namesystem, String file) throws IOException{ - long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen(); - List res1 = NameNodeRaidUtil.getBlockLocations(namesystem, - file, 0L, len, false, false).getLocatedBlocks(); - List res2 = cachedBlocks.get(file); - for (int i = 0; i < res1.size(); i++) { - LOG.info("Actual block: " + res1.get(i).getBlock()); - LOG.info("Cached block: " + res2.get(i).getBlock()); - Assert.assertEquals(res1.get(i).getBlock(), res2.get(i).getBlock()); - } - } - - private Collection getCompanionBlocks( - FSNamesystem namesystem, BlockPlacementPolicyRaid policy, - ExtendedBlock block) throws IOException { - INodeFile inode = (INodeFile)blockManager.blocksMap.getBlockCollection(block - .getLocalBlock()); - FileType type = policy.getFileType(inode.getFullPathName()); - return policy.getCompanionBlocks(inode.getFullPathName(), type, - block.getLocalBlock()); - } - - private List getBlocks(FSNamesystem namesystem, String file) - throws IOException { - long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen(); - return NameNodeRaidUtil.getBlockLocations(namesystem, - file, 0, len, false, false).getLocatedBlocks(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java deleted file mode 100644 index 967df15632c..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import org.apache.hadoop.fs.UnresolvedLinkException; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; - -public class NameNodeRaidTestUtil { - public static BlockCollection[] getBlockCollections(final FSNamesystem namesystem, - final String... files) throws UnresolvedLinkException { - final BlockCollection[] inodes = new BlockCollection[files.length]; - final FSDirectory dir = namesystem.dir; - dir.readLock(); - try { - for(int i = 0; i < files.length; i++) { - inodes[i] = (BlockCollection)dir.rootDir.getNode(files[i], true); - } - return inodes; - } finally { - dir.readUnlock(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java deleted file mode 100644 index 960d11e30df..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java +++ /dev/null @@ -1,671 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.URI; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; -import java.util.zip.CRC32; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.util.JarFinder; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; -import org.junit.Test; - - -public class TestBlockFixer { - final static Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.TestBlockFixer"); - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static String CONFIG_FILE = new File(TEST_DIR, - "test-raid.xml").getAbsolutePath(); - public static final String DistBlockFixer_JAR = - JarFinder.getJar(DistBlockFixer.class); - final static long RELOAD_INTERVAL = 1000; - final static int NUM_DATANODES = 3; - Configuration conf; - String namenode = null; - MiniDFSCluster dfs = null; - String hftp = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - RaidNode cnode = null; - String jobTrackerName = null; - Random rand = new Random(); - - /** - * Tests isXorParityFile and isRsParityFile - */ - @Test - public void testIsParityFile() throws IOException { - Configuration testConf = new Configuration(); - testConf.set("hdfs.raid.locations", "/raid"); - testConf.set("hdfs.raidrs.locations", "/raidrs"); - - BlockFixer.BlockFixerHelper helper = - new BlockFixer.BlockFixerHelper(testConf); - - assertFalse("incorrectly identified rs parity file as xor parity file", - helper.isXorParityFile(new Path("/raidrs/test/test"))); - assertTrue("could not identify rs parity file", - helper.isRsParityFile(new Path("/raidrs/test/test"))); - assertTrue("could not identify xor parity file", - helper.isXorParityFile(new Path("/raid/test/test"))); - assertFalse("incorrectly identified xor parity file as rs parity file", - helper.isRsParityFile(new Path("/raid/test/test"))); - } - - - /** - * Test the filtering of trash files from the list of corrupt files. - */ - @Test - public void testTrashFilter() { - List files = new LinkedList(); - // Paths that do not match the trash pattern. - Path p1 = new Path("/user/raid/raidtest/f1"); - Path p2 = new Path("/user/.Trash/"); - // Paths that match the trash pattern. - Path p3 = new Path("/user/raid/.Trash/raidtest/f1"); - Path p4 = new Path("/user/raid/.Trash/"); - files.add(p1); - files.add(p3); - files.add(p4); - files.add(p2); - - Configuration conf = new Configuration(); - RaidUtils.filterTrash(conf, files); - - assertEquals("expected 2 non-trash files but got " + files.size(), - 2, files.size()); - for (Path p: files) { - assertTrue("wrong file returned by filterTrash", - p == p1 || p == p2); - } - } - - @Test - public void testBlockFixLocal() throws Exception { - implBlockFix(true); - } - - /** - * Create a file with three stripes, corrupt a block each in two stripes, - * and wait for the the file to be fixed. - */ - protected void implBlockFix(boolean local) throws Exception { - LOG.info("Test testBlockFix started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); // never har - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 7, blockSize); - long file1Len = fileSys.getFileStatus(file1).getLen(); - LOG.info("Test testBlockFix created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - if (local) { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - } else { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - } - localConf.setLong("raid.blockfix.filespertask", 2L); - - try { - cnode = RaidNode.createRaidNode(null, localConf); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - cnode.stop(); cnode.join(); - - FileStatus srcStat = fileSys.getFileStatus(file1); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks locs = RaidDFSUtil.getBlockLocations( - dfs, file1.toUri().getPath(), 0, srcStat.getLen()); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - // Corrupt blocks in two different stripes. We can fix them. - int[] corruptBlockIdxs = new int[]{0, 4, 6}; - for (int idx: corruptBlockIdxs) - corruptBlock(locs.get(idx).getBlock()); - reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize); - - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("file not corrupted", 1, corruptFiles.length); - assertEquals("wrong file corrupted", - corruptFiles[0], file1.toUri().getPath()); - assertEquals("wrong number of corrupt blocks", 3, - RaidDFSUtil.corruptBlocksInFile(dfs, file1.toUri().getPath(), 0, - srcStat.getLen()).size()); - - cnode = RaidNode.createRaidNode(null, localConf); - long start = Time.now(); - while (cnode.blockFixer.filesFixed() < 1 && - Time.now() - start < 120000) { - LOG.info("Test testBlockFix waiting for files to be fixed."); - Thread.sleep(1000); - } - assertEquals("file not fixed", 1, cnode.blockFixer.filesFixed()); - - dfs = getDFS(conf, dfs); - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file1, file1Len, crc1)); - - } catch (Exception e) { - LOG.info("Test testBlockFix Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test testBlockFix completed."); - } - - /** - * Tests integrity of generated block. - * Create a file and delete a block entirely. Wait for the block to be - * regenerated. Now stop RaidNode and corrupt the generated block. - * Test that corruption in the generated block can be detected by clients. - */ - protected void generatedBlockTestCommon(String testName, int blockToCorrupt, - boolean local) throws Exception { - LOG.info("Test " + testName + " started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); // never har - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - long crc1 = TestRaidDfs.createTestFile(fileSys, file1, 1, 7, blockSize); - long file1Len = fileSys.getFileStatus(file1).getLen(); - LOG.info("Test " + testName + " created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - if (local) { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - } else { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - } - localConf.setLong("raid.blockfix.filespertask", 2L); - try { - cnode = RaidNode.createRaidNode(null, localConf); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - cnode.stop(); cnode.join(); - - FileStatus srcStat = fileSys.getFileStatus(file1); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks locs = RaidDFSUtil.getBlockLocations( - dfs, file1.toUri().getPath(), 0, srcStat.getLen()); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - corruptBlock(locs.get(0).getBlock()); - reportCorruptBlocks(dfs, file1, new int[]{0}, blockSize); - - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("file not corrupted", - 1, corruptFiles.length); - assertEquals("wrong file corrupted", - corruptFiles[0], file1.toUri().getPath()); - - cnode = RaidNode.createRaidNode(null, localConf); - long start = Time.now(); - while (cnode.blockFixer.filesFixed() < 1 && - Time.now() - start < 120000) { - LOG.info("Test " + testName + " waiting for files to be fixed."); - Thread.sleep(1000); - } - assertEquals("file not fixed", - 1, cnode.blockFixer.filesFixed()); - - // Stop RaidNode - cnode.stop(); cnode.join(); cnode = null; - - // The block has successfully been reconstructed. - dfs = getDFS(conf, dfs); - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file1, file1Len, crc1)); - - // Now corrupt the generated block. - locs = RaidDFSUtil.getBlockLocations( - dfs, file1.toUri().getPath(), 0, srcStat.getLen()); - corruptBlock(locs.get(0).getBlock()); - reportCorruptBlocks(dfs, file1, new int[]{0}, blockSize); - - try { - Thread.sleep(5*1000); - } catch (InterruptedException ignore) { - } - try { - TestRaidDfs.validateFile(dfs, file1, file1Len, crc1); - fail("Expected exception not thrown"); - } catch (org.apache.hadoop.fs.ChecksumException ce) { - } catch (org.apache.hadoop.hdfs.BlockMissingException bme) { - } - } catch (Exception e) { - LOG.info("Test " + testName + " Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test " + testName + " completed."); - } - - /** - * Tests integrity of generated block. - * Create a file and delete a block entirely. Wait for the block to be - * regenerated. Now stop RaidNode and corrupt the generated block. - * Test that corruption in the generated block can be detected by clients. - */ - @Test - public void testGeneratedBlockLocal() throws Exception { - generatedBlockTestCommon("testGeneratedBlock", 3, true); - } - - /** - * Tests integrity of generated last block. - * Create a file and delete a block entirely. Wait for the block to be - * regenerated. Now stop RaidNode and corrupt the generated block. - * Test that corruption in the generated block can be detected by clients. - */ - @Test - public void testGeneratedLastBlockLocal() throws Exception { - generatedBlockTestCommon("testGeneratedLastBlock", 6, true); - } - - @Test - public void testParityBlockFixLocal() throws Exception { - implParityBlockFix("testParityBlockFixLocal", true); - } - - /** - * Corrupt a parity file and wait for it to get fixed. - */ - protected void implParityBlockFix(String testName, boolean local) - throws Exception { - LOG.info("Test " + testName + " started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); // never har - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - Path parityFile = new Path("/destraid/user/dhruba/raidtest/file1"); - TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 7, blockSize); - LOG.info("Test " + testName + " created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - if (local) { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - } else { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - } - localConf.setLong("raid.blockfix.filespertask", 2L); - - try { - cnode = RaidNode.createRaidNode(null, localConf); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - cnode.stop(); cnode.join(); - - long parityCRC = getCRC(fileSys, parityFile); - - FileStatus parityStat = fileSys.getFileStatus(parityFile); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks locs = RaidDFSUtil.getBlockLocations( - dfs, parityFile.toUri().getPath(), 0, parityStat.getLen()); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - // Corrupt parity blocks for different stripes. - int[] corruptBlockIdxs = new int[]{0, 1, 2}; - for (int idx: corruptBlockIdxs) - corruptBlock(locs.get(idx).getBlock()); - reportCorruptBlocks(dfs, parityFile, corruptBlockIdxs, blockSize); - - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("file not corrupted", - 1, corruptFiles.length); - assertEquals("wrong file corrupted", - corruptFiles[0], parityFile.toUri().getPath()); - - cnode = RaidNode.createRaidNode(null, localConf); - long start = Time.now(); - while (cnode.blockFixer.filesFixed() < 1 && - Time.now() - start < 120000) { - LOG.info("Test " + testName + " waiting for files to be fixed."); - Thread.sleep(1000); - } - assertEquals("file not fixed", - 1, cnode.blockFixer.filesFixed()); - - long checkCRC = getCRC(fileSys, parityFile); - - assertEquals("file not fixed", - parityCRC, checkCRC); - - } catch (Exception e) { - LOG.info("Test " + testName + " Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test " + testName + " completed."); - } - - @Test - public void testParityHarBlockFixLocal() throws Exception { - implParityHarBlockFix("testParityHarBlockFixLocal", true); - } - - protected void implParityHarBlockFix(String testName, boolean local) - throws Exception { - LOG.info("Test " + testName + " started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, 0); // Time before har = 0 days. - Path file1 = new Path("/user/dhruba/raidtest/file1"); - // Parity file will have 7 blocks. - TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 20, blockSize); - LOG.info("Test " + testName + " created test files"); - - // create an instance of the RaidNode - // HAR block size = 2 * src block size = 2 * parity block size. - Configuration localConf = new Configuration(conf); - localConf.setLong("har.block.size", blockSize * 2); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - if (local) { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - } else { - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - } - localConf.setLong("raid.blockfix.filespertask", 2L); - - try { - cnode = RaidNode.createRaidNode(null, localConf); - Path harDirectory = - new Path("/destraid/user/dhruba/raidtest/raidtest" + - RaidNode.HAR_SUFFIX); - long start = Time.now(); - while (Time.now() - start < 1000 * 120) { - if (fileSys.exists(harDirectory)) { - break; - } - LOG.info("Test " + testName + " waiting for har"); - Thread.sleep(1000); - } - - Path partFile = new Path(harDirectory, "part-0"); - long partCRC = getCRC(fileSys, partFile); - FileStatus partStat = fileSys.getFileStatus(partFile); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks locs = RaidDFSUtil.getBlockLocations( - dfs, partFile.toUri().getPath(), 0, partStat.getLen()); - // 7 parity blocks => 4 har blocks. - assertEquals("wrong number of har blocks", - 4, locs.getLocatedBlocks().size()); - cnode.stop(); cnode.join(); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - // Corrupt parity blocks for different stripes. - int[] corruptBlockIdxs = new int[]{0, 3}; - for (int idx: corruptBlockIdxs) - corruptBlock(locs.get(idx).getBlock()); - reportCorruptBlocks(dfs, partFile, corruptBlockIdxs, - partStat.getBlockSize()); - - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("file not corrupted", 1, corruptFiles.length); - assertEquals("wrong file corrupted", - corruptFiles[0], partFile.toUri().getPath()); - - cnode = RaidNode.createRaidNode(null, localConf); - start = Time.now(); - while (cnode.blockFixer.filesFixed() < 1 && - Time.now() - start < 120000) { - LOG.info("Test " + testName + " waiting for files to be fixed."); - Thread.sleep(1000); - } - assertEquals("file not fixed", - 1, cnode.blockFixer.filesFixed()); - - long checkCRC = getCRC(fileSys, partFile); - - assertEquals("file not fixed", - partCRC, checkCRC); - } catch (Exception e) { - LOG.info("Test " + testName + " Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test " + testName + " completed."); - } - - - protected static DistributedFileSystem getDFS( - Configuration conf, FileSystem dfs) throws IOException { - Configuration clientConf = new Configuration(conf); - clientConf.set("fs.hdfs.impl", - "org.apache.hadoop.hdfs.DistributedFileSystem"); - clientConf.setBoolean("fs.hdfs.impl.disable.cache", true); - URI dfsUri = dfs.getUri(); - FileSystem.closeAll(); - return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf); - } - - protected void mySetup(int stripeLength, int timeBeforeHar) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - conf.set("raid.config.file", CONFIG_FILE); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // do not use map-reduce cluster for Raiding - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - conf.set("raid.server.address", "localhost:0"); - conf.setInt("hdfs.raid.stripeLength", stripeLength); - conf.set("hdfs.raid.locations", "/destraid"); - - conf.setBoolean("dfs.permissions", false); - - conf.set("mapreduce.framework.name", "yarn"); - - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - - FileSystem.setDefaultUri(conf, namenode); - mr = new MiniMRCluster(4, namenode, 3); - JobConf jobConf = mr.createJobConf(); - jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - conf.set("mapred.job.tracker", jobTrackerName); - conf.set("mapreduce.framework.name", "yarn"); - String rmAdress = jobConf.get("yarn.resourcemanager.address"); - if (rmAdress != null) { - conf.set("yarn.resourcemanager.address", rmAdress); - } - String schedulerAdress = - jobConf.get("yarn.resourcemanager.scheduler.address"); - if (schedulerAdress != null) { - conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); - } - String jobHistoryAddress = - jobConf.get("mapreduce.jobhistory.address"); - if (jobHistoryAddress != null) { - conf.set("mapreduce.jobhistory.address", jobHistoryAddress); - } - conf.set(JobContext.JAR, TestBlockFixer.DistBlockFixer_JAR); - - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - String str = " " + - " " + - " " + - "xor " + - " /destraid " + - " " + - "targetReplication " + - "1 " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "1 " + - " replication factor of parity file" + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " "; - if (timeBeforeHar >= 0) { - str += - " " + - "time_before_har " + - "" + timeBeforeHar + " " + - " amount of time waited before har'ing parity files" + - " " + - " "; - } - - str += - "" + - "" + - ""; - fileWriter.write(str); - fileWriter.close(); - } - - protected void myTearDown() throws Exception { - if (cnode != null) { cnode.stop(); cnode.join(); } - if (mr != null) { mr.shutdown(); } - if (dfs != null) { dfs.shutdown(); } - } - - public static long getCRC(FileSystem fs, Path p) throws IOException { - CRC32 crc = new CRC32(); - FSDataInputStream stm = fs.open(p); - int b; - while ((b = stm.read())>=0) { - crc.update(b); - } - stm.close(); - return crc.getValue(); - } - - void corruptBlock(ExtendedBlock block) throws IOException { - assertTrue("Could not corrupt block", - dfs.corruptBlockOnDataNodes(block) > 0); - } - - static void reportCorruptBlocks(FileSystem fs, Path file, int[] idxs, - long blockSize) throws IOException { - - FSDataInputStream in = fs.open(file); - for (int idx: idxs) { - long offset = idx * blockSize; - LOG.info("Reporting corrupt block " + file + ":" + offset); - in.seek(offset); - try { - in.readFully(new byte[(int)blockSize]); - fail("Expected exception not thrown for " + file + ":" + offset); - } catch (org.apache.hadoop.fs.ChecksumException e) { - } catch (org.apache.hadoop.hdfs.BlockMissingException bme) { - } - } - } -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java deleted file mode 100644 index 6eeba5be4f5..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; -import org.junit.Test; - -public class TestBlockFixerBlockFixDist extends TestBlockFixer { - @Test - public void testBlockFixDist() throws Exception { - implBlockFix(false); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java deleted file mode 100644 index c4257722743..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; -import org.junit.Test; - -public class TestBlockFixerDistConcurrency extends TestBlockFixer { - /** - * tests that we can have 2 concurrent jobs fixing files - * (dist block fixer) - */ - @Test - public void testConcurrentJobs() throws Exception { - LOG.info("Test testConcurrentJobs started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); // never har - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path file2 = new Path("/user/dhruba/raidtest/file2"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 20, blockSize); - long crc2 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file2, - 1, 20, blockSize); - long file1Len = fileSys.getFileStatus(file1).getLen(); - long file2Len = fileSys.getFileStatus(file2).getLen(); - LOG.info("Test testConcurrentJobs created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - localConf.setLong("raid.blockfix.filespertask", 2L); - - try { - cnode = RaidNode.createRaidNode(null, localConf); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file2, destPath); - cnode.stop(); cnode.join(); - - FileStatus file1Stat = fileSys.getFileStatus(file1); - FileStatus file2Stat = fileSys.getFileStatus(file2); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks file1Loc = - RaidDFSUtil.getBlockLocations(dfs, file1.toUri().getPath(), - 0, file1Stat.getLen()); - LocatedBlocks file2Loc = - RaidDFSUtil.getBlockLocations(dfs, file2.toUri().getPath(), - 0, file2Stat.getLen()); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - // corrupt file1 - int[] corruptBlockIdxs = new int[]{0, 4, 6}; - for (int idx: corruptBlockIdxs) - corruptBlock(file1Loc.get(idx).getBlock()); - reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize); - - cnode = RaidNode.createRaidNode(null, localConf); - DistBlockFixer blockFixer = (DistBlockFixer) cnode.blockFixer; - long start = Time.now(); - - while (blockFixer.jobsRunning() < 1 && - Time.now() - start < 240000) { - LOG.info("Test testBlockFix waiting for fixing job 1 to start"); - Thread.sleep(10); - } - assertEquals("job 1 not running", 1, blockFixer.jobsRunning()); - - // corrupt file2 - for (int idx: corruptBlockIdxs) - corruptBlock(file2Loc.get(idx).getBlock()); - reportCorruptBlocks(dfs, file2, corruptBlockIdxs, blockSize); - - while (blockFixer.jobsRunning() < 2 && - Time.now() - start < 240000) { - LOG.info("Test testBlockFix waiting for fixing job 2 to start"); - Thread.sleep(10); - } - assertEquals("2 jobs not running", 2, blockFixer.jobsRunning()); - - while (blockFixer.filesFixed() < 2 && - Time.now() - start < 240000) { - LOG.info("Test testBlockFix waiting for files to be fixed."); - Thread.sleep(10); - } - assertEquals("files not fixed", 2, blockFixer.filesFixed()); - - dfs = getDFS(conf, dfs); - - try { - Thread.sleep(5*1000); - } catch (InterruptedException ignore) { - } - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file1, file1Len, crc1)); - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file2, file2Len, crc2)); - } catch (Exception e) { - LOG.info("Test testConcurrentJobs exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - - } - - /** - * tests that the distributed block fixer obeys - * the limit on how many files to fix simultaneously - */ - @Test - public void testMaxPendingFiles() throws Exception { - LOG.info("Test testMaxPendingFiles started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); // never har - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path file2 = new Path("/user/dhruba/raidtest/file2"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 20, blockSize); - long crc2 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file2, - 1, 20, blockSize); - long file1Len = fileSys.getFileStatus(file1).getLen(); - long file2Len = fileSys.getFileStatus(file2).getLen(); - LOG.info("Test testMaxPendingFiles created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - localConf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.DistBlockFixer"); - localConf.setLong("raid.blockfix.filespertask", 2L); - localConf.setLong("raid.blockfix.maxpendingfiles", 1L); - - try { - cnode = RaidNode.createRaidNode(null, localConf); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file2, destPath); - cnode.stop(); cnode.join(); - - FileStatus file1Stat = fileSys.getFileStatus(file1); - FileStatus file2Stat = fileSys.getFileStatus(file2); - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - LocatedBlocks file1Loc = - RaidDFSUtil.getBlockLocations(dfs, file1.toUri().getPath(), - 0, file1Stat.getLen()); - LocatedBlocks file2Loc = - RaidDFSUtil.getBlockLocations(dfs, file2.toUri().getPath(), - 0, file2Stat.getLen()); - - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals("no corrupt files expected", 0, corruptFiles.length); - assertEquals("filesFixed() should return 0 before fixing files", - 0, cnode.blockFixer.filesFixed()); - - // corrupt file1 - int[] corruptBlockIdxs = new int[]{0, 4, 6}; - for (int idx: corruptBlockIdxs) - corruptBlock(file1Loc.get(idx).getBlock()); - reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize); - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - - cnode = RaidNode.createRaidNode(null, localConf); - DistBlockFixer blockFixer = (DistBlockFixer) cnode.blockFixer; - long start = Time.now(); - - while (blockFixer.jobsRunning() < 1 && - Time.now() - start < 240000) { - LOG.info("Test testBlockFix waiting for fixing job 1 to start"); - Thread.sleep(10); - } - assertEquals("job not running", 1, blockFixer.jobsRunning()); - - // corrupt file2 - for (int idx: corruptBlockIdxs) - corruptBlock(file2Loc.get(idx).getBlock()); - reportCorruptBlocks(dfs, file2, corruptBlockIdxs, blockSize); - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - - // wait until both files are fixed - while (blockFixer.filesFixed() < 2 && - Time.now() - start < 240000) { - // make sure the block fixer does not start a second job while - // the first one is still running - assertTrue("too many jobs running", blockFixer.jobsRunning() <= 1); - Thread.sleep(10); - } - assertEquals("files not fixed", 2, blockFixer.filesFixed()); - - dfs = getDFS(conf, dfs); - - try { - Thread.sleep(5*1000); - } catch (InterruptedException ignore) { - } - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file1, file1Len, crc1)); - assertTrue("file not fixed", - TestRaidDfs.validateFile(dfs, file2, file2Len, crc2)); - } catch (Exception e) { - LOG.info("Test testMaxPendingFiles exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java deleted file mode 100644 index bb061a004b6..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import org.junit.Test; - -public class TestBlockFixerGeneratedBlockDist extends TestBlockFixer { - /** - * Tests integrity of generated block. - * Create a file and delete a block entirely. Wait for the block to be - * regenerated. Now stop RaidNode and corrupt the generated block. - * Test that corruption in the generated block can be detected by clients. - */ - @Test - public void testGeneratedBlockDist() throws Exception { - generatedBlockTestCommon("testGeneratedBlock", 3, false); - } - - /** - * Tests integrity of generated last block. - * Create a file and delete a block entirely. Wait for the block to be - * regenerated. Now stop RaidNode and corrupt the generated block. - * Test that corruption in the generated block can be detected by clients. - */ - @Test - public void testGeneratedLastBlockDist() throws Exception { - generatedBlockTestCommon("testGeneratedLastBlock", 6, false); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java deleted file mode 100644 index 91d2ae54c29..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import org.junit.Test; - -public class TestBlockFixerParityBlockFixDist extends TestBlockFixer { - @Test - public void testParityBlockFixDist() throws Exception { - implParityBlockFix("testParityBlockFixDist", false); - } - - @Test - public void testParityHarBlockFixDist() throws Exception { - implParityHarBlockFix("testParityHarBlockFixDist", false); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java deleted file mode 100644 index 89cb6334fe0..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.util.Time; -import org.junit.Test; - -public class TestDirectoryTraversal { - final static Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.TestDirectoryTraversal"); - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - - MiniDFSCluster dfs = null; - FileSystem fs = null; - Configuration conf = null; - - /** - * Test basic enumeration. - */ - @Test - public void testEnumeration() throws IOException { - mySetup(); - - try { - Path topDir = new Path(TEST_DIR + "/testenumeration"); - - createTestTree(topDir); - - LOG.info("Enumerating files"); - List startPaths = new LinkedList(); - startPaths.add(fs.getFileStatus(topDir)); - DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths, 2); - - List selected = new LinkedList(); - while (true) { - FileStatus f = dt.getNextFile(); - if (f == null) break; - assertEquals(false, f.isDir()); - LOG.info(f.getPath()); - selected.add(f); - } - assertEquals(5, selected.size()); - - LOG.info("Enumerating directories"); - startPaths.clear(); - startPaths.add(fs.getFileStatus(topDir)); - dt = new DirectoryTraversal(fs, startPaths); - selected.clear(); - while (true) { - FileStatus dir = dt.getNextDirectory(); - if (dir == null) break; - assertEquals(true, dir.isDir()); - LOG.info(dir.getPath()); - selected.add(dir); - } - assertEquals(4, selected.size()); - } finally { - myTearDown(); - } - } - - @Test - public void testSuspension() throws IOException { - LOG.info("Starting testSuspension"); - mySetup(); - - try { - Path topDir = new Path(TEST_DIR + "/testenumeration"); - - createTestTree(topDir); - - String top = topDir.toString(); - List startPaths = new LinkedList(); - startPaths.add(fs.getFileStatus(new Path(top + "/a"))); - startPaths.add(fs.getFileStatus(new Path(top + "/b"))); - DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths); - - int limit = 2; - short targetRepl = 1; - Path raid = new Path("/raid"); - DirectoryTraversal.FileFilter filter = - new RaidFilter.TimeBasedFilter(conf, - RaidNode.xorDestinationPath(conf), 1, Time.now(), 0); - List selected = dt.getFilteredFiles(filter, limit); - for (FileStatus f: selected) { - LOG.info(f.getPath()); - } - assertEquals(limit, selected.size()); - - selected = dt.getFilteredFiles(filter, limit); - for (FileStatus f: selected) { - LOG.info(f.getPath()); - } - assertEquals(limit, selected.size()); - } finally { - myTearDown(); - } - } - - @Test - public void testFileFilter() throws IOException { - mySetup(); - - try { - Path topDir = new Path(TEST_DIR + "/testFileFilter"); - int targetRepl = 1; - createTestTree(topDir); - Path file = new Path(topDir.toString() + "/a/f1"); - FileStatus stat = fs.getFileStatus(file); - PolicyInfo info = new PolicyInfo("testFileFilter", conf); - info.setSrcPath(topDir.toString()); - info.setErasureCode("rs"); - info.setDescription("test policy"); - info.setProperty("targetReplication", "1"); - info.setProperty("metaReplication", "1"); - - DirectoryTraversal.FileFilter timeBasedXORFilter = - new RaidFilter.TimeBasedFilter(conf, - RaidNode.xorDestinationPath(conf), targetRepl, - Time.now(), 0); - DirectoryTraversal.FileFilter timeBasedRSFilter = - new RaidFilter.TimeBasedFilter(conf, - RaidNode.rsDestinationPath(conf), targetRepl, - Time.now(), 0); - DirectoryTraversal.FileFilter preferenceForRSFilter = - new RaidFilter.PreferenceFilter( - conf, RaidNode.rsDestinationPath(conf), - RaidNode.xorDestinationPath(conf), 1, Time.now(), 0); - - assertTrue(timeBasedXORFilter.check(stat)); - assertTrue(timeBasedRSFilter.check(stat)); - assertTrue(preferenceForRSFilter.check(stat)); - - RaidNode.doRaid( - conf, info, stat, new RaidNode.Statistics(), Reporter.NULL); - - assertTrue(timeBasedXORFilter.check(stat)); - assertFalse(timeBasedRSFilter.check(stat)); - assertFalse(preferenceForRSFilter.check(stat)); - } finally { - myTearDown(); - } - } - - /** - * Creates a test directory tree. - * top - * / | \ - * / | f5 - * a b___ - * / \ |\ \ - * f1 f2 f3f4 c - */ - private void createTestTree(Path topDir) throws IOException { - String top = topDir.toString(); - fs.delete(topDir, true); - - fs.mkdirs(topDir); - fs.create(new Path(top + "/f5")).close(); - - fs.mkdirs(new Path(top + "/a")); - createTestFile(new Path(top + "/a/f1")); - createTestFile(new Path(top + "/a/f2")); - - fs.mkdirs(new Path(top + "/b")); - fs.mkdirs(new Path(top + "/b/c")); - createTestFile(new Path(top + "/b/f3")); - createTestFile(new Path(top + "/b/f4")); - } - - private void createTestFile(Path file) throws IOException { - long blockSize = 8192; - byte[] bytes = new byte[(int)blockSize]; - FSDataOutputStream stm = fs.create(file, false, 4096, (short)1, blockSize); - stm.write(bytes); - stm.write(bytes); - stm.write(bytes); - stm.close(); - FileStatus stat = fs.getFileStatus(file); - assertEquals(blockSize, stat.getBlockSize()); - } - - private void mySetup() throws IOException { - conf = new Configuration(); - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); - dfs.waitActive(); - fs = dfs.getFileSystem(); - } - - private void myTearDown() { - if (dfs != null) { dfs.shutdown(); } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java deleted file mode 100644 index d5e4b8e9fdb..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.HashSet; -import java.util.Random; -import java.util.Set; - -import org.apache.hadoop.util.Time; -import org.junit.Test; - -public class TestErasureCodes { - final int TEST_CODES = 100; - final int TEST_TIMES = 1000; - final Random RAND = new Random(); - - @Test - public void testEncodeDecode() { - for (int n = 0; n < TEST_CODES; n++) { - int stripeSize = RAND.nextInt(99) + 1; // 1, 2, 3, ... 100 - int paritySize = RAND.nextInt(9) + 1; //1, 2, 3, 4, ... 10 - ErasureCode ec = new ReedSolomonCode(stripeSize, paritySize); - for (int m = 0; m < TEST_TIMES; m++) { - int symbolMax = (int) Math.pow(2, ec.symbolSize()); - int[] message = new int[stripeSize]; - for (int i = 0; i < stripeSize; i++) { - message[i] = RAND.nextInt(symbolMax); - } - int[] parity = new int[paritySize]; - ec.encode(message, parity); - int[] data = new int[stripeSize + paritySize]; - int[] copy = new int[data.length]; - for (int i = 0; i < paritySize; i++) { - data[i] = parity[i]; - copy[i] = parity[i]; - } - for (int i = 0; i < stripeSize; i++) { - data[i + paritySize] = message[i]; - copy[i + paritySize] = message[i]; - } - int erasedLen = paritySize == 1 ? 1 : RAND.nextInt(paritySize - 1) + 1; - int[] erasedLocations = randomErasedLocation(erasedLen, data.length); - for (int i = 0; i < erasedLocations.length; i++) { - data[erasedLocations[i]] = 0; - } - int[] erasedValues = new int[erasedLen]; - ec.decode(data, erasedLocations, erasedValues); - for (int i = 0; i < erasedLen; i++) { - assertEquals("Decode failed", copy[erasedLocations[i]], erasedValues[i]); - } - } - } - } - - @Test - public void testRSPerformance() { - int stripeSize = 10; - int paritySize = 4; - ErasureCode ec = new ReedSolomonCode(stripeSize, paritySize); - int symbolMax = (int) Math.pow(2, ec.symbolSize()); - byte[][] message = new byte[stripeSize][]; - int bufsize = 1024 * 1024 * 10; - for (int i = 0; i < stripeSize; i++) { - message[i] = new byte[bufsize]; - for (int j = 0; j < bufsize; j++) { - message[i][j] = (byte) RAND.nextInt(symbolMax); - } - } - byte[][] parity = new byte[paritySize][]; - for (int i = 0; i < paritySize; i++) { - parity[i] = new byte[bufsize]; - } - long encodeStart = Time.now(); - int[] tmpIn = new int[stripeSize]; - int[] tmpOut = new int[paritySize]; - for (int i = 0; i < bufsize; i++) { - // Copy message. - for (int j = 0; j < stripeSize; j++) tmpIn[j] = 0x000000FF & message[j][i]; - ec.encode(tmpIn, tmpOut); - // Copy parity. - for (int j = 0; j < paritySize; j++) parity[j][i] = (byte)tmpOut[j]; - } - long encodeEnd = Time.now(); - float encodeMSecs = (encodeEnd - encodeStart); - System.out.println("Time to encode rs = " + encodeMSecs + - "msec (" + message[0].length / (1000 * encodeMSecs) + " MB/s)"); - - // Copy erased array. - int[] data = new int[paritySize + stripeSize]; - // 4th location is the 0th symbol in the message - int[] erasedLocations = new int[]{4, 1, 5, 7}; - int[] erasedValues = new int[erasedLocations.length]; - byte[] copy = new byte[bufsize]; - for (int j = 0; j < bufsize; j++) { - copy[j] = message[0][j]; - message[0][j] = 0; - } - - long decodeStart = Time.now(); - for (int i = 0; i < bufsize; i++) { - // Copy parity first. - for (int j = 0; j < paritySize; j++) { - data[j] = 0x000000FF & parity[j][i]; - } - // Copy message. Skip 0 as the erased symbol - for (int j = 1; j < stripeSize; j++) { - data[j + paritySize] = 0x000000FF & message[j][i]; - } - // Use 0, 2, 3, 6, 8, 9, 10, 11, 12, 13th symbol to reconstruct the data - ec.decode(data, erasedLocations, erasedValues); - message[0][i] = (byte)erasedValues[0]; - } - long decodeEnd = Time.now(); - float decodeMSecs = (decodeEnd - decodeStart); - System.out.println("Time to decode = " + decodeMSecs + - "msec (" + message[0].length / (1000 * decodeMSecs) + " MB/s)"); - assertTrue("Decode failed", java.util.Arrays.equals(copy, message[0])); - } - - @Test - public void testXorPerformance() { - java.util.Random RAND = new java.util.Random(); - int stripeSize = 10; - byte[][] message = new byte[stripeSize][]; - int bufsize = 1024 * 1024 * 10; - for (int i = 0; i < stripeSize; i++) { - message[i] = new byte[bufsize]; - for (int j = 0; j < bufsize; j++) { - message[i][j] = (byte)RAND.nextInt(256); - } - } - byte[] parity = new byte[bufsize]; - - long encodeStart = Time.now(); - for (int i = 0; i < bufsize; i++) { - for (int j = 0; j < stripeSize; j++) parity[i] ^= message[j][i]; - } - long encodeEnd = Time.now(); - float encodeMSecs = encodeEnd - encodeStart; - System.out.println("Time to encode xor = " + encodeMSecs + - " msec (" + message[0].length / (1000 * encodeMSecs) + "MB/s)"); - - byte[] copy = new byte[bufsize]; - for (int j = 0; j < bufsize; j++) { - copy[j] = message[0][j]; - message[0][j] = 0; - } - - long decodeStart = Time.now(); - for (int i = 0; i < bufsize; i++) { - for (int j = 1; j < stripeSize; j++) message[0][i] ^= message[j][i]; - message[0][i] ^= parity[i]; - } - long decodeEnd = Time.now(); - float decodeMSecs = decodeEnd - decodeStart; - System.out.println("Time to decode xor = " + decodeMSecs + - " msec (" + message[0].length / (1000 * decodeMSecs) + "MB/s)"); - assertTrue("Decode failed", java.util.Arrays.equals(copy, message[0])); - } - - @Test - public void testComputeErrorLocations() { - for (int i = 0; i < TEST_TIMES; ++i) { - verifyErrorLocations(10, 4, 1); - verifyErrorLocations(10, 4, 2); - } - } - - public void verifyErrorLocations(int stripeSize, int paritySize, int errors) { - int[] message = new int[stripeSize]; - int[] parity = new int[paritySize]; - Set errorLocations = new HashSet(); - for (int i = 0; i < message.length; ++i) { - message[i] = RAND.nextInt(256); - } - while (errorLocations.size() < errors) { - int loc = RAND.nextInt(stripeSize + paritySize); - errorLocations.add(loc); - } - ReedSolomonCode codec = new ReedSolomonCode(stripeSize, paritySize); - codec.encode(message, parity); - int[] data = combineArrays(parity, message); - for (Integer i : errorLocations) { - data[i] = randError(data[i]); - } - Set recoveredLocations = new HashSet(); - boolean resolved = codec.computeErrorLocations(data, recoveredLocations); - if (resolved) { - assertEquals(errorLocations, recoveredLocations); - } - } - - private int randError(int actual) { - while (true) { - int r = RAND.nextInt(256); - if (r != actual) { - return r; - } - } - } - - private int[] combineArrays(int[] array1, int[] array2) { - int[] result = new int[array1.length + array2.length]; - for (int i = 0; i < array1.length; ++i) { - result[i] = array1[i]; - } - for (int i = 0; i < array2.length; ++i) { - result[i + array1.length] = array2[i]; - } - return result; - } - - private int[] randomErasedLocation(int erasedLen, int dataLen) { - int[] erasedLocations = new int[erasedLen]; - for (int i = 0; i < erasedLen; i++) { - Set s = new HashSet(); - while (s.size() != erasedLen) { - s.add(RAND.nextInt(dataLen)); - } - int t = 0; - for (int erased : s) { - erasedLocations[t++] = erased; - } - } - return erasedLocations; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java deleted file mode 100644 index 523a79e8018..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertTrue; - -import java.util.HashSet; -import java.util.Random; -import java.util.Set; - -import org.junit.Test; - -public class TestGaloisField { - - final int TEST_TIMES = 10000; - final Random RAND = new Random(); - final static GaloisField GF = GaloisField.getInstance(); - - private int randGF() { - return 0x000000FF & RAND.nextInt(GF.getFieldSize()); - } - private int[] randGFPoly(int len) { - int[] result = new int[len]; - for (int i = 0; i < len; i++) { - result[i] = randGF(); - } - return result; - } - - @Test - public void testGetInstance() { - GaloisField gf1 = GaloisField.getInstance(256, 285); - GaloisField gf2 = GaloisField.getInstance(); - GaloisField gf3 = GaloisField.getInstance(128, 137); - GaloisField gf4 = GaloisField.getInstance(128, 137); - GaloisField gf5 = GaloisField.getInstance(512, 529); - GaloisField gf6 = GaloisField.getInstance(512, 529); - assertTrue(gf1 == gf2); - assertTrue(gf3 == gf4); - assertTrue(gf5 == gf6); - } - - @Test - public void testDistributivity() { - for (int i = 0; i < TEST_TIMES; i++) { - int a = RAND.nextInt(GF.getFieldSize()); - int b = RAND.nextInt(GF.getFieldSize()); - int c = RAND.nextInt(GF.getFieldSize()); - int result1 = GF.multiply(a, GF.add(b, c)); - int result2 = GF.add(GF.multiply(a, b), GF.multiply(a, c)); - assertTrue("Distributivity test #" + i + " failed: " + a + ", " + b + ", " - + c, result1 == result2); - } - } - - @Test - public void testDevision() { - for (int i = 0; i < TEST_TIMES; i++) { - int a = RAND.nextInt(GF.getFieldSize()); - int b = RAND.nextInt(GF.getFieldSize()); - if (b == 0) { - continue; - } - int c = GF.divide(a, b); - assertTrue("Division test #" + i + " failed: " + a + "/" + b + " = " + c, - a == GF.multiply(c, b)); - } - } - - @Test - public void testPower() { - for (int i = 0; i < TEST_TIMES; i++) { - int a = randGF(); - int n = RAND.nextInt(10); - int result1 = GF.power(a, n); - int result2 = 1; - for (int j = 0; j < n; j++) { - result2 = GF.multiply(result2, a); - } - assert(result1 == result2); - } - } - - @Test - public void testPolynomialDistributivity() { - final int TEST_LEN = 15; - for (int i = 0; i < TEST_TIMES; i++) { - int[] a = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int[] b = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int[] c = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int[] result1 = GF.multiply(a, GF.add(b, c)); - int[] result2 = GF.add(GF.multiply(a, b), GF.multiply(a, c)); - assertTrue("Distributivity test on polynomials failed", - java.util.Arrays.equals(result1, result2)); - } - } - - @Test - public void testSubstitute() { - final int TEST_LEN = 15; - for (int i = 0; i < TEST_TIMES; i++) { - int[] a = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int[] b = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int[] c = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - int x = randGF(); - // (a * b * c)(x) - int result1 = GF.substitute(GF.multiply(GF.multiply(a, b), c), x); - // a(x) * b(x) * c(x) - int result2 = - GF.multiply(GF.multiply(GF.substitute(a, x), GF.substitute(b, x)), - GF.substitute(c, x)); - assertTrue("Substitute test on polynomial failed", - result1 == result2); - } - } - - @Test - public void testSolveVandermondeSystem() { - final int TEST_LEN = 15; - for (int i = 0; i < TEST_TIMES; i++) { - int[] z = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1); - // generate distinct values for x - int[] x = new int[z.length]; - Set s = new HashSet(); - while (s.size() != z.length) { - s.add(randGF()); - } - int t = 0; - for (int v : s) { - x[t++] = v; - } - // compute the output for the Vandermonde system - int[] y = new int[x.length]; - for (int j = 0; j < x.length; j++) { - y[j] = 0; - for (int k = 0; k < x.length; k++) { - //y[j] = y[j] + z[k] * pow(x[k], j); - y[j] = GF.add(y[j], GF.multiply(GF.power(x[k], j), z[k])); - } - } - - GF.solveVandermondeSystem(x, y); - assertTrue("Solving Vandermonde system failed", - java.util.Arrays.equals(y, z)); - } - } - - @Test - public void testRemainder() { - final int TEST_LEN = 15; - for (int i = 0; i < TEST_TIMES; i++) { - int[] quotient = null; - int[] divisor = null; - int[] remainder = null; - int[] dividend = null; - while (true) { - quotient = randGFPoly(RAND.nextInt(TEST_LEN - 3) + 3); - divisor = randGFPoly(RAND.nextInt(quotient.length - 2) + 2); - remainder = randGFPoly(RAND.nextInt(divisor.length - 1) + 1); - dividend = GF.add(remainder, GF.multiply(quotient, divisor)); - if (quotient[quotient.length - 1] != 0 && - divisor[divisor.length - 1] != 0 && - remainder[remainder.length - 1] != 0) { - // make sure all the leading terms are not zero - break; - } - } - GF.remainder(dividend, divisor); - for (int j = 0; j < remainder.length; j++) { - assertTrue("Distributivity test on polynomials failed", - dividend[j] == remainder[j]); - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java deleted file mode 100644 index 791280ce881..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.nio.charset.Charset; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestHarIndexParser { - final static Log LOG = LogFactory.getLog(TestHarIndexParser.class); - File indexFile = null; - - @Before - public void setUp() throws FileNotFoundException, IOException { - LOG.info("TestHarIndexParser.setUp()"); - indexFile = File.createTempFile("harindex", ".tmp"); - indexFile.deleteOnExit(); - OutputStreamWriter out = new OutputStreamWriter( - new FileOutputStream(indexFile), - Charset.forName("UTF-8")); - out.write("%2F dir 1282018162460+0+493+hadoop+hadoop 0 0 f1 f2 f3 f4\n"); - out.write("%2Ff1 file part-0 0 1024 1282018141145+1282018140822+420+hadoop+hadoop\n"); - out.write("%2Ff3 file part-0 2048 1024 1282018148590+1282018148255+420+hadoop+hadoop\n"); - out.write("%2Ff2 file part-0 1024 1024 1282018144198+1282018143852+420+hadoop+hadoop\n"); - out.write("%2Ff4 file part-1 0 1024000 1282018162959+1282018162460+420+hadoop+hadoop\n"); - out.flush(); - out.close(); - } - - @After - public void tearDown() { - LOG.info("TestHarIndexParser.tearDown()"); - if (indexFile != null) - indexFile.delete(); - } - - @Test - public void testHarIndexParser() - throws UnsupportedEncodingException, IOException { - LOG.info("testHarIndexParser started."); - InputStream in = new FileInputStream(indexFile); - long size = indexFile.length(); - HarIndex parser = new HarIndex(in, size); - - HarIndex.IndexEntry entry = parser.findEntry("part-0", 2100); - assertEquals("/f3", entry.fileName); - - LOG.info("testHarIndexParser finished."); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java deleted file mode 100644 index e22339d4095..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.util.Time; -import org.junit.Test; - -public class TestRaidFilter { - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static Log LOG = - LogFactory.getLog("org.apache.hadoop.raid.TestRaidFilter"); - - Configuration conf; - MiniDFSCluster dfs = null; - FileSystem fs = null; - - private void mySetup() throws Exception { - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 2, true, null); - dfs.waitActive(); - fs = dfs.getFileSystem(); - String namenode = fs.getUri().toString(); - FileSystem.setDefaultUri(conf, namenode); - } - - private void myTearDown() throws Exception { - if (dfs != null) { dfs.shutdown(); } - } - - @Test - public void testLayeredPolicies() throws Exception { - mySetup(); - Path src1 = new Path("/user/foo"); - Path src2 = new Path("/user/foo/bar"); - - PolicyInfo info1 = new PolicyInfo("p1", conf); - info1.setSrcPath(src1.toString()); - info1.setErasureCode("xor"); - info1.setDescription("test policy"); - info1.setProperty("targetReplication", "1"); - info1.setProperty("metaReplication", "1"); - info1.setProperty("modTimePeriod", "0"); - - PolicyInfo info2 = new PolicyInfo("p2", conf); - info2.setSrcPath(src2.toString()); - info2.setErasureCode("xor"); - info2.setDescription("test policy"); - info2.setProperty("targetReplication", "1"); - info2.setProperty("metaReplication", "1"); - info2.setProperty("modTimePeriod", "0"); - - ArrayList all = new ArrayList(); - all.add(info1); - all.add(info2); - - try { - long blockSize = 1024; - byte[] bytes = new byte[(int)blockSize]; - Path f1 = new Path(src1, "f1"); - Path f2 = new Path(src2, "f2"); - FSDataOutputStream stm1 = fs.create(f1, false, 4096, (short)1, blockSize); - FSDataOutputStream stm2 = fs.create(f2, false, 4096, (short)1, blockSize); - FSDataOutputStream[] stms = new FSDataOutputStream[]{stm1, stm2}; - for (FSDataOutputStream stm: stms) { - stm.write(bytes); - stm.write(bytes); - stm.write(bytes); - stm.close(); - } - - Thread.sleep(1000); - - FileStatus stat1 = fs.getFileStatus(f1); - FileStatus stat2 = fs.getFileStatus(f2); - - RaidFilter.Statistics stats = new RaidFilter.Statistics(); - RaidFilter.TimeBasedFilter filter = new RaidFilter.TimeBasedFilter( - conf, RaidNode.xorDestinationPath(conf), info1, all, - Time.now(), stats); - System.out.println("Stats " + stats); - - assertTrue(filter.check(stat1)); - assertFalse(filter.check(stat2)); - - } finally { - myTearDown(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java deleted file mode 100644 index 9df30b39a60..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.util.StringUtils; -import org.apache.log4j.Level; -import org.junit.Test; - -/** - * If a file gets deleted, then verify that the parity file gets deleted too. - */ -public class TestRaidHar { - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static String CONFIG_FILE = new File(TEST_DIR, - "test-raid.xml").getAbsolutePath(); - final static long RELOAD_INTERVAL = 1000; - final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode"); - final Random rand = new Random(); - - { - ((Log4JLogger)RaidNode.LOG).getLogger().setLevel(Level.ALL); - } - - - Configuration conf; - String namenode = null; - String hftp = null; - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - String jobTrackerName = null; - - /** - * create mapreduce and dfs clusters - */ - private void createClusters(boolean local) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - conf.set("raid.config.file", CONFIG_FILE); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // the RaidNode does the raiding inline (instead of submitting to map/reduce) - if (local) { - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - } else { - conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode"); - } - - conf.set("raid.server.address", "localhost:0"); - conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - - // create a dfs and map-reduce cluster - final int taskTrackers = 4; - - dfs = new MiniDFSCluster(conf, 3, true, null); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - mr = new MiniMRCluster(taskTrackers, namenode, 3); - JobConf jobConf = mr.createJobConf(); - jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - conf.set("mapred.job.tracker", jobTrackerName); - conf.set("mapreduce.framework.name", "yarn"); - String rmAdress = jobConf.get("yarn.resourcemanager.address"); - if (rmAdress != null) { - conf.set("yarn.resourcemanager.address", rmAdress); - } - String schedulerAdress = - jobConf.get("yarn.resourcemanager.scheduler.address"); - if (schedulerAdress != null) { - conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); - } - String jobHistoryAddress = - jobConf.get("mapreduce.jobhistory.address"); - if (jobHistoryAddress != null) { - conf.set("mapreduce.jobhistory.address", jobHistoryAddress); - } - } - - /** - * create raid.xml file for RaidNode - */ - private void mySetup(long targetReplication, - long metaReplication, long stripeLength) throws Exception { - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - String str = " " + - " " + - " " + - "xor " + - " " + - "targetReplication " + - "" + targetReplication + " " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "" + metaReplication + " " + - " replication factor of parity file" + - " " + - " " + - " " + - "stripeLength " + - "" + stripeLength + " " + - " the max number of blocks in a file to RAID together " + - " " + - " " + - " " + - "time_before_har " + - "0 " + - " amount of time waited before har'ing parity files" + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " " + - "" + - "" + - ""; - fileWriter.write(str); - fileWriter.close(); - } - - /** - * stop clusters created earlier - */ - private void stopClusters() throws Exception { - if (mr != null) { mr.shutdown(); } - if (dfs != null) { dfs.shutdown(); } - } - - /** - * Test that parity files that do not have an associated master file - * get deleted. - */ - @Test - public void testRaidHar() throws Exception { - LOG.info("Test testRaidHar started."); - - long blockSizes [] = {1024L}; - long stripeLengths [] = {5}; - long targetReplication = 1; - long metaReplication = 1; - int numBlock = 9; - int iter = 0; - - createClusters(true); - try { - for (long blockSize : blockSizes) { - for (long stripeLength : stripeLengths) { - doTestHar(iter, targetReplication, metaReplication, - stripeLength, blockSize, numBlock); - iter++; - } - } - } finally { - stopClusters(); - } - LOG.info("Test testRaidHar completed."); - } - - /** - * Create parity file, delete original file and then validate that - * parity file is automatically deleted. - */ - private void doTestHar(int iter, long targetReplication, - long metaReplication, long stripeLength, - long blockSize, int numBlock) throws Exception { - LOG.info("doTestHar started---------------------------:" + " iter " + iter + - " blockSize=" + blockSize + " stripeLength=" + stripeLength); - mySetup(targetReplication, metaReplication, stripeLength); - Path dir = new Path("/user/test/raidtest/subdir/"); - Path file1 = new Path(dir + "/file" + iter); - RaidNode cnode = null; - try { - Path destPath = new Path("/destraid/user/test/raidtest/subdir"); - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - for (int i = 0; i < 10; i++) { - Path file = new Path(dir + "/file" + i); - TestRaidNode.createOldFile(fileSys, file, 1, numBlock, blockSize); - } - LOG.info("doTestHar created test files for iteration " + iter); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - cnode = RaidNode.createRaidNode(null, localConf); - FileStatus[] listPaths = null; - - int maxFilesFound = 0; - // wait till file is raided - while (true) { - try { - listPaths = fileSys.listStatus(destPath); - int count = 0; - Path harPath = null; - int filesFound = 0; - if (listPaths != null) { - for (FileStatus s : listPaths) { - LOG.info("doTestHar found path " + s.getPath()); - - if (!s.isDir()) - filesFound++; - if (filesFound > maxFilesFound) - maxFilesFound = filesFound; - - if (s.getPath().toString().endsWith(".har")) { - // If a HAR directory is found, ensure that we have seen - // 10 parity files. We have to keep track of the max # of - // files since some parity files might get deleted by the - // purge thread. - assertEquals(10, maxFilesFound); - harPath = s.getPath(); - count++; - } - } - } - if (count == 1 && listPaths.length == 1) { - Path partfile = new Path(harPath, "part-0"); - assertEquals(fileSys.getFileStatus(partfile).getReplication(), - targetReplication); - break; - } - } catch (FileNotFoundException e) { - //ignore - } - LOG.info("doTestHar waiting for files to be raided and parity files to be har'ed and deleted. Found " + - (listPaths == null ? "none" : listPaths.length)); - Thread.sleep(1000); // keep waiting - - } - - fileSys.delete(dir, true); - // wait till raid file is deleted - int count = 1; - while (count > 0) { - count = 0; - try { - listPaths = fileSys.listStatus(destPath); - if (listPaths != null) { - for (FileStatus s : listPaths) { - LOG.info("doTestHar found path " + s.getPath()); - if (s.getPath().toString().endsWith(".har")) { - count++; - } - } - } - } catch (FileNotFoundException e) { } //ignoring - LOG.info("doTestHar waiting for har file to be deleted. Found " + - (listPaths == null ? "none" : listPaths.length) + " files"); - Thread.sleep(1000); - } - - } catch (Exception e) { - LOG.info("doTestHar Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - } - LOG.info("doTestHar completed:" + " blockSize=" + blockSize + - " stripeLength=" + stripeLength); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java deleted file mode 100644 index aae0bcbab09..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java +++ /dev/null @@ -1,738 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; -import java.util.Random; -import java.util.zip.CRC32; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.raid.protocol.PolicyList; -import org.apache.hadoop.util.JarFinder; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; -import org.junit.Test; - -/** - * Test the generation of parity blocks for files with different block - * sizes. Also test that a data block can be regenerated from a raid stripe - * using the parity block - */ -public class TestRaidNode { - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - public static final String DistRaid_JAR = JarFinder.getJar(DistRaid.class); - final static String CONFIG_FILE = new File(TEST_DIR, - "test-raid.xml").getAbsolutePath(); - final static long RELOAD_INTERVAL = 1000; - final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode"); - final static Random rand = new Random(); - - Configuration conf; - String namenode = null; - String hftp = null; - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - String jobTrackerName = null; - - /** - * create mapreduce and dfs clusters - */ - private void createClusters(boolean local) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - conf.set("raid.config.file", CONFIG_FILE); - conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - conf.setBoolean("dfs.permissions.enabled", true); - conf.setLong(JobMonitor.JOBMONITOR_INTERVAL_KEY, 20000); - conf.setLong(RaidNode.TRIGGER_MONITOR_SLEEP_TIME_KEY, 3000L); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // the RaidNode does the raiding inline (instead of submitting to map/reduce) - if (local) { - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - } else { - conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode"); - } - - conf.set("raid.server.address", "localhost:0"); - - // create a dfs and map-reduce cluster - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); - builder.numDataNodes(6); - builder.format(true); - dfs = builder.build(); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - - namenode = fileSys.getUri().toString(); - final int taskTrackers = 4; - mr = new MiniMRCluster(taskTrackers, namenode, 3); - JobConf jobConf = mr.createJobConf(); - jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - conf.set("mapred.job.tracker", jobTrackerName); - conf.set("mapreduce.framework.name", "yarn"); - String rmAdress = jobConf.get("yarn.resourcemanager.address"); - if (rmAdress != null) { - conf.set("yarn.resourcemanager.address", rmAdress); - } - String schedulerAdress = - jobConf.get("yarn.resourcemanager.scheduler.address"); - if (schedulerAdress != null) { - conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); - } - String jobHistoryAddress = - jobConf.get("mapreduce.jobhistory.address"); - if (jobHistoryAddress != null) { - conf.set("mapreduce.jobhistory.address", jobHistoryAddress); - } - } - - class ConfigBuilder { - private List policies; - - public ConfigBuilder() { - policies = new java.util.ArrayList(); - } - - public void addPolicy(String name, String path, String parent) { - String str = - " " + - " " + - "" + parent + "" + - "" + - ""; - policies.add(str); - } - - public void addPolicy(String name, short srcReplication, - long targetReplication, long metaReplication, long stripeLength) { - String str = - " " + - " " + - "xor " + - " " + - "srcReplication " + - "" + srcReplication + " " + - " pick only files whole replFactor is greater than or equal to " + - " " + - " " + - " " + - "targetReplication " + - "" + targetReplication + " " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "" + metaReplication + " " + - " replication factor of parity file" + - " " + - " " + - " " + - "stripeLength " + - "" + stripeLength + " " + - " the max number of blocks in a file to RAID together " + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " " + - "" + - ""; - policies.add(str); - } - - public void addPolicy(String name, String path, short srcReplication, - long targetReplication, long metaReplication, long stripeLength) { - String str = - " " + - " " + - "xor " + - " " + - "srcReplication " + - "" + srcReplication + " " + - " pick only files whole replFactor is greater than or equal to " + - " " + - " " + - " " + - "targetReplication " + - "" + targetReplication + " " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "" + metaReplication + " " + - " replication factor of parity file" + - " " + - " " + - " " + - "stripeLength " + - "" + stripeLength + " " + - " the max number of blocks in a file to RAID together " + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " " + - "" + - ""; - policies.add(str); - } - - public void persist() throws IOException { - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - fileWriter.write(""); - for (String policy: policies) { - fileWriter.write(policy); - } - fileWriter.write(""); - fileWriter.close(); - } - } - - /** - * stop clusters created earlier - */ - private void stopClusters() throws Exception { - if (mr != null) { mr.shutdown(); } - if (dfs != null) { dfs.shutdown(); } - } - - /** - * Test to run a filter - */ - @Test - public void testPathFilter() throws Exception { - LOG.info("Test testPathFilter started."); - - long blockSizes [] = {1024L}; - int stripeLengths [] = {5, 6, 10, 11, 12}; - int targetReplication = 1; - int metaReplication = 1; - int numBlock = 11; - int iter = 0; - - createClusters(true); - try { - for (long blockSize : blockSizes) { - for (long stripeLength : stripeLengths) { - doTestPathFilter(iter, targetReplication, metaReplication, - stripeLength, blockSize, numBlock); - iter++; - } - } - doCheckPolicy(); - } finally { - stopClusters(); - } - LOG.info("Test testPathFilter completed."); - } - - /** - * Test to run a filter - */ - private void doTestPathFilter(int iter, long targetReplication, - long metaReplication, long stripeLength, - long blockSize, int numBlock) throws Exception { - LOG.info("doTestPathFilter started---------------------------:" + " iter " + iter + - " blockSize=" + blockSize + " stripeLength=" + stripeLength); - ConfigBuilder cb = new ConfigBuilder(); - cb.addPolicy("policy1", "/user/dhruba/raidtest", (short)1, targetReplication, metaReplication, stripeLength); - cb.persist(); - - RaidShell shell = null; - Path dir = new Path("/user/dhruba/raidtest/"); - Path file1 = new Path(dir + "/file" + iter); - RaidNode cnode = null; - try { - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - long crc1 = createOldFile(fileSys, file1, 1, numBlock, blockSize); - LOG.info("doTestPathFilter created test files for iteration " + iter); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - cnode = RaidNode.createRaidNode(null, localConf); - FileStatus[] listPaths = null; - - // wait till file is raided - while (true) { - try { - listPaths = fileSys.listStatus(destPath); - int count = 0; - if (listPaths != null && listPaths.length == 1) { - for (FileStatus s : listPaths) { - LOG.info("doTestPathFilter found path " + s.getPath()); - if (!s.getPath().toString().endsWith(".tmp") && - fileSys.getFileStatus(file1).getReplication() == - targetReplication) { - count++; - } - } - } - if (count > 0) { - break; - } - } catch (FileNotFoundException e) { - //ignore - } - LOG.info("doTestPathFilter waiting for files to be raided. Found " + - (listPaths == null ? "none" : listPaths.length)); - Thread.sleep(1000); // keep waiting - } - // assertEquals(listPaths.length, 1); // all files raided - LOG.info("doTestPathFilter all files found in Raid."); - - // check for error at beginning of file - shell = new RaidShell(conf); - shell.initializeRpc(conf, cnode.getListenerAddress()); - if (numBlock >= 1) { - LOG.info("doTestPathFilter Check error at beginning of file."); - simulateError(shell, fileSys, file1, crc1, 0); - } - - // check for error at the beginning of second block - if (numBlock >= 2) { - LOG.info("doTestPathFilter Check error at beginning of second block."); - simulateError(shell, fileSys, file1, crc1, blockSize + 1); - } - - // check for error at the middle of third block - if (numBlock >= 3) { - LOG.info("doTestPathFilter Check error at middle of third block."); - simulateError(shell, fileSys, file1, crc1, 2 * blockSize + 10); - } - - // check for error at the middle of second stripe - if (numBlock >= stripeLength + 1) { - LOG.info("doTestPathFilter Check error at middle of second stripe."); - simulateError(shell, fileSys, file1, crc1, - stripeLength * blockSize + 100); - } - - } catch (Exception e) { - LOG.info("doTestPathFilter Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (shell != null) shell.close(); - if (cnode != null) { cnode.stop(); cnode.join(); } - LOG.info("doTestPathFilter delete file " + file1); - fileSys.delete(file1, true); - } - LOG.info("doTestPathFilter completed:" + " blockSize=" + blockSize + - " stripeLength=" + stripeLength); - } - - // Check that raid occurs only on files that have a replication factor - // greater than or equal to the specified value - private void doCheckPolicy() throws Exception { - LOG.info("doCheckPolicy started---------------------------:"); - short srcReplication = 1; - long targetReplication = 2; - long metaReplication = 1; - long stripeLength = 2; - long blockSize = 1024; - int numBlock = 3; - ConfigBuilder cb = new ConfigBuilder(); - cb.addPolicy("policy1", "/user/dhruba/policytest", srcReplication, - targetReplication, metaReplication, stripeLength); - cb.persist(); - Path dir = new Path("/user/dhruba/policytest/"); - Path file1 = new Path(dir + "/file1"); - Path file2 = new Path(dir + "/file2"); - RaidNode cnode = null; - try { - Path destPath = new Path("/destraid/user/dhruba/policytest"); - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - cnode = RaidNode.createRaidNode(null, localConf); - - // this file should be picked up RaidNode - createOldFile(fileSys, file2, 2, numBlock, blockSize); - FileStatus[] listPaths = null; - - long firstmodtime = 0; - // wait till file is raided - while (true) { - Thread.sleep(1000); // waiting - try { - listPaths = fileSys.listStatus(destPath); - } catch (FileNotFoundException e) { - LOG.warn("File not found " + destPath); - // The directory have been deleted by the purge thread. - continue; - } - int count = 0; - if (listPaths != null && listPaths.length == 1) { - for (FileStatus s : listPaths) { - LOG.info("doCheckPolicy found path " + s.getPath()); - if (!s.getPath().toString().endsWith(".tmp") && - fileSys.getFileStatus(file2).getReplication() == - targetReplication) { - count++; - firstmodtime = s.getModificationTime(); - } - } - } - if (count > 0) { - break; - } - LOG.info("doCheckPolicy waiting for files to be raided. Found " + - (listPaths == null ? "none" : listPaths.length)); - } - assertEquals(listPaths.length, 1); - - LOG.info("doCheckPolicy all files found in Raid the first time."); - - LOG.info("doCheckPolicy: recreating source file"); - createOldFile(fileSys, file2, 2, numBlock, blockSize); - - FileStatus st = fileSys.getFileStatus(file2); - assertTrue(st.getModificationTime() > firstmodtime); - - // wait till file is raided - while (true) { - Thread.sleep(20000L); // waiting - listPaths = fileSys.listStatus(destPath); - int count = 0; - if (listPaths != null && listPaths.length == 1) { - for (FileStatus s : listPaths) { - LOG.info("doCheckPolicy found path " + s.getPath() + " " + s.getModificationTime()); - if (!s.getPath().toString().endsWith(".tmp") && - s.getModificationTime() > firstmodtime && - fileSys.getFileStatus(file2).getReplication() == - targetReplication) { - count++; - } - } - } - if (count > 0) { - break; - } - LOG.info("doCheckPolicy waiting for files to be raided. Found " + - (listPaths == null ? "none" : listPaths.length)); - } - assertEquals(listPaths.length, 1); - - LOG.info("doCheckPolicy: file got re-raided as expected."); - - } catch (Exception e) { - LOG.info("doCheckPolicy Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - LOG.info("doTestPathFilter delete file " + file1); - fileSys.delete(file1, false); - } - LOG.info("doCheckPolicy completed:"); - } - - static public void createTestFiles(FileSystem fileSys, - String path, String destpath, int nfile, - int nblock) throws IOException { - createTestFiles(fileSys, path, destpath, nfile, nblock, (short)1); - } - - static void createTestFiles(FileSystem fileSys, String path, String destpath, int nfile, - int nblock, short repl) throws IOException { - long blockSize = 1024L; - Path dir = new Path(path); - Path destPath = new Path(destpath); - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - - for(int i = 0 ; i < nfile; i++){ - Path file = new Path(path + "file" + i); - createOldFile(fileSys, file, repl, nblock, blockSize); - } - } - - /** - * Test dist Raid - */ - @Test - public void testDistRaid() throws Exception { - LOG.info("Test testDistRaid started."); - long targetReplication = 2; - long metaReplication = 2; - long stripeLength = 3; - short srcReplication = 1; - - createClusters(false); - ConfigBuilder cb = new ConfigBuilder(); - cb.addPolicy("policy1", "/user/dhruba/raidtest", - srcReplication, targetReplication, metaReplication, stripeLength); - cb.addPolicy("policy2", "/user/dhruba/raidtest2", - srcReplication, targetReplication, metaReplication, stripeLength); - cb.persist(); - - RaidNode cnode = null; - try { - createTestFiles(fileSys, "/user/dhruba/raidtest/", - "/destraid/user/dhruba/raidtest", 5, 7); - createTestFiles(fileSys, "/user/dhruba/raidtest2/", - "/destraid/user/dhruba/raidtest2", 5, 7); - LOG.info("Test testDistRaid created test files"); - - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR); - cnode = RaidNode.createRaidNode(null, localConf); - // Verify the policies are parsed correctly - for (PolicyList policyList : cnode.getAllPolicies()) { - for (PolicyInfo p : policyList.getAll()) { - if (p.getName().equals("policy1")) { - Path srcPath = new Path("/user/dhruba/raidtest"); - FileSystem fs = srcPath.getFileSystem(conf); - assertTrue(p.getSrcPath().equals( - srcPath.makeQualified(fs.getUri(), fs.getWorkingDirectory()))); - } else { - assertTrue(p.getName().equals("policy2")); - Path srcPath = new Path("/user/dhruba/raidtest2"); - FileSystem fs = srcPath.getFileSystem(conf); - assertTrue(p.getSrcPath().equals( - srcPath.makeQualified(fs.getUri(), fs.getWorkingDirectory()))); - } - assertEquals(targetReplication, - Integer.parseInt(p.getProperty("targetReplication"))); - assertEquals(metaReplication, - Integer.parseInt(p.getProperty("metaReplication"))); - assertEquals(stripeLength, - Integer.parseInt(p.getProperty("stripeLength"))); - } - } - - long start = Time.now(); - final int MAX_WAITTIME = 300000; - - assertTrue("cnode is not DistRaidNode", cnode instanceof DistRaidNode); - DistRaidNode dcnode = (DistRaidNode) cnode; - - while (dcnode.jobMonitor.jobsMonitored() < 2 && - Time.now() - start < MAX_WAITTIME) { - Thread.sleep(1000); - } - - start = Time.now(); - while (dcnode.jobMonitor.jobsSucceeded() < 2 && - Time.now() - start < MAX_WAITTIME) { - Thread.sleep(1000); - } - assertEquals(dcnode.jobMonitor.jobsSucceeded(), dcnode.jobMonitor.jobsMonitored()); - LOG.info("Test testDistRaid successful."); - - } catch (Exception e) { - LOG.info("testDistRaid Exception " + e + StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - stopClusters(); - } - LOG.info("Test testDistRaid completed."); - } - - // - // simulate a corruption at specified offset and verify that eveyrthing is good - // - void simulateError(RaidShell shell, FileSystem fileSys, Path file1, - long crc, long corruptOffset) throws IOException { - // recover the file assuming that we encountered a corruption at offset 0 - String[] args = new String[3]; - args[0] = "-recover"; - args[1] = file1.toString(); - args[2] = Long.toString(corruptOffset); - Path recover1 = shell.recover(args[0], args, 1)[0]; - - // compare that the recovered file is identical to the original one - LOG.info("Comparing file " + file1 + " with recovered file " + recover1); - validateFile(fileSys, file1, recover1, crc); - fileSys.delete(recover1, false); - } - - // - // creates a file and populate it with random data. Returns its crc. - // - static long createOldFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) - throws IOException { - CRC32 crc = new CRC32(); - FSDataOutputStream stm = fileSys.create(name, true, - fileSys.getConf().getInt("io.file.buffer.size", 4096), - (short)repl, blocksize); - // fill random data into file - byte[] b = new byte[(int)blocksize]; - for (int i = 0; i < numBlocks; i++) { - if (i == (numBlocks-1)) { - b = new byte[(int)blocksize/2]; - } - rand.nextBytes(b); - stm.write(b); - crc.update(b); - } - - stm.close(); - return crc.getValue(); - } - - // - // validates that file matches the crc. - // - private void validateFile(FileSystem fileSys, Path name1, Path name2, long crc) - throws IOException { - - FileStatus stat1 = fileSys.getFileStatus(name1); - FileStatus stat2 = fileSys.getFileStatus(name2); - assertTrue(" Length of file " + name1 + " is " + stat1.getLen() + - " is different from length of file " + name1 + " " + stat2.getLen(), - stat1.getLen() == stat2.getLen()); - - CRC32 newcrc = new CRC32(); - FSDataInputStream stm = fileSys.open(name2); - final byte[] b = new byte[4192]; - int num = 0; - while (num >= 0) { - num = stm.read(b); - if (num < 0) { - break; - } - newcrc.update(b, 0, num); - } - stm.close(); - if (newcrc.getValue() != crc) { - fail("CRC mismatch of files " + name1 + " with file " + name2); - } - } - - @Test - public void testSuspendTraversal() throws Exception { - LOG.info("Test testSuspendTraversal started."); - long targetReplication = 2; - long metaReplication = 2; - long stripeLength = 3; - short srcReplication = 1; - - createClusters(false); - ConfigBuilder cb = new ConfigBuilder(); - cb.addPolicy("policy1", "/user/dhruba/raidtest", - srcReplication, targetReplication, metaReplication, stripeLength); - cb.persist(); - - RaidNode cnode = null; - try { - for(int i = 0; i < 4; i++){ - Path file = new Path("/user/dhruba/raidtest/dir" + i + "/file" + i); - createOldFile(fileSys, file, 1, 7, 1024L); - } - - LOG.info("Test testSuspendTraversal created test files"); - - Configuration localConf = new Configuration(conf); - localConf.setInt("raid.distraid.max.jobs", 2); - localConf.setInt("raid.distraid.max.files", 2); - localConf.setInt("raid.directorytraversal.threads", 1); - localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR); - // 4 test files: 2 jobs with 2 files each. - final int numJobsExpected = 2; - cnode = RaidNode.createRaidNode(null, localConf); - - long start = Time.now(); - final int MAX_WAITTIME = 300000; - - assertTrue("cnode is not DistRaidNode", cnode instanceof DistRaidNode); - DistRaidNode dcnode = (DistRaidNode) cnode; - - start = Time.now(); - while (dcnode.jobMonitor.jobsSucceeded() < numJobsExpected && - Time.now() - start < MAX_WAITTIME) { - LOG.info("Waiting for num jobs succeeded " + dcnode.jobMonitor.jobsSucceeded() + - " to reach " + numJobsExpected); - Thread.sleep(3000); - } - // Wait for any running jobs to finish. - start = Time.now(); - while (dcnode.jobMonitor.runningJobsCount() > 0 && - Time.now() - start < MAX_WAITTIME) { - LOG.info("Waiting for zero running jobs: " + - dcnode.jobMonitor.runningJobsCount()); - Thread.sleep(1000); - } - assertEquals(numJobsExpected, dcnode.jobMonitor.jobsMonitored()); - assertEquals(numJobsExpected, dcnode.jobMonitor.jobsSucceeded()); - - LOG.info("Test testSuspendTraversal successful."); - - } catch (Exception e) { - LOG.info("testSuspendTraversal Exception " + e + StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - stopClusters(); - } - LOG.info("Test testSuspendTraversal completed."); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java deleted file mode 100644 index 5010dcf9fca..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java +++ /dev/null @@ -1,521 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.raid.protocol.PolicyInfo; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; -import org.apache.log4j.Level; -import org.junit.Test; - -/** - * If a file gets deleted, then verify that the parity file gets deleted too. - */ -public class TestRaidPurge { - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static String CONFIG_FILE = new File(TEST_DIR, - "test-raid.xml").getAbsolutePath(); - final static long RELOAD_INTERVAL = 1000; - final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode"); - final Random rand = new Random(); - - { - ((Log4JLogger)RaidNode.LOG).getLogger().setLevel(Level.ALL); - } - - - Configuration conf; - String namenode = null; - String hftp = null; - MiniDFSCluster dfs = null; - MiniMRCluster mr = null; - FileSystem fileSys = null; - String jobTrackerName = null; - - /** - * create mapreduce and dfs clusters - */ - private void createClusters(boolean local) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - conf.set("raid.config.file", CONFIG_FILE); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.dfs.DFSClient"); - - // the RaidNode does the raiding inline (instead of submitting to map/reduce) - if (local) { - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - } else { - conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode"); - } - - conf.set("raid.server.address", "localhost:0"); - - // create a dfs and map-reduce cluster - final int taskTrackers = 4; - final int jobTrackerPort = 60050; - - dfs = new MiniDFSCluster(conf, 3, true, null); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - mr = new MiniMRCluster(taskTrackers, namenode, 3); - JobConf jobConf = mr.createJobConf(); - jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - conf.set("mapred.job.tracker", jobTrackerName); - conf.set("mapreduce.framework.name", "yarn"); - String rmAdress = jobConf.get("yarn.resourcemanager.address"); - if (rmAdress != null) { - conf.set("yarn.resourcemanager.address", rmAdress); - } - String schedulerAdress = - jobConf.get("yarn.resourcemanager.scheduler.address"); - if (schedulerAdress != null) { - conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); - } - String jobHistoryAddress = - jobConf.get("mapreduce.jobhistory.address"); - if (jobHistoryAddress != null) { - conf.set("mapreduce.jobhistory.address", jobHistoryAddress); - } - } - - /** - * create raid.xml file for RaidNode - */ - private void mySetup(long targetReplication, - long metaReplication, long stripeLength) throws Exception { - int harDelay = 1; // 1 day. - mySetup(targetReplication, metaReplication, stripeLength, harDelay); - } - - private void mySetup(long targetReplication, - long metaReplication, long stripeLength, int harDelay) throws Exception { - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - String str = " " + - " " + - " " + - "xor " + - " /destraid " + - " " + - "targetReplication " + - "" + targetReplication + " " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "" + metaReplication + " " + - " replication factor of parity file" + - " " + - " " + - " " + - "stripeLength " + - "" + stripeLength + " " + - " the max number of blocks in a file to RAID together " + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " " + - " " + - "time_before_har " + - " " + harDelay + " " + - " amount of time waited before har'ing parity files" + - " " + - " " + - "" + - "" + - ""; - fileWriter.write(str); - fileWriter.close(); - } - - /** - * stop clusters created earlier - */ - private void stopClusters() throws Exception { - if (mr != null) { mr.shutdown(); } - if (dfs != null) { dfs.shutdown(); } - } - - /** - * Test that parity files that do not have an associated master file - * get deleted. - */ - @Test - public void testPurge() throws Exception { - LOG.info("Test testPurge started."); - - long blockSizes [] = {1024L}; - long stripeLengths [] = {5}; - long targetReplication = 1; - long metaReplication = 1; - int numBlock = 9; - int iter = 0; - - createClusters(true); - try { - for (long blockSize : blockSizes) { - for (long stripeLength : stripeLengths) { - doTestPurge(iter, targetReplication, metaReplication, - stripeLength, blockSize, numBlock); - iter++; - } - } - } finally { - stopClusters(); - } - LOG.info("Test testPurge completed."); - } - - /** - * Create parity file, delete original file and then validate that - * parity file is automatically deleted. - */ - private void doTestPurge(int iter, long targetReplication, - long metaReplication, long stripeLength, - long blockSize, int numBlock) throws Exception { - LOG.info("doTestPurge started---------------------------:" + " iter " + iter + - " blockSize=" + blockSize + " stripeLength=" + stripeLength); - mySetup(targetReplication, metaReplication, stripeLength); - Path dir = new Path("/user/dhruba/raidtest/"); - Path file1 = new Path(dir + "/file" + iter); - RaidNode cnode = null; - try { - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize); - LOG.info("doTestPurge created test files for iteration " + iter); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - cnode = RaidNode.createRaidNode(null, localConf); - FileStatus[] listPaths = null; - - // wait till file is raided - while (true) { - try { - listPaths = fileSys.listStatus(destPath); - int count = 0; - if (listPaths != null && listPaths.length == 1) { - for (FileStatus s : listPaths) { - LOG.info("doTestPurge found path " + s.getPath()); - if (!s.getPath().toString().endsWith(".tmp")) { - count++; - } - } - } - if (count > 0) { - break; - } - } catch (FileNotFoundException e) { - //ignore - } - LOG.info("doTestPurge waiting for files to be raided. Found " + - (listPaths == null ? "none" : listPaths.length)); - Thread.sleep(1000); // keep waiting - } - // assertEquals(listPaths.length, 1); // all files raided - LOG.info("doTestPurge all files found in Raid."); - - // delete original file - assertTrue("Unable to delete original file " + file1 , - fileSys.delete(file1, true)); - LOG.info("deleted file " + file1); - - // wait till parity file and directory are automatically deleted - while (fileSys.exists(destPath)) { - LOG.info("doTestPurge waiting for parity files to be removed."); - Thread.sleep(1000); // keep waiting - } - - } catch (Exception e) { - LOG.info("doTestPurge Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - LOG.info("doTestPurge delete file " + file1); - fileSys.delete(file1, true); - } - LOG.info("doTestPurge completed:" + " blockSize=" + blockSize + - " stripeLength=" + stripeLength); - } - - /** - * Create a file, wait for parity file to get HARed. Then modify the file, - * wait for the HAR to get purged. - */ - @Test - public void testPurgeHar() throws Exception { - LOG.info("testPurgeHar started"); - int harDelay = 0; - createClusters(true); - mySetup(1, 1, 5, harDelay); - Path dir = new Path("/user/dhruba/raidtest/"); - Path destPath = new Path("/raid/user/dhruba/raidtest"); - Path file1 = new Path(dir + "/file"); - RaidNode cnode = null; - try { - TestRaidNode.createOldFile(fileSys, file1, 1, 8, 8192L); - LOG.info("testPurgeHar created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - cnode = RaidNode.createRaidNode(null, localConf); - - // Wait till har is created. - while (true) { - try { - FileStatus[] listPaths = listPaths = fileSys.listStatus(destPath); - if (listPaths != null && listPaths.length == 1) { - FileStatus s = listPaths[0]; - LOG.info("testPurgeHar found path " + s.getPath()); - if (s.getPath().toString().endsWith(".har")) { - break; - } - } - } catch (FileNotFoundException e) { - //ignore - } - Thread.sleep(1000); // keep waiting - } - - // Set an old timestamp. - fileSys.setTimes(file1, 0, 0); - - boolean found = false; - FileStatus[] listPaths = null; - while (!found || listPaths == null || listPaths.length > 1) { - listPaths = fileSys.listStatus(destPath); - if (listPaths != null) { - for (FileStatus s: listPaths) { - LOG.info("testPurgeHar waiting for parity file to be recreated" + - " and har to be deleted found " + s.getPath()); - if (s.getPath().toString().endsWith("file") && - s.getModificationTime() == 0) { - found = true; - } - } - } - Thread.sleep(1000); - } - } catch (Exception e) { - LOG.info("testPurgeHar Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - fileSys.delete(dir, true); - fileSys.delete(destPath, true); - stopClusters(); - } - } - - /** - * Create parity file, delete original file's directory and then validate that - * parity directory is automatically deleted. - */ - @Test - public void testPurgeDirectory() throws Exception { - long stripeLength = 5; - long blockSize = 8192; - long targetReplication = 1; - long metaReplication = 1; - int numBlock = 9; - - createClusters(true); - mySetup(targetReplication, metaReplication, stripeLength); - Path dir = new Path("/user/dhruba/raidtest/"); - Path file1 = new Path(dir + "/file1"); - RaidNode cnode = null; - try { - TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - cnode = RaidNode.createRaidNode(null, localConf); - - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - - // delete original directory. - assertTrue("Unable to delete original directory " + file1 , - fileSys.delete(file1.getParent(), true)); - LOG.info("deleted file " + file1); - - // wait till parity file and directory are automatically deleted - long start = Time.now(); - while (fileSys.exists(destPath) && - Time.now() - start < 120000) { - LOG.info("testPurgeDirectory waiting for parity files to be removed."); - Thread.sleep(1000); // keep waiting - } - assertFalse(fileSys.exists(destPath)); - - } catch (Exception e) { - LOG.info("testPurgeDirectory Exception " + e + - StringUtils.stringifyException(e)); - throw e; - } finally { - if (cnode != null) { cnode.stop(); cnode.join(); } - LOG.info("testPurgeDirectory delete file " + file1); - fileSys.delete(file1, true); - stopClusters(); - } - } - - /** - * Test that an XOR parity file is removed when a RS parity file is detected. - */ - @Test - public void testPurgePreference() throws Exception { - createClusters(true); - Path dir = new Path("/user/test/raidtest/"); - Path file1 = new Path(dir + "/file1"); - - PolicyInfo infoXor = new PolicyInfo("testPurgePreference", conf); - infoXor.setSrcPath("/user/test/raidtest"); - infoXor.setErasureCode("xor"); - infoXor.setDescription("test policy"); - infoXor.setProperty("targetReplication", "2"); - infoXor.setProperty("metaReplication", "2"); - - PolicyInfo infoRs = new PolicyInfo("testPurgePreference", conf); - infoRs.setSrcPath("/user/test/raidtest"); - infoRs.setErasureCode("rs"); - infoRs.setDescription("test policy"); - infoRs.setProperty("targetReplication", "1"); - infoRs.setProperty("metaReplication", "1"); - try { - TestRaidNode.createOldFile(fileSys, file1, 1, 9, 8192L); - FileStatus stat = fileSys.getFileStatus(file1); - - // Create the parity files. - RaidNode.doRaid( - conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL); - RaidNode.doRaid( - conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL); - Path xorParity = - new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/file1"); - Path rsParity = - new Path(RaidNode.DEFAULT_RAIDRS_LOCATION, "user/test/raidtest/file1"); - assertTrue(fileSys.exists(xorParity)); - assertTrue(fileSys.exists(rsParity)); - - // Check purge of a single parity file. - RaidNode cnode = RaidNode.createRaidNode(conf); - FileStatus raidRsStat = - fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAIDRS_LOCATION)); - cnode.purgeMonitor.recursePurge(infoRs.getErasureCode(), fileSys, fileSys, - RaidNode.DEFAULT_RAIDRS_LOCATION, raidRsStat); - - // Calling purge under the RS path has no effect. - assertTrue(fileSys.exists(xorParity)); - assertTrue(fileSys.exists(rsParity)); - - FileStatus raidStat = - fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAID_LOCATION)); - cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys, - RaidNode.DEFAULT_RAID_LOCATION, raidStat); - // XOR parity must have been purged by now. - assertFalse(fileSys.exists(xorParity)); - assertTrue(fileSys.exists(rsParity)); - - // Now check the purge of a parity har. - // Delete the RS parity for now. - fileSys.delete(rsParity); - // Recreate the XOR parity. - Path xorHar = - new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/raidtest" + - RaidNode.HAR_SUFFIX); - RaidNode.doRaid( - conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL); - assertTrue(fileSys.exists(xorParity)); - assertFalse(fileSys.exists(xorHar)); - - // Create the har. - long cutoff = Time.now(); - cnode.recurseHar(infoXor, fileSys, raidStat, - RaidNode.DEFAULT_RAID_LOCATION, fileSys, cutoff, - RaidNode.tmpHarPathForCode(conf, infoXor.getErasureCode())); - - // Call purge to get rid of the parity file. The har should remain. - cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys, - RaidNode.DEFAULT_RAID_LOCATION, raidStat); - // XOR har should exist but xor parity file should have been purged. - assertFalse(fileSys.exists(xorParity)); - assertTrue(fileSys.exists(xorHar)); - - // Now create the RS parity. - RaidNode.doRaid( - conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL); - cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys, - RaidNode.DEFAULT_RAID_LOCATION, raidStat); - // XOR har should get deleted. - assertTrue(fileSys.exists(rsParity)); - assertFalse(fileSys.exists(xorParity)); - assertFalse(fileSys.exists(xorHar)); - - } finally { - stopClusters(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java deleted file mode 100644 index 1245e90b24f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Random; -import java.util.zip.CRC32; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.util.ToolRunner; -import org.junit.Test; - - -public class TestRaidShell { - final static Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.TestRaidShell"); - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static String CONFIG_FILE = new File(TEST_DIR, - "test-raid.xml").getAbsolutePath(); - final static long RELOAD_INTERVAL = 1000; - final static int NUM_DATANODES = 3; - Configuration conf; - String namenode = null; - MiniDFSCluster dfs = null; - String hftp = null; - FileSystem fileSys = null; - RaidNode cnode = null; - Random rand = new Random(); - - /** - * Create a file with three stripes, corrupt a block each in two stripes, - * and wait for the the file to be fixed. - */ - @Test - public void testBlockFix() throws Exception { - LOG.info("Test testBlockFix started."); - long blockSize = 8192L; - int stripeLength = 3; - mySetup(stripeLength, -1); - Path file1 = new Path("/user/dhruba/raidtest/file1"); - Path destPath = new Path("/destraid/user/dhruba/raidtest"); - Path parityFile = new Path(destPath, "file1"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 7, blockSize); - long file1Len = fileSys.getFileStatus(file1).getLen(); - LOG.info("Test testBlockFix created test files"); - - // create an instance of the RaidNode - Configuration localConf = new Configuration(conf); - localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid"); - localConf.setInt("raid.blockfix.interval", 1000); - // the RaidNode does the raiding inline (instead of submitting to map/reduce) - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - conf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - cnode = RaidNode.createRaidNode(null, localConf); - - try { - TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath); - cnode.stop(); - cnode.join(); - cnode = null; - - FileStatus srcStat = fileSys.getFileStatus(file1); - LocatedBlocks locations = RaidDFSUtil.getBlockLocations( - (DistributedFileSystem) fileSys, file1.toUri().getPath(), - 0, srcStat.getLen()); - - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - - // Corrupt blocks in different stripes. We can fix them. - int[] corruptBlockIdxs = new int[]{0, 4, 6}; - for (int idx: corruptBlockIdxs) { - LOG.info("Corrupting block " + locations.get(idx).getBlock()); - corruptBlock(locations.get(idx).getBlock()); - } - TestBlockFixer.reportCorruptBlocks(fileSys, file1, corruptBlockIdxs, - srcStat.getBlockSize()); - - waitForCorruptBlocks(corruptBlockIdxs.length, dfs, file1); - - // Create RaidShell and fix the file. - RaidShell shell = new RaidShell(conf); - String[] args = new String[2]; - args[0] = "-recoverBlocks"; - args[1] = file1.toUri().getPath(); - ToolRunner.run(shell, args); - - waitForCorruptBlocks(0, dfs, file1); - - assertTrue(TestRaidDfs.validateFile(dfs, file1, file1Len, crc1)); - - // Now corrupt and fix the parity file. - FileStatus parityStat = fileSys.getFileStatus(parityFile); - long parityCrc = getCRC(fileSys, parityFile); - locations = RaidDFSUtil.getBlockLocations( - dfs, parityFile.toUri().getPath(), 0, parityStat.getLen()); - corruptBlock(locations.get(0).getBlock()); - TestBlockFixer.reportCorruptBlocks(fileSys, parityFile, new int[]{0}, - srcStat.getBlockSize()); - waitForCorruptBlocks(1, dfs, parityFile); - - args[1] = parityFile.toUri().getPath(); - ToolRunner.run(shell, args); - - waitForCorruptBlocks(0, dfs, file1); - assertEquals(parityCrc, getCRC(fileSys, parityFile)); - - } catch (Exception e) { - LOG.info("Test testBlockFix Exception " + e + StringUtils.stringifyException(e)); - throw e; - } finally { - myTearDown(); - } - LOG.info("Test testBlockFix completed."); - } - - private void waitForCorruptBlocks( - int numCorruptBlocks, DistributedFileSystem dfs, Path file) - throws Exception { - String path = file.toUri().getPath(); - FileStatus stat = dfs.getFileStatus(file); - long start = Time.now(); - long actual = 0; - do { - actual = RaidDFSUtil.corruptBlocksInFile( - dfs, path, 0, stat.getLen()).size(); - if (actual == numCorruptBlocks) break; - if (Time.now() - start > 120000) break; - LOG.info("Waiting for " + numCorruptBlocks + " corrupt blocks in " + - path + ", found " + actual); - Thread.sleep(1000); - } while (true); - assertEquals(numCorruptBlocks, actual); - } - - private void mySetup(int stripeLength, int timeBeforeHar) throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - conf.set("raid.config.file", CONFIG_FILE); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // do not use map-reduce cluster for Raiding - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - conf.set("raid.server.address", "localhost:0"); - conf.setInt("hdfs.raid.stripeLength", stripeLength); - conf.set("hdfs.raid.locations", "/destraid"); - - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - - FileSystem.setDefaultUri(conf, namenode); - hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); - - FileSystem.setDefaultUri(conf, namenode); - - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - String str = " " + - " " + - " " + - "xor " + - " /destraid " + - " " + - "targetReplication " + - "1 " + - "after RAIDing, decrease the replication factor of a file to this value." + - " " + - " " + - " " + - "metaReplication " + - "1 " + - " replication factor of parity file" + - " " + - " " + - " " + - "modTimePeriod " + - "2000 " + - " time (milliseconds) after a file is modified to make it " + - "a candidate for RAIDing " + - " " + - " "; - if (timeBeforeHar >= 0) { - str += - " " + - "time_before_har " + - "" + timeBeforeHar + " " + - " amount of time waited before har'ing parity files" + - " " + - " "; - } - - str += - "" + - "" + - ""; - fileWriter.write(str); - fileWriter.close(); - } - - private void myTearDown() throws Exception { - if (cnode != null) { cnode.stop(); cnode.join(); } - if (dfs != null) { dfs.shutdown(); } - } - - private long getCRC(FileSystem fs, Path p) throws IOException { - CRC32 crc = new CRC32(); - FSDataInputStream stm = fs.open(p); - int b; - while ((b = stm.read())>=0) { - crc.update(b); - } - stm.close(); - return crc.getValue(); - } - - void corruptBlock(ExtendedBlock block) throws IOException { - assertTrue("Could not corrupt block", - dfs.corruptBlockOnDataNodes(block) > 0); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java deleted file mode 100644 index bf7cdc05ebc..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java +++ /dev/null @@ -1,724 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http:www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.Test; - - -public class TestRaidShellFsck { - final static Log LOG = - LogFactory.getLog("org.apache.hadoop.raid.TestRaidShellFsck"); - final static String TEST_DIR = - new File(System. - getProperty("test.build.data", "target/test-data")).getAbsolutePath(); - - final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml"). - getAbsolutePath(); - final static long RELOAD_INTERVAL = 1000; - final static int NUM_DATANODES = 4; - final static int STRIPE_BLOCKS = 3; // number of blocks per stripe - final static int FILE_BLOCKS = 6; // number of blocks that file consists of - final static short REPL = 1; // replication factor before raiding - final static long BLOCK_SIZE = 8192L; // size of block in byte - final static String DIR_PATH = "/user/pkling/raidtest"; - final static Path FILE_PATH0 = - new Path("/user/pkling/raidtest/raidfsck.test"); - final static Path FILE_PATH1 = - new Path("/user/pkling/raidtest/raidfsck2.test"); - final static Path RAID_PATH = new Path("/destraid/user/pkling/raidtest"); - final static String HAR_NAME = "raidtest_raid.har"; - final static String RAID_DIR = "/destraid"; - - Configuration conf = null; - Configuration raidConf = null; - Configuration clientConf = null; - MiniDFSCluster cluster = null; - DistributedFileSystem dfs = null; - RaidNode rnode = null; - - - RaidShell shell = null; - String[] args = null; - - - /** - * creates a MiniDFS instance with a raided file in it - */ - private void setUp(boolean doHar) throws IOException, ClassNotFoundException { - - final int timeBeforeHar; - if (doHar) { - timeBeforeHar = 0; - } else { - timeBeforeHar = -1; - } - - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - conf.set("raid.config.file", CONFIG_FILE); - conf.setBoolean("raid.config.reload", true); - conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL); - - // scan all policies once every 5 second - conf.setLong("raid.policy.rescan.interval", 5000); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - // do not use map-reduce cluster for Raiding - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - // use local block fixer - conf.set("raid.blockfix.classname", - "org.apache.hadoop.raid.LocalBlockFixer"); - - conf.set("raid.server.address", "localhost:0"); - conf.setInt("hdfs.raid.stripeLength", STRIPE_BLOCKS); - conf.set("hdfs.raid.locations", RAID_DIR); - - conf.setInt("dfs.corruptfilesreturned.max", 500); - - conf.setBoolean("dfs.permissions", false); - - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) - .build(); - cluster.waitActive(); - dfs = (DistributedFileSystem) cluster.getFileSystem(); - String namenode = dfs.getUri().toString(); - - FileSystem.setDefaultUri(conf, namenode); - - FileWriter fileWriter = new FileWriter(CONFIG_FILE); - fileWriter.write("\n"); - String str = - " " + - " " + - " " + - " xor " + - " " + RAID_DIR + " " + - " " + - " targetReplication " + - " 1 " + - " after RAIDing, decrease the replication " + - "factor of a file to this value. " + - " " + - " " + - " metaReplication " + - " 1 " + - " replication factor of parity file " + - " " + - " " + - " modTimePeriod " + - " 2000 " + - " time (milliseconds) after a file is modified " + - "to make it a candidate for RAIDing " + - " "; - - if (timeBeforeHar >= 0) { - str += - " " + - " time_before_har " + - " " + timeBeforeHar + " " + - " amount of time waited before har'ing parity " + - "files " + - " "; - } - - str += - " " + - " " + - ""; - - fileWriter.write(str); - fileWriter.close(); - - createTestFile(FILE_PATH0); - createTestFile(FILE_PATH1); - - Path[] filePaths = { FILE_PATH0, FILE_PATH1 }; - raidTestFiles(RAID_PATH, filePaths, doHar); - - clientConf = new Configuration(raidConf); - clientConf.set("fs.hdfs.impl", - "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); - clientConf.set("fs.raid.underlyingfs.impl", - "org.apache.hadoop.hdfs.DistributedFileSystem"); - - // prepare shell and arguments - shell = new RaidShell(clientConf); - args = new String[2]; - args[0] = "-fsck"; - args[1] = DIR_PATH; - - } - - /** - * Creates test file consisting of random data - */ - private void createTestFile(Path filePath) throws IOException { - Random rand = new Random(); - FSDataOutputStream stm = dfs.create(filePath, true, - conf.getInt("io.file.buffer.size", - 4096), REPL, BLOCK_SIZE); - - final byte[] b = new byte[(int) BLOCK_SIZE]; - for (int i = 0; i < FILE_BLOCKS; i++) { - rand.nextBytes(b); - stm.write(b); - } - stm.close(); - LOG.info("test file created"); - - } - - /** - * raids test file - */ - private void raidTestFiles(Path raidPath, Path[] filePaths, boolean doHar) - throws IOException, ClassNotFoundException { - // create RaidNode - raidConf = new Configuration(conf); - raidConf.set(RaidNode.RAID_LOCATION_KEY, RAID_DIR); - raidConf.setInt("raid.blockfix.interval", 1000); - raidConf.setLong("har.block.size", BLOCK_SIZE * 3); - // the RaidNode does the raiding inline (instead of submitting to MR node) - conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); - rnode = RaidNode.createRaidNode(null, raidConf); - - for (Path filePath: filePaths) { - long waitStart = Time.now(); - boolean raided = false; - - Path parityFilePath = new Path(RAID_DIR, - filePath.toString().substring(1)); - - while (!raided) { - try { - FileStatus[] listPaths = dfs.listStatus(raidPath); - if (listPaths != null) { - if (doHar) { - // case with HAR - for (FileStatus f: listPaths) { - if (f.getPath().toString().endsWith(".har")) { - // check if the parity file is in the index - final Path indexPath = new Path(f.getPath(), "_index"); - final FileStatus indexFileStatus = - dfs.getFileStatus(indexPath); - final HarIndex harIndex = - new HarIndex(dfs.open(indexPath), indexFileStatus.getLen()); - final HarIndex.IndexEntry indexEntry = - harIndex.findEntryByFileName(parityFilePath.toString()); - if (indexEntry != null) { - LOG.info("raid file " + parityFilePath.toString() + - " found in Har archive: " + - f.getPath().toString() + - " ts=" + indexEntry.mtime); - raided = true; - break; - } - } - } - - } else { - // case without HAR - for (FileStatus f : listPaths) { - Path found = new Path(f.getPath().toUri().getPath()); - if (parityFilePath.equals(found)) { - LOG.info("raid file found: " + f.getPath().toString()); - raided = true; - break; - } - } - } - } - } catch (FileNotFoundException ignore) { - } - if (!raided) { - if (Time.now() > waitStart + 40000L) { - LOG.error("parity file not created after 40s"); - throw new IOException("parity file not HARed after 40s"); - } else { - try { - Thread.sleep(1000); - } catch (InterruptedException ignore) { - } - } - } - } - } - - rnode.stop(); - rnode.join(); - rnode = null; - LOG.info("test file raided"); - } - - /** - * sleeps for up to 20s until the number of corrupt files - * in the file system is equal to the number specified - */ - private void waitUntilCorruptFileCount(DistributedFileSystem dfs, - int corruptFiles) - throws IOException { - long waitStart = Time.now(); - while (RaidDFSUtil.getCorruptFiles(dfs).length != corruptFiles) { - try { - Thread.sleep(1000); - } catch (InterruptedException ignore) { - - } - - if (Time.now() > waitStart + 20000L) { - break; - } - } - - int corruptFilesFound = RaidDFSUtil.getCorruptFiles(dfs).length; - if (corruptFilesFound != corruptFiles) { - throw new IOException("expected " + corruptFiles + - " corrupt files but got " + - corruptFilesFound); - } - } - - /** - * removes a specified block from MiniDFS storage and reports it as corrupt - */ - private void removeAndReportBlock(DistributedFileSystem blockDfs, - Path filePath, - LocatedBlock block) - throws IOException { - TestRaidDfs.corruptBlock(cluster, filePath, block.getBlock(), NUM_DATANODES, true); - - // report deleted block to the name node - LocatedBlock[] toReport = { block }; - blockDfs.getClient().getNamenode().reportBadBlocks(toReport); - - } - - - /** - * removes a file block in the specified stripe - */ - private void removeFileBlock(Path filePath, int stripe, int blockInStripe) - throws IOException { - LocatedBlocks fileBlocks = dfs.getClient().getNamenode(). - getBlockLocations(filePath.toString(), 0, FILE_BLOCKS * BLOCK_SIZE); - if (fileBlocks.locatedBlockCount() != FILE_BLOCKS) { - throw new IOException("expected " + FILE_BLOCKS + - " file blocks but found " + - fileBlocks.locatedBlockCount()); - } - if (blockInStripe >= STRIPE_BLOCKS) { - throw new IOException("blockInStripe is " + blockInStripe + - " but must be smaller than " + STRIPE_BLOCKS); - } - LocatedBlock block = fileBlocks.get(stripe * STRIPE_BLOCKS + blockInStripe); - removeAndReportBlock(dfs, filePath, block); - LOG.info("removed file " + filePath.toString() + " block " + - stripe * STRIPE_BLOCKS + " in stripe " + stripe); - } - - /** - * removes a parity block in the specified stripe - */ - private void removeParityBlock(Path filePath, int stripe) throws IOException { - // find parity file - Path destPath = new Path(RAID_DIR); - RaidNode.ParityFilePair ppair = null; - - ppair = RaidNode.getParityFile(destPath, filePath, conf); - String parityPathStr = ppair.getPath().toUri().getPath(); - LOG.info("parity path: " + parityPathStr); - FileSystem parityFS = ppair.getFileSystem(); - if (!(parityFS instanceof DistributedFileSystem)) { - throw new IOException("parity file is not on distributed file system"); - } - DistributedFileSystem parityDFS = (DistributedFileSystem) parityFS; - - - // now corrupt the block corresponding to the stripe selected - FileStatus parityFileStatus = - parityDFS.getFileStatus(new Path(parityPathStr)); - long parityBlockSize = parityFileStatus.getBlockSize(); - long parityFileLength = parityFileStatus.getLen(); - long parityFileLengthInBlocks = (parityFileLength / parityBlockSize) + - (((parityFileLength % parityBlockSize) == 0) ? 0L : 1L); - if (parityFileLengthInBlocks <= stripe) { - throw new IOException("selected stripe " + stripe + - " but parity file only has " + - parityFileLengthInBlocks + " blocks"); - } - if (parityBlockSize != BLOCK_SIZE) { - throw new IOException("file block size is " + BLOCK_SIZE + - " but parity file block size is " + - parityBlockSize); - } - LocatedBlocks parityFileBlocks = parityDFS.getClient().getNamenode(). - getBlockLocations(parityPathStr, 0, parityFileLength); - if (parityFileBlocks.locatedBlockCount() != parityFileLengthInBlocks) { - throw new IOException("expected " + parityFileLengthInBlocks + - " parity file blocks but got " + - parityFileBlocks.locatedBlockCount() + - " blocks"); - } - LocatedBlock parityFileBlock = parityFileBlocks.get(stripe); - removeAndReportBlock(parityDFS, new Path(parityPathStr), parityFileBlock); - LOG.info("removed parity file block/stripe " + stripe + - " for " + filePath.toString()); - - } - - /** - * removes a block from the har part file - */ - private void removeHarParityBlock(int block) throws IOException { - Path harPath = new Path(RAID_PATH, HAR_NAME); - FileStatus [] listPaths = dfs.listStatus(harPath); - - boolean deleted = false; - - for (FileStatus f: listPaths) { - if (f.getPath().getName().startsWith("part-")) { - final Path partPath = new Path(f.getPath().toUri().getPath()); - final LocatedBlocks partBlocks = dfs.getClient().getNamenode(). - getBlockLocations(partPath.toString(), - 0, - f.getLen()); - - if (partBlocks.locatedBlockCount() <= block) { - throw new IOException("invalid har block " + block); - } - - final LocatedBlock partBlock = partBlocks.get(block); - removeAndReportBlock(dfs, partPath, partBlock); - LOG.info("removed block " + block + "/" + - partBlocks.locatedBlockCount() + - " of file " + partPath.toString() + - " block size " + partBlock.getBlockSize()); - deleted = true; - break; - } - } - - if (!deleted) { - throw new IOException("cannot find part file in " + harPath.toString()); - } - } - - /** - * checks fsck with no missing blocks - */ - @Test - public void testClean() throws Exception { - LOG.info("testClean"); - setUp(false); - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - - /** - * checks fsck with missing block in file block but not in parity block - */ - @Test - public void testFileBlockMissing() throws Exception { - LOG.info("testFileBlockMissing"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 0, 0); - waitUntilCorruptFileCount(dfs, 1); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - /** - * checks fsck with missing block in parity block but not in file block - */ - @Test - public void testParityBlockMissing() throws Exception { - LOG.info("testParityBlockMissing"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeParityBlock(FILE_PATH0, 0); - waitUntilCorruptFileCount(dfs, 1); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - /** - * checks fsck with missing block in both file block and parity block - * in different stripes - */ - @Test - public void testFileBlockAndParityBlockMissingInDifferentStripes() - throws Exception { - LOG.info("testFileBlockAndParityBlockMissingInDifferentStripes"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 0, 0); - waitUntilCorruptFileCount(dfs, 1); - removeParityBlock(FILE_PATH0, 1); - waitUntilCorruptFileCount(dfs, 2); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - /** - * checks fsck with missing block in both file block and parity block - * in same stripe - */ - @Test - public void testFileBlockAndParityBlockMissingInSameStripe() - throws Exception { - LOG.info("testFileBlockAndParityBlockMissingInSameStripe"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeParityBlock(FILE_PATH0, 1); - waitUntilCorruptFileCount(dfs, 1); - removeFileBlock(FILE_PATH0, 1, 0); - waitUntilCorruptFileCount(dfs, 2); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 1, but returns " + - Integer.toString(result), result == 1); - } - - /** - * checks fsck with two missing file blocks in same stripe - */ - @Test - public void test2FileBlocksMissingInSameStripe() - throws Exception { - LOG.info("test2FileBlocksMissingInSameStripe"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 1, 1); - waitUntilCorruptFileCount(dfs, 1); - removeFileBlock(FILE_PATH0, 1, 0); - waitUntilCorruptFileCount(dfs, 1); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 1, but returns " + - Integer.toString(result), result == 1); - } - - /** - * checks fsck with two missing file blocks in different stripes - */ - @Test - public void test2FileBlocksMissingInDifferentStripes() - throws Exception { - LOG.info("test2FileBlocksMissingInDifferentStripes"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 1, 1); - waitUntilCorruptFileCount(dfs, 1); - removeFileBlock(FILE_PATH0, 0, 0); - waitUntilCorruptFileCount(dfs, 1); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - /** - * checks fsck with file block missing (HAR) - * use 2 files to verify HAR offset logic in RaidShell fsck - * both files have one corrupt block, parity blocks are clean - * - * parity blocks in har (file.stripe): - * +-----+-----+-----+ +-----+ - * | 0.0 | 0.1 | 1.0 | | 1.1 | - * +-----+-----+-----+ +-----+ - * 0 1 - * - */ - @Test - public void testFileBlockMissingHar() - throws Exception { - LOG.info("testFileBlockMissingHar"); - setUp(true); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 1, 1); - removeFileBlock(FILE_PATH1, 1, 1); - waitUntilCorruptFileCount(dfs, 2); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - /** - * checks fsck with file block missing (HAR) - * use 2 files to verify HAR offset logic in RaidShell fsck - * - * parity blocks in har (file.stripe): - * +-----+-----+-----+ +-----+ - * | 0.0 | 0.1 | 1.0 | | 1.1 | - * +-----+-----+-----+ +-----+ - * 0 1 - * - * corrupt file 0, stripe 0 file block 0 - * corrupt file 0, stripe 1 file block 0 - * corrupt file 1, stripe 0 file block 0 - * corrupt file 1, stripe 1 file block 0 - * corrupt har block 0 - * both files should be corrupt - */ - @Test - public void testFileBlockAndParityBlockMissingHar1() - throws Exception { - LOG.info("testFileBlockAndParityBlockMissingHar1"); - setUp(true); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 0, 0); - removeFileBlock(FILE_PATH0, 1, 0); - removeFileBlock(FILE_PATH1, 0, 0); - removeFileBlock(FILE_PATH1, 1, 0); - removeHarParityBlock(0); - waitUntilCorruptFileCount(dfs, 3); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 2, but returns " + - Integer.toString(result), result == 2); - } - - /** - * checks fsck with file block missing (HAR) - * use 2 files to verify HAR offset logic in RaidShell fsck - * - * parity blocks in har (file.stripe): - * +-----+-----+-----+ +-----+ - * | 0.0 | 0.1 | 1.0 | | 1.1 | - * +-----+-----+-----+ +-----+ - * 0 1 - * - * corrupt file 0, stripe 0 file block 0 - * corrupt file 0, stripe 1 file block 0 - * corrupt file 1, stripe 0 file block 0 - * corrupt file 1, stripe 1 file block 0 - * corrupt har block 1 - * only file 2 should be corrupt - */ - @Test - public void testFileBlockAndParityBlockMissingHar2() - throws Exception { - LOG.info("testFileBlockAndParityBlockMissingHar2"); - setUp(true); - waitUntilCorruptFileCount(dfs, 0); - removeFileBlock(FILE_PATH0, 0, 0); - removeFileBlock(FILE_PATH0, 1, 0); - removeFileBlock(FILE_PATH1, 0, 0); - removeFileBlock(FILE_PATH1, 1, 0); - removeHarParityBlock(1); - waitUntilCorruptFileCount(dfs, 3); - - int result = ToolRunner.run(shell, args); - - assertTrue("fsck should return 1, but returns " + - Integer.toString(result), result == 1); - } - - /** - * checks that fsck does not report corrupt file that is not in - * the specified path - */ - @Test - public void testPathFilter() - throws Exception { - LOG.info("testPathFilter"); - setUp(false); - waitUntilCorruptFileCount(dfs, 0); - removeParityBlock(FILE_PATH0, 1); - waitUntilCorruptFileCount(dfs, 1); - removeFileBlock(FILE_PATH0, 1, 0); - waitUntilCorruptFileCount(dfs, 2); - - String[] otherArgs = new String[2]; - otherArgs[0] = "-fsck"; - otherArgs[1] = "/user/pkling/other"; - int result = ToolRunner.run(shell, otherArgs); - - assertTrue("fsck should return 0, but returns " + - Integer.toString(result), result == 0); - } - - - @After - public void tearDown() throws Exception { - if (rnode != null) { - rnode.stop(); - rnode.join(); - rnode = null; - } - - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - - dfs = null; - - LOG.info("Test cluster shut down"); - } - - -} - diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java deleted file mode 100644 index d51617569cf..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.RaidDFSUtil; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.mapred.Reporter; -import org.junit.Test; - - -public class TestReedSolomonDecoder { - final static Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.TestReedSolomonDecoder"); - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static int NUM_DATANODES = 3; - - Configuration conf; - MiniDFSCluster dfs = null; - FileSystem fileSys = null; - - @Test - public void testDecoder() throws Exception { - mySetup(); - int stripeSize = 10; - int paritySize = 4; - long blockSize = 8192; - Path file1 = new Path("/user/raidtest/file1"); - Path recoveredFile1 = new Path("/user/raidtest/file1.recovered"); - Path parityFile1 = new Path("/rsraid/user/raidtest/file1"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 25, blockSize); - FileStatus file1Stat = fileSys.getFileStatus(file1); - - conf.setInt("raid.rsdecoder.bufsize", 512); - conf.setInt("raid.rsencoder.bufsize", 512); - - try { - // First encode the file. - ReedSolomonEncoder encoder = new ReedSolomonEncoder( - conf, stripeSize, paritySize); - short parityRepl = 1; - encoder.encodeFile(fileSys, file1, fileSys, parityFile1, parityRepl, - Reporter.NULL); - - // Ensure there are no corrupt files yet. - DistributedFileSystem dfs = (DistributedFileSystem)fileSys; - String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals(corruptFiles.length, 0); - - // Now corrupt the file. - long corruptOffset = blockSize * 5; - FileStatus srcStat = fileSys.getFileStatus(file1); - LocatedBlocks locations = RaidDFSUtil.getBlockLocations(dfs, - file1.toUri().getPath(), 0, srcStat.getLen()); - corruptBlock(locations.get(5).getBlock()); - corruptBlock(locations.get(6).getBlock()); - TestBlockFixer.reportCorruptBlocks(dfs, file1, new int[]{5, 6}, - srcStat.getBlockSize()); - - // Ensure file is corrupted. - corruptFiles = RaidDFSUtil.getCorruptFiles(dfs); - assertEquals(corruptFiles.length, 1); - assertEquals(corruptFiles[0], file1.toString()); - - // Fix the file. - ReedSolomonDecoder decoder = new ReedSolomonDecoder( - conf, stripeSize, paritySize); - decoder.decodeFile(fileSys, file1, fileSys, parityFile1, - corruptOffset, recoveredFile1); - assertTrue(TestRaidDfs.validateFile( - fileSys, recoveredFile1, file1Stat.getLen(), crc1)); - } finally { - myTearDown(); - } - } - - void corruptBlock(ExtendedBlock block) throws IOException { - assertTrue("Could not corrupt block", - dfs.corruptBlockOnDataNodes(block) > 0); - } - - private void mySetup() throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - conf.setBoolean("dfs.permissions", false); - - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - String namenode = fileSys.getUri().toString(); - FileSystem.setDefaultUri(conf, namenode); - } - - private void myTearDown() throws Exception { - if (dfs != null) { dfs.shutdown(); } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java b/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java deleted file mode 100644 index 86111d05e00..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.raid; - -import static org.junit.Assert.assertEquals; - -import java.io.File; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.TestRaidDfs; -import org.apache.hadoop.mapred.Reporter; -import org.junit.Test; - - -public class TestReedSolomonEncoder { - final static Log LOG = LogFactory.getLog( - "org.apache.hadoop.raid.TestReedSolomonEncoder"); - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "target/test-data")).getAbsolutePath(); - final static int NUM_DATANODES = 3; - - Configuration conf; - String namenode = null; - MiniDFSCluster dfs = null; - FileSystem fileSys = null; - - @Test - public void testEncoder() throws Exception { - mySetup(); - int stripeSize = 10; - int paritySize = 4; - long blockSize = 8192; - Path file1 = new Path("/user/raidtest/file1"); - Path parityFile1 = new Path("/rsraid/user/raidtest/file1"); - long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, - 1, 25, blockSize); - try { - ReedSolomonEncoder encoder = new ReedSolomonEncoder( - conf, stripeSize, paritySize); - short parityRepl = 1; - encoder.encodeFile(fileSys, file1, fileSys, parityFile1, parityRepl, - Reporter.NULL); - - FileStatus parityStat = fileSys.getFileStatus(parityFile1); - assertEquals(4*8192*3, parityStat.getLen()); - - } finally { - myTearDown(); - } - } - - private void mySetup() throws Exception { - - new File(TEST_DIR).mkdirs(); // Make sure data directory exists - conf = new Configuration(); - - // make all deletions not go through Trash - conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient"); - - dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null); - dfs.waitActive(); - fileSys = dfs.getFileSystem(); - namenode = fileSys.getUri().toString(); - - FileSystem.setDefaultUri(conf, namenode); - - } - - private void myTearDown() throws Exception { - if (dfs != null) { dfs.shutdown(); } - } -} diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 38c93fa8bd1..27161004a36 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -34,7 +34,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs hadoop-hdfs-httpfs hadoop-hdfs/src/contrib/bkjournal - hadoop-hdfs-raid diff --git a/hadoop-mapreduce-project/build.xml b/hadoop-mapreduce-project/build.xml index 6a8a7d0be1f..286af16530b 100644 --- a/hadoop-mapreduce-project/build.xml +++ b/hadoop-mapreduce-project/build.xml @@ -1711,10 +1711,6 @@ output="${build.dir.eclipse-contrib-classes}/gridmix/main" /> - - - Tests failed! diff --git a/hadoop-mapreduce-project/src/contrib/raid/README b/hadoop-mapreduce-project/src/contrib/raid/README deleted file mode 100644 index b1662b42bef..00000000000 --- a/hadoop-mapreduce-project/src/contrib/raid/README +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2008 The Apache Software Foundation Licensed under the -# Apache License, Version 2.0 (the "License"); you may not use this -# file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless -# required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. - -This package implements a Distributed Raid File System. It is used alongwith -an instance of the Hadoop Distributed File System (HDFS). It can be used to -provide better protection against data corruption. It can also be used to -reduce the total storage requirements of HDFS. - -Distributed Raid File System consists of two main software components. The first component -is the RaidNode, a daemon that creates parity files from specified HDFS files. -The second component "raidfs" is a software that is layered over a HDFS client and it -intercepts all calls that an application makes to the HDFS client. If HDFS encounters -corrupted data while reading a file, the raidfs client detects it; it uses the -relevant parity blocks to recover the corrupted data (if possible) and returns -the data to the application. The application is completely transparent to the -fact that parity data was used to satisfy it's read request. - -The primary use of this feature is to save disk space for HDFS files. -HDFS typically stores data in triplicate. -The Distributed Raid File System can be configured in such a way that a set of -data blocks of a file are combined together to form one or more parity blocks. -This allows one to reduce the replication factor of a HDFS file from 3 to 2 -while keeping the failure probabilty relatively same as before. This typically -results in saving 25% to 30% of storage space in a HDFS cluster. - --------------------------------------------------------------------------------- - -BUILDING: - -In HADOOP_PREFIX, run ant package to build Hadoop and its contrib packages. - --------------------------------------------------------------------------------- - -INSTALLING and CONFIGURING: - -The entire code is packaged in the form of a single jar file hadoop-*-raid.jar. -To use HDFS Raid, you need to put the above mentioned jar file on -the CLASSPATH. The easiest way is to copy the hadoop-*-raid.jar -from HADOOP_PREFIX/build/contrib/raid to HADOOP_PREFIX/lib. Alternatively -you can modify HADOOP_CLASSPATH to include this jar, in conf/hadoop-env.sh. - -There is a single configuration file named raid.xml that describes the HDFS -path(s) that you want to raid. A sample of this file can be found in -sc/contrib/raid/conf/raid.xml. Please edit the entries in this file to list the -path(s) that you want to raid. Then, edit the hdfs-site.xml file for -your installation to include a reference to this raid.xml. You can add the -following to your hdfs-site.xml - - raid.config.file - /mnt/hdfs/DFS/conf/raid.xml - This is needed by the RaidNode - - -Please add an entry to your hdfs-site.xml to enable hdfs clients to use the -parity bits to recover corrupted data. - - - fs.hdfs.impl - org.apache.hadoop.dfs.DistributedRaidFileSystem - The FileSystem for hdfs: uris. - - - --------------------------------------------------------------------------------- - -OPTIONAL CONFIGIURATION: - -The following properties can be set in hdfs-site.xml to further tune you configuration: - - Specifies the location where parity files are located. - - hdfs.raid.locations - hdfs://newdfs.data:8000/raid - The location for parity files. If this is - is not defined, then defaults to /raid. - - - - Specify the parity stripe length - - hdfs.raid.stripeLength - 10 - The number of blocks in a file to be combined into - a single raid parity block. The default value is 5. The lower - the number the greater is the disk space you will save when you - enable raid. - - - - Specify the size of HAR part-files - - raid.har.partfile.size - 4294967296 - The size of HAR part files that store raid parity - files. The default is 4GB. The higher the number the fewer the - number of files used to store the HAR archive. - - - - Specify which implementation of RaidNode to use. - - raid.classname - org.apache.hadoop.raid.DistRaidNode - Specify which implementation of RaidNode to use - (class name). - - - - - Specify the periodicy at which the RaidNode re-calculates (if necessary) - the parity blocks - - raid.policy.rescan.interval - 5000 - Specify the periodicity in milliseconds after which - all source paths are rescanned and parity blocks recomputed if - necessary. By default, this value is 1 hour. - - - - By default, the DistributedRaidFileSystem assumes that the underlying file - system is the DistributedFileSystem. If you want to layer the DistributedRaidFileSystem - over some other file system, then define a property named fs.raid.underlyingfs.impl - that specifies the name of the underlying class. For example, if you want to layer - The DistributedRaidFileSystem over an instance of the NewFileSystem, then - - fs.raid.underlyingfs.impl - org.apche.hadoop.new.NewFileSystem - Specify the filesystem that is layered immediately below the - DistributedRaidFileSystem. By default, this value is DistributedFileSystem. - - - --------------------------------------------------------------------------------- - -ADMINISTRATION: - -The Distributed Raid File System provides support for administration at runtime without -any downtime to cluster services. It is possible to add/delete new paths to be raided without -interrupting any load on the cluster. If you change raid.xml, its contents will be -reload within seconds and the new contents will take effect immediately. - -Designate one machine in your cluster to run the RaidNode software. You can run this daemon -on any machine irrespective of whether that machine is running any other hadoop daemon or not. -You can start the RaidNode by running the following on the selected machine: -nohup $HADOOP_PREFIX/bin/hadoop org.apache.hadoop.raid.RaidNode >> /xxx/logs/hadoop-root-raidnode-hadoop.xxx.com.log & - -Optionally, we provide two scripts to start and stop the RaidNode. Copy the scripts -start-raidnode.sh and stop-raidnode.sh to the directory $HADOOP_PREFIX/bin in the machine -you would like to deploy the daemon. You can start or stop the RaidNode by directly -callying the scripts from that machine. If you want to deploy the RaidNode remotely, -copy start-raidnode-remote.sh and stop-raidnode-remote.sh to $HADOOP_PREFIX/bin at -the machine from which you want to trigger the remote deployment and create a text -file $HADOOP_PREFIX/conf/raidnode at the same machine containing the name of the server -where the RaidNode should run. These scripts run ssh to the specified machine and -invoke start/stop-raidnode.sh there. As an example, you might want to change -start-mapred.sh in the JobTracker machine so that it automatically calls -start-raidnode-remote.sh (and do the equivalent thing for stop-mapred.sh and -stop-raidnode-remote.sh). - -To validate the integrity of a file system, run RaidFSCK as follows: -$HADOOP_PREFIX/bin/hadoop org.apache.hadoop.raid.RaidShell -fsck [path] - -This will print a list of corrupt files (i.e., files which have lost too many -blocks and can no longer be fixed by Raid). - - --------------------------------------------------------------------------------- - -IMPLEMENTATION: - -The RaidNode periodically scans all the specified paths in the configuration -file. For each path, it recursively scans all files that have more than 2 blocks -and that has not been modified during the last few hours (default is 24 hours). -It picks the specified number of blocks (as specified by the stripe size), -from the file, generates a parity block by combining them and -stores the results as another HDFS file in the specified destination -directory. There is a one-to-one mapping between a HDFS -file and its parity file. The RaidNode also periodically finds parity files -that are orphaned and deletes them. - -The Distributed Raid FileSystem is layered over a DistributedFileSystem -instance intercepts all calls that go into HDFS. HDFS throws a ChecksumException -or a BlocMissingException when a file read encounters bad data. The layered -Distributed Raid FileSystem catches these exceptions, locates the corresponding -parity file, extract the original data from the parity files and feeds the -extracted data back to the application in a completely transparent way. - -The layered Distributed Raid FileSystem does not fix the data-loss that it -encounters while serving data. It merely make the application transparently -use the parity blocks to re-create the original data. A command line tool -"fsckraid" is currently under development that will fix the corrupted files -by extracting the data from the associated parity files. An adminstrator -can run "fsckraid" manually as and when needed. diff --git a/hadoop-mapreduce-project/src/contrib/raid/build.xml b/hadoop-mapreduce-project/src/contrib/raid/build.xml deleted file mode 100644 index 80a2ad63abe..00000000000 --- a/hadoop-mapreduce-project/src/contrib/raid/build.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Tests failed! - - - - diff --git a/hadoop-mapreduce-project/src/contrib/raid/ivy.xml b/hadoop-mapreduce-project/src/contrib/raid/ivy.xml deleted file mode 100644 index 55ea7aa1a2e..00000000000 --- a/hadoop-mapreduce-project/src/contrib/raid/ivy.xml +++ /dev/null @@ -1,145 +0,0 @@ - - - - - - Rumen - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hadoop-mapreduce-project/src/contrib/raid/ivy/libraries.properties b/hadoop-mapreduce-project/src/contrib/raid/ivy/libraries.properties deleted file mode 100644 index a9f5ebb283e..00000000000 --- a/hadoop-mapreduce-project/src/contrib/raid/ivy/libraries.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#This properties file lists the versions of the various artifacts used by hadoop. -#It drives ivy and the generation of a maven POM -#These are the versions of our dependencies (in alphabetical order) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 99083001893..017437e2747 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -257,11 +257,6 @@ hadoop-client ${project.version} - - org.apache.hadoop - hadoop-hdfs-raid - ${project.version} - org.apache.hadoop