Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13024. Distcp with -delete feature on raw data not implemented. Contributed by Mavin Martin. 

(cherry picked from commit 0a85d07983)
(cherry picked from commit ad69baf6a9)
This commit is contained in:
Jing Zhao 2016-10-13 13:24:37 -07:00
parent 3892fd810b
commit d3c4465147
4 changed files with 56 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
View File

@ -18,6 +18,8 @@ package org.apache.hadoop.tools;
* limitations under the License. * limitations under the License.
*/ */
import org.apache.hadoop.fs.Path;
/** /**
* Utility class to hold commonly used constants. * Utility class to hold commonly used constants.
*/ */
@ -140,10 +142,18 @@ public class DistCpConstants {
public static final int MIN_RECORDS_PER_CHUNK_DEFAULT = 5; public static final int MIN_RECORDS_PER_CHUNK_DEFAULT = 5;
public static final int SPLIT_RATIO_DEFAULT = 2; public static final int SPLIT_RATIO_DEFAULT = 2;
/**
* Constants for NONE file deletion
*/
public static final String NONE_PATH_NAME = "/NONE";
public static final Path NONE_PATH = new Path(NONE_PATH_NAME);
public static final Path RAW_NONE_PATH = new Path(
DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME + NONE_PATH_NAME);
/** /**
* Value of reserved raw HDFS directory when copying raw.* xattrs. * Value of reserved raw HDFS directory when copying raw.* xattrs.
*/ */
static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw"; public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp"; static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp";
} }

5
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
View File

@ -238,7 +238,10 @@ public class CopyCommitter extends FileOutputCommitter {
List<Path> targets = new ArrayList<Path>(1); List<Path> targets = new ArrayList<Path>(1);
Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
targets.add(targetFinalPath); targets.add(targetFinalPath);
DistCpOptions options = new DistCpOptions(targets, new Path("/NONE")); Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath)
.toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME)
? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH;
DistCpOptions options = new DistCpOptions(targets, resultNonePath);
// //
// Set up options to be the same from the CopyListing.buildListing's perspective, // Set up options to be the same from the CopyListing.buildListing's perspective,
// so to collect similar listings as when doing the copy // so to collect similar listings as when doing the copy

45
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
View File

@ -82,14 +82,7 @@ public class TestDistCpWithRawXAttrs {
final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest"; final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest";
doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true, doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true,
DistCpConstants.SUCCESS); DistCpConstants.SUCCESS);
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px", doTestStandardPreserveRawXAttrs("-px", true);
false, true, DistCpConstants.SUCCESS);
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px",
false, true, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px",
false, true, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px",
true, true, DistCpConstants.SUCCESS);
final Path savedWd = fs.getWorkingDirectory(); final Path savedWd = fs.getWorkingDirectory();
try { try {
fs.setWorkingDirectory(new Path("/.reserved/raw")); fs.setWorkingDirectory(new Path("/.reserved/raw"));
@ -103,27 +96,18 @@ public class TestDistCpWithRawXAttrs {
/* Test that XAttrs are not preserved and raw.* are when appropriate. */ /* Test that XAttrs are not preserved and raw.* are when appropriate. */
@Test @Test
public void testPreserveRawXAttrs2() throws Exception { public void testPreserveRawXAttrs2() throws Exception {
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p", doTestStandardPreserveRawXAttrs("-p", false);
false, false, DistCpConstants.SUCCESS);
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p",
false, false, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p",
false, false, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p",
true, false, DistCpConstants.SUCCESS);
} }
/* Test that XAttrs are not preserved and raw.* are when appropriate. */ /* Test that XAttrs are not preserved and raw.* are when appropriate. */
@Test @Test
public void testPreserveRawXAttrs3() throws Exception { public void testPreserveRawXAttrs3() throws Exception {
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null, doTestStandardPreserveRawXAttrs(null, false);
false, false, DistCpConstants.SUCCESS); }
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null,
false, false, DistCpConstants.INVALID_ARGUMENT); @Test
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null, public void testPreserveRawXAttrs4() throws Exception {
false, false, DistCpConstants.INVALID_ARGUMENT); doTestStandardPreserveRawXAttrs("-update -delete", false);
doTestPreserveRawXAttrs(rawSrcName, rawDestName, null,
true, false, DistCpConstants.SUCCESS);
} }
private static Path[] pathnames = { new Path("dir1"), private static Path[] pathnames = { new Path("dir1"),
@ -145,6 +129,19 @@ public class TestDistCpWithRawXAttrs {
} }
} }
private void doTestStandardPreserveRawXAttrs(String options,
boolean expectUser)
throws Exception {
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, options,
false, expectUser, DistCpConstants.SUCCESS);
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, options,
false, expectUser, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, options,
false, expectUser, DistCpConstants.INVALID_ARGUMENT);
doTestPreserveRawXAttrs(rawSrcName, rawDestName, options,
true, expectUser, DistCpConstants.SUCCESS);
}
private void doTestPreserveRawXAttrs(String src, String dest, private void doTestPreserveRawXAttrs(String src, String dest,
String preserveOpts, boolean expectRaw, boolean expectUser, String preserveOpts, boolean expectRaw, boolean expectUser,
int expectedExitCode) throws Exception { int expectedExitCode) throws Exception {

32
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
View File

@ -18,20 +18,19 @@
package org.apache.hadoop.tools.util; package org.apache.hadoop.tools.util;
import static org.junit.Assert.assertArrayEquals; import org.apache.hadoop.conf.Configuration;
import static org.junit.Assert.assertEquals; import org.apache.hadoop.fs.FileSystem;
import static org.junit.Assert.assertTrue; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.util.ToolRunner;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration; import static org.junit.Assert.assertArrayEquals;
import org.apache.hadoop.fs.FileSystem; import static org.junit.Assert.assertEquals;
import org.apache.hadoop.fs.Path; import static org.junit.Assert.assertTrue;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.util.ToolRunner;
/** /**
* Utility class for DistCpTests * Utility class for DistCpTests
@ -79,10 +78,19 @@ public class DistCpTestUtils {
public static void assertRunDistCp(int exitCode, String src, String dst, public static void assertRunDistCp(int exitCode, String src, String dst,
String options, Configuration conf) String options, Configuration conf)
throws Exception { throws Exception {
assertRunDistCp(exitCode, src, dst,
options == null ? new String[0] : options.trim().split(" "), conf);
}
private static void assertRunDistCp(int exitCode, String src, String dst,
String[] options, Configuration conf)
throws Exception {
DistCp distCp = new DistCp(conf, null); DistCp distCp = new DistCp(conf, null);
String[] optsArr = options == null ? String[] optsArr = new String[options.length + 2];
new String[] { src, dst } : System.arraycopy(options, 0, optsArr, 0, options.length);
new String[] { options, src, dst }; optsArr[optsArr.length - 2] = src;
optsArr[optsArr.length - 1] = dst;
assertEquals(exitCode, assertEquals(exitCode,
ToolRunner.run(conf, distCp, optsArr)); ToolRunner.run(conf, distCp, optsArr));
} }