HADOOP-16018. DistCp won't reassemble chunks when blocks per chunk > 0.

Contributed by Kai Xie.
This commit is contained in:
Kai Xie 2019-01-08 11:56:48 +00:00 committed by Steve Loughran
parent 0921b706f7
commit 188bebbe7e
3 changed files with 34 additions and 1 deletions

View File

@ -123,6 +123,10 @@ public final class DistCpConstants {
public static final String CONF_LABEL_COPY_BUFFER_SIZE =
"distcp.copy.buffer.size";
/** DistCp Blocks Per Chunk: {@value}. */
public static final String CONF_LABEL_BLOCKS_PER_CHUNK =
"distcp.blocks.per.chunk";
/**
* Constants for DistCp return code to shell / consumer of ToolRunner's run
*/

View File

@ -192,7 +192,7 @@ public enum DistCpOptionSwitch {
new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
"copied to <= n bytes")),
BLOCKS_PER_CHUNK("",
BLOCKS_PER_CHUNK(DistCpConstants.CONF_LABEL_BLOCKS_PER_CHUNK,
new Option("blocksperchunk", true, "If set to a positive value, files"
+ "with more blocks than this value will be split into chunks of "
+ "<blocksperchunk> blocks to be transferred in parallel, and "

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.tools;
import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@ -533,4 +534,32 @@ public class TestDistCpOptions {
builder.withLogPath(logPath).withVerboseLog(true);
Assert.assertTrue(builder.build().shouldVerboseLog());
}
@Test
public void testAppendToConf() {
final int expectedBlocksPerChunk = 999;
final String expectedValForEmptyConfigKey = "VALUE_OF_EMPTY_CONFIG_KEY";
DistCpOptions options = new DistCpOptions.Builder(
Collections.singletonList(
new Path("hdfs://localhost:8020/source")),
new Path("hdfs://localhost:8020/target/"))
.withBlocksPerChunk(expectedBlocksPerChunk)
.build();
Configuration config = new Configuration();
config.set("", expectedValForEmptyConfigKey);
options.appendToConf(config);
Assert.assertEquals(expectedBlocksPerChunk,
config.getInt(
DistCpOptionSwitch
.BLOCKS_PER_CHUNK
.getConfigLabel(), 0));
Assert.assertEquals(
"Some DistCpOptionSwitch's config label is empty! " +
"Pls ensure the config label is provided when apply to config, " +
"otherwise it may not be fetched properly",
expectedValForEmptyConfigKey, config.get(""));
}
}