HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic
This commit is contained in:
parent
cc01ed7026
commit
9faccd1046
|
@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool {
|
|||
private static final String PREFIX = "_distcp";
|
||||
private static final String WIP_PREFIX = "._WIP_";
|
||||
private static final String DISTCP_DEFAULT_XML = "distcp-default.xml";
|
||||
private static final String DISTCP_SITE_XML = "distcp-site.xml";
|
||||
static final Random rand = new Random();
|
||||
|
||||
private boolean submitted;
|
||||
|
@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool {
|
|||
public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
|
||||
Configuration config = new Configuration(configuration);
|
||||
config.addResource(DISTCP_DEFAULT_XML);
|
||||
config.addResource(DISTCP_SITE_XML);
|
||||
setConf(config);
|
||||
this.inputOptions = inputOptions;
|
||||
this.metaFolder = createMetaFolderPath();
|
||||
|
@ -393,10 +395,12 @@ public class DistCp extends Configured implements Tool {
|
|||
* Loads properties from distcp-default.xml into configuration
|
||||
* object
|
||||
* @return Configuration which includes properties from distcp-default.xml
|
||||
* and distcp-site.xml
|
||||
*/
|
||||
private static Configuration getDefaultConf() {
|
||||
Configuration config = new Configuration();
|
||||
config.addResource(DISTCP_DEFAULT_XML);
|
||||
config.addResource(DISTCP_SITE_XML);
|
||||
return config;
|
||||
}
|
||||
|
||||
|
|
|
@ -47,7 +47,7 @@ public class DistCpOptions {
|
|||
public static final int maxNumListstatusThreads = 40;
|
||||
private int numListstatusThreads = 0; // Indicates that flag is not set.
|
||||
private int maxMaps = DistCpConstants.DEFAULT_MAPS;
|
||||
private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB;
|
||||
private float mapBandwidth = 0; // Indicates that we should use the default.
|
||||
|
||||
private String copyStrategy = DistCpConstants.UNIFORMSIZE;
|
||||
|
||||
|
@ -609,8 +609,10 @@ public class DistCpOptions {
|
|||
String.valueOf(useDiff));
|
||||
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC,
|
||||
String.valueOf(skipCRC));
|
||||
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
|
||||
String.valueOf(mapBandwidth));
|
||||
if (mapBandwidth > 0) {
|
||||
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
|
||||
String.valueOf(mapBandwidth));
|
||||
}
|
||||
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
|
||||
DistCpUtils.packAttributes(preserveStatus));
|
||||
if (filtersFile != null) {
|
||||
|
|
|
@ -107,7 +107,7 @@ public class TestOptionsParser {
|
|||
DistCpOptions options = OptionsParser.parse(new String[] {
|
||||
"hdfs://localhost:9820/source/first",
|
||||
"hdfs://localhost:9820/target/"});
|
||||
Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
|
||||
Assert.assertEquals(options.getMapBandwidth(), 0, DELTA);
|
||||
|
||||
options = OptionsParser.parse(new String[] {
|
||||
"-bandwidth",
|
||||
|
@ -389,7 +389,7 @@ public class TestOptionsParser {
|
|||
+ "deleteMissing=false, ignoreFailures=false, overwrite=false, "
|
||||
+ "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, "
|
||||
+ "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
|
||||
+ "mapBandwidth=100.0, "
|
||||
+ "mapBandwidth=0.0, "
|
||||
+ "copyStrategy='uniformsize', preserveStatus=[], "
|
||||
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
|
||||
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
|
||||
|
@ -572,6 +572,8 @@ public class TestOptionsParser {
|
|||
Configuration conf = new Configuration();
|
||||
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
|
||||
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
|
||||
Assert.assertEquals(
|
||||
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
|
||||
DistCpOptions options = OptionsParser.parse(new String[] {
|
||||
"-atomic",
|
||||
"-i",
|
||||
|
@ -581,7 +583,7 @@ public class TestOptionsParser {
|
|||
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
|
||||
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
|
||||
Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
|
||||
DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
|
||||
-1.0, DELTA);
|
||||
|
||||
conf = new Configuration();
|
||||
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
|
||||
|
@ -602,6 +604,62 @@ public class TestOptionsParser {
|
|||
Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11.2, DELTA);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOptionsAppendToConfDoesntOverwriteBandwidth() {
|
||||
Configuration conf = new Configuration();
|
||||
Assert.assertEquals(
|
||||
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
|
||||
DistCpOptions options = OptionsParser.parse(new String[] {
|
||||
"hdfs://localhost:8020/source/first",
|
||||
"hdfs://localhost:8020/target/"});
|
||||
options.appendToConf(conf);
|
||||
Assert.assertEquals(
|
||||
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0,
|
||||
DELTA);
|
||||
|
||||
conf = new Configuration();
|
||||
Assert.assertEquals(
|
||||
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
|
||||
options = OptionsParser.parse(new String[] {
|
||||
"-update",
|
||||
"-delete",
|
||||
"-pu",
|
||||
"-bandwidth",
|
||||
"77",
|
||||
"hdfs://localhost:8020/source/first",
|
||||
"hdfs://localhost:8020/target/"});
|
||||
options.appendToConf(conf);
|
||||
Assert.assertEquals(
|
||||
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0,
|
||||
DELTA);
|
||||
|
||||
conf = new Configuration();
|
||||
conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88");
|
||||
Assert.assertEquals(
|
||||
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88");
|
||||
options = OptionsParser.parse(new String[] {
|
||||
"hdfs://localhost:8020/source/first",
|
||||
"hdfs://localhost:8020/target/"});
|
||||
options.appendToConf(conf);
|
||||
Assert.assertEquals(
|
||||
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0,
|
||||
DELTA);
|
||||
|
||||
conf = new Configuration();
|
||||
conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0");
|
||||
Assert.assertEquals(
|
||||
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0");
|
||||
options = OptionsParser.parse(new String[] {
|
||||
"-bandwidth",
|
||||
"99",
|
||||
"hdfs://localhost:8020/source/first",
|
||||
"hdfs://localhost:8020/target/"});
|
||||
options.appendToConf(conf);
|
||||
Assert.assertEquals(
|
||||
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0,
|
||||
DELTA);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppendOption() {
|
||||
Configuration conf = new Configuration();
|
||||
|
|
Loading…
Reference in New Issue