MAPREDUCE-6751. Add debug log message when splitting is not possible due to unsplittable compression. (Peter Vary via rchiang)
(cherry picked from commit 6c154abd33
)
This commit is contained in:
parent
96e9f8bec9
commit
d1c4ba2d7c
|
@ -369,6 +369,13 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
|
|||
splitHosts[0], splitHosts[1]));
|
||||
}
|
||||
} else {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
// Log only if the file is big enough to be splitted
|
||||
if (length > Math.min(file.getBlockSize(), minSize)) {
|
||||
LOG.debug("File is not splittable so no parallelization "
|
||||
+ "is possible: " + file.getPath());
|
||||
}
|
||||
}
|
||||
String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,0,length,clusterMap);
|
||||
splits.add(makeSplit(path, 0, length, splitHosts[0], splitHosts[1]));
|
||||
}
|
||||
|
|
|
@ -600,6 +600,10 @@ public abstract class CombineFileInputFormat<K, V>
|
|||
if (!isSplitable) {
|
||||
// if the file is not splitable, just create the one block with
|
||||
// full file length
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("File is not splittable so no parallelization "
|
||||
+ "is possible: " + stat.getPath());
|
||||
}
|
||||
blocks = new OneBlockInfo[1];
|
||||
fileSize = stat.getLen();
|
||||
blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize,
|
||||
|
|
|
@ -422,6 +422,13 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
|
|||
blkLocations[blkIndex].getCachedHosts()));
|
||||
}
|
||||
} else { // not splitable
|
||||
if (LOG.isDebugEnabled()) {
|
||||
// Log only if the file is big enough to be splitted
|
||||
if (length > Math.min(file.getBlockSize(), minSize)) {
|
||||
LOG.debug("File is not splittable so no parallelization "
|
||||
+ "is possible: " + file.getPath());
|
||||
}
|
||||
}
|
||||
splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
|
||||
blkLocations[0].getCachedHosts()));
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue