HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer. Contributed by Akira Ajisaka.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1573694 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2626d3751e
commit
146bf6c01e
|
@ -517,6 +517,9 @@ Release 2.4.0 - UNRELEASED
|
||||||
HDFS-5956. A file size is multiplied by the replication factor in 'hdfs oiv
|
HDFS-5956. A file size is multiplied by the replication factor in 'hdfs oiv
|
||||||
-p FileDistribution' option. (Akira Ajisaka via wheat9)
|
-p FileDistribution' option. (Akira Ajisaka via wheat9)
|
||||||
|
|
||||||
|
HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer.
|
||||||
|
(Akira Ajisaka via wheat9)
|
||||||
|
|
||||||
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
|
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
|
||||||
|
|
||||||
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
|
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
|
||||||
|
|
|
@ -62,6 +62,7 @@ import com.google.common.io.LimitInputStream;
|
||||||
final class FileDistributionCalculator {
|
final class FileDistributionCalculator {
|
||||||
private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37
|
private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37
|
||||||
private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21
|
private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21
|
||||||
|
private final static int MAX_INTERVALS = 0x8000000; // 128 M = 2^27
|
||||||
|
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private final long maxSize;
|
private final long maxSize;
|
||||||
|
@ -82,9 +83,11 @@ final class FileDistributionCalculator {
|
||||||
this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
|
this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
|
||||||
this.out = out;
|
this.out = out;
|
||||||
long numIntervals = this.maxSize / this.steps;
|
long numIntervals = this.maxSize / this.steps;
|
||||||
|
// avoid OutOfMemoryError when allocating an array
|
||||||
|
Preconditions.checkState(numIntervals <= MAX_INTERVALS,
|
||||||
|
"Too many distribution intervals (maxSize/step): " + numIntervals +
|
||||||
|
", should be less than " + (MAX_INTERVALS+1) + ".");
|
||||||
this.distribution = new int[1 + (int) (numIntervals)];
|
this.distribution = new int[1 + (int) (numIntervals)];
|
||||||
Preconditions.checkState(numIntervals < Integer.MAX_VALUE,
|
|
||||||
"Too many distribution intervals");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(RandomAccessFile file) throws IOException {
|
void visit(RandomAccessFile file) throws IOException {
|
||||||
|
|
|
@ -101,9 +101,8 @@ public class OfflineImageViewerPB {
|
||||||
|
|
||||||
options.addOption("p", "processor", true, "");
|
options.addOption("p", "processor", true, "");
|
||||||
options.addOption("h", "help", false, "");
|
options.addOption("h", "help", false, "");
|
||||||
options.addOption("skipBlocks", false, "");
|
options.addOption("maxSize", true, "");
|
||||||
options.addOption("printToScreen", false, "");
|
options.addOption("step", true, "");
|
||||||
options.addOption("delimiter", true, "");
|
|
||||||
|
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
@ -118,10 +117,15 @@ public class OfflineImageViewerPB {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void main(String[] args) throws IOException {
|
public static void main(String[] args) throws IOException {
|
||||||
|
int status = run(args);
|
||||||
|
System.exit(status);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int run(String[] args) throws IOException {
|
||||||
Options options = buildOptions();
|
Options options = buildOptions();
|
||||||
if (args.length == 0) {
|
if (args.length == 0) {
|
||||||
printUsage();
|
printUsage();
|
||||||
return;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
CommandLineParser parser = new PosixParser();
|
CommandLineParser parser = new PosixParser();
|
||||||
|
@ -132,12 +136,12 @@ public class OfflineImageViewerPB {
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
System.out.println("Error parsing command-line options: ");
|
System.out.println("Error parsing command-line options: ");
|
||||||
printUsage();
|
printUsage();
|
||||||
return;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cmd.hasOption("h")) { // print help and exit
|
if (cmd.hasOption("h")) { // print help and exit
|
||||||
printUsage();
|
printUsage();
|
||||||
return;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
String inputFile = cmd.getOptionValue("i");
|
String inputFile = cmd.getOptionValue("i");
|
||||||
|
@ -160,6 +164,7 @@ public class OfflineImageViewerPB {
|
||||||
} else {
|
} else {
|
||||||
new LsrPBImage(conf, out).visit(new RandomAccessFile(inputFile, "r"));
|
new LsrPBImage(conf, out).visit(new RandomAccessFile(inputFile, "r"));
|
||||||
}
|
}
|
||||||
|
return 0;
|
||||||
} catch (EOFException e) {
|
} catch (EOFException e) {
|
||||||
System.err.println("Input file ended unexpectedly. Exiting");
|
System.err.println("Input file ended unexpectedly. Exiting");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -167,7 +172,7 @@ public class OfflineImageViewerPB {
|
||||||
} finally {
|
} finally {
|
||||||
IOUtils.cleanup(null, out);
|
IOUtils.cleanup(null, out);
|
||||||
}
|
}
|
||||||
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -277,6 +277,14 @@ public class TestOfflineImageViewer {
|
||||||
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
|
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFileDistributionCalculatorWithOptions() throws IOException {
|
||||||
|
int status = OfflineImageViewerPB.run(new String[] {"-i",
|
||||||
|
originalFsimage.getAbsolutePath(), "-o", "-", "-p", "FileDistribution",
|
||||||
|
"-maxSize", "512", "-step", "8"});
|
||||||
|
assertEquals(0, status);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testPBImageXmlWriter() throws IOException, SAXException,
|
public void testPBImageXmlWriter() throws IOException, SAXException,
|
||||||
ParserConfigurationException {
|
ParserConfigurationException {
|
||||||
|
|
Loading…
Reference in New Issue