Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

log sizes of created smoosh files (#3817) 

* log when merging of intermediate segments starts during batch ingestion

* log sizes of created smoosh files
This commit is contained in:
Himanshu 2017-01-04 18:52:22 -06:00 committed by Fangjin Yang
parent 220ca7ebb6
commit 7ced0e8759
2 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java
View File

@ -695,9 +695,16 @@ public class IndexGeneratorJob implements Jobby
for (File file : toMerge) {
indexes.add(HadoopDruidIndexerConfig.INDEX_IO.loadIndex(file));
}
log.info("starting merge of intermediate persisted segments.");
long mergeStartTime = System.currentTimeMillis();
mergedBase = mergeQueryableIndex(
indexes, aggregators, new File(baseFlushFile, "merged"), progressIndicator
);
log.info(
"finished merge of intermediate persisted segments. time taken [%d] ms.",
(System.currentTimeMillis() - mergeStartTime)
);
}
final FileSystem outputFS = new Path(config.getSchema().getIOConfig().getSegmentOutputPath())
.getFileSystem(context.getConfiguration());

13
java-util/src/main/java/io/druid/java/util/common/io/smoosh/FileSmoosher.java
View File

@ -414,25 +414,27 @@ public class FileSmoosher implements Closeable
final int fileNum = outFiles.size();
File outFile = makeChunkFile(baseDir, fileNum);
outFiles.add(outFile);
return new Outer(fileNum, new FileOutputStream(outFile), maxChunkSize);
return new Outer(fileNum, outFile, maxChunkSize);
}
public static class Outer implements SmooshedWriter
{
private final int fileNum;
private final int maxLength;
private final File outFile;
private final GatheringByteChannel channel;
private final Closer closer = Closer.create();
private int currOffset = 0;
Outer(int fileNum, FileOutputStream output, int maxLength)
Outer(int fileNum, File outFile, int maxLength) throws FileNotFoundException
{
this.fileNum = fileNum;
this.channel = output.getChannel();
this.outFile = outFile;
this.maxLength = maxLength;
closer.register(output);
closer.register(channel);
FileOutputStream outStream = closer.register(new FileOutputStream(outFile));
this.channel = closer.register(outStream.getChannel());
}
public int getFileNum()
@ -494,6 +496,7 @@ public class FileSmoosher implements Closeable
public void close() throws IOException
{
closer.close();
FileSmoosher.LOG.info("Created smoosh file [%s] of size [%s] bytes.", outFile.getAbsolutePath(), outFile.length());
}
}
}