Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

say explicitly that local firehose searches directories recursively for files

This commit is contained in:
sahner 2015-07-05 14:46:44 -05:00
parent f883ff2dab
commit acd20e8c00
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/content/ingestion/firehose.md
View File

@ -108,7 +108,7 @@ A sample local firehose spec is shown below:
|--------|-----------|---------| |--------|-----------|---------|
|type|This should be "local".|yes| |type|This should be "local".|yes|
|filter|A wildcard filter for files. See [here](http://commons.apache.org/proper/commons-io/apidocs/org/apache/commons/io/filefilter/WildcardFileFilter.html) for more information.|yes| |filter|A wildcard filter for files. See [here](http://commons.apache.org/proper/commons-io/apidocs/org/apache/commons/io/filefilter/WildcardFileFilter.html) for more information.|yes|
|baseDir|location of baseDirectory containing files to be ingested. |yes| |baseDir|directory to search recursively for files to be ingested. |yes|
#### IngestSegmentFirehose #### IngestSegmentFirehose

3
server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java
View File

@ -82,7 +82,7 @@ public class LocalFirehoseFactory implements FirehoseFactory<StringInputRowParse
@Override @Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException public Firehose connect(StringInputRowParser firehoseParser) throws IOException
{ {
log.info("Searching for all [%s] in [%s]", filter, baseDir.getAbsoluteFile()); log.info("Searching for all [%s] in and beneath [%s]", filter, baseDir.getAbsoluteFile());
Collection<File> foundFiles = FileUtils.listFiles( Collection<File> foundFiles = FileUtils.listFiles(
baseDir.getAbsoluteFile(), baseDir.getAbsoluteFile(),
@ -93,6 +93,7 @@ public class LocalFirehoseFactory implements FirehoseFactory<StringInputRowParse
if (foundFiles == null || foundFiles.isEmpty()) { if (foundFiles == null || foundFiles.isEmpty()) {
throw new ISE("Found no files to ingest! Check your schema."); throw new ISE("Found no files to ingest! Check your schema.");
} }
log.info ("Found files: " + foundFiles);
final LinkedList<File> files = Lists.newLinkedList( final LinkedList<File> files = Lists.newLinkedList(
foundFiles foundFiles