Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1823 from himanshug/datasource_pathspec_querygranularity 

For dataSource inputSpec in hadoop batch ingestion, use configured query granularity for reading existing segments instead of NONE
This commit is contained in:
Xavier Léauté 2015-10-13 10:28:15 -07:00
parent f4e4dac434 0368260018
commit 75061dad67
2 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceIngestionSpec.java
View File

@ -102,6 +102,11 @@ public class DatasourceIngestionSpec
return new DatasourceIngestionSpec(dataSource, interval, filter, granularity, dimensions, metrics); return new DatasourceIngestionSpec(dataSource, interval, filter, granularity, dimensions, metrics);
} }
public DatasourceIngestionSpec withQueryGranularity(QueryGranularity granularity)
{
return new DatasourceIngestionSpec(dataSource, interval, filter, granularity, dimensions, metrics);
}
@Override @Override
public boolean equals(Object o) public boolean equals(Object o)
{ {

2
indexing-hadoop/src/main/java/io/druid/indexer/path/DatasourcePathSpec.java
View File

@ -147,6 +147,8 @@ public class DatasourcePathSpec implements PathSpec
updatedIngestionSpec = updatedIngestionSpec.withMetrics(Lists.newArrayList(metrics)); updatedIngestionSpec = updatedIngestionSpec.withMetrics(Lists.newArrayList(metrics));
} }
updatedIngestionSpec = updatedIngestionSpec.withQueryGranularity(config.getGranularitySpec().getQueryGranularity());
job.getConfiguration().set(DatasourceInputFormat.CONF_DRUID_SCHEMA, mapper.writeValueAsString(updatedIngestionSpec)); job.getConfiguration().set(DatasourceInputFormat.CONF_DRUID_SCHEMA, mapper.writeValueAsString(updatedIngestionSpec));
job.getConfiguration().set(DatasourceInputFormat.CONF_INPUT_SEGMENTS, mapper.writeValueAsString(segments)); job.getConfiguration().set(DatasourceInputFormat.CONF_INPUT_SEGMENTS, mapper.writeValueAsString(segments));
job.getConfiguration().set(DatasourceInputFormat.CONF_MAX_SPLIT_SIZE, String.valueOf(maxSplitSize)); job.getConfiguration().set(DatasourceInputFormat.CONF_MAX_SPLIT_SIZE, String.valueOf(maxSplitSize));