mirror of
https://github.com/apache/druid.git
synced 2025-02-10 03:55:02 +00:00
90640bb316
Add support for hadoop 3 profiles . Most of the details are captured in #11791 . We use a combination of maven profiles and resource filtering to achieve this. Hadoop2 is supported by default and a new maven profile with the name hadoop3 is created. This will allow the user to choose the profile which is best suited for the use case.
81 lines
2.5 KiB
JSON
81 lines
2.5 KiB
JSON
{
|
|
"type" : "index_hadoop",
|
|
"spec" : {
|
|
"dataSchema" : {
|
|
"dataSource" : "wikipedia",
|
|
"parser" : {
|
|
"type" : "hadoopyString",
|
|
"parseSpec" : {
|
|
"format" : "json",
|
|
"dimensionsSpec" : {
|
|
"dimensions" : [
|
|
"channel",
|
|
"cityName",
|
|
"comment",
|
|
"countryIsoCode",
|
|
"countryName",
|
|
"isAnonymous",
|
|
"isMinor",
|
|
"isNew",
|
|
"isRobot",
|
|
"isUnpatrolled",
|
|
"metroCode",
|
|
"namespace",
|
|
"page",
|
|
"regionIsoCode",
|
|
"regionName",
|
|
"user",
|
|
{ "name": "added", "type": "long" },
|
|
{ "name": "deleted", "type": "long" },
|
|
{ "name": "delta", "type": "long" }
|
|
]
|
|
},
|
|
"timestampSpec" : {
|
|
"format" : "auto",
|
|
"column" : "time"
|
|
}
|
|
}
|
|
},
|
|
"metricsSpec" : [],
|
|
"granularitySpec" : {
|
|
"type" : "uniform",
|
|
"segmentGranularity" : "day",
|
|
"queryGranularity" : "none",
|
|
"intervals" : ["2015-09-12/2015-09-13"],
|
|
"rollup" : false
|
|
}
|
|
},
|
|
"ioConfig" : {
|
|
"type" : "hadoop",
|
|
"inputSpec" : {
|
|
"type" : "static",
|
|
"paths" : "/quickstart/wikiticker-2015-09-12-sampled.json.gz"
|
|
}
|
|
},
|
|
"tuningConfig" : {
|
|
"type" : "hadoop",
|
|
"partitionsSpec" : {
|
|
"type" : "hashed",
|
|
"targetPartitionSize" : 5000000
|
|
},
|
|
"forceExtendableShardSpecs" : true,
|
|
"jobProperties" : {
|
|
"fs.default.name" : "hdfs://druid-hadoop-demo:9000",
|
|
"fs.defaultFS" : "hdfs://druid-hadoop-demo:9000",
|
|
"dfs.datanode.address" : "druid-hadoop-demo",
|
|
"dfs.client.use.datanode.hostname" : "true",
|
|
"dfs.datanode.use.datanode.hostname" : "true",
|
|
"yarn.resourcemanager.hostname" : "druid-hadoop-demo",
|
|
"yarn.nodemanager.vmem-check-enabled" : "false",
|
|
"mapreduce.map.java.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
|
|
"mapreduce.job.user.classpath.first" : "true",
|
|
"mapreduce.reduce.java.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
|
|
"mapreduce.map.memory.mb" : 1024,
|
|
"mapreduce.reduce.memory.mb" : 1024,
|
|
"mapreduce.job.classloader" : "true"
|
|
}
|
|
}
|
|
},
|
|
"hadoopDependencyCoordinates": ["org.apache.hadoop:hadoop-client-api:3.3.1","org.apache.hadoop:hadoop-client-runtime:3.3.1"]
|
|
}
|