mirror of
https://github.com/apache/druid.git
synced 2025-02-06 10:08:26 +00:00
1224d8b746
* move parquet-extensions from contrib to core, adds new hadoop parquet parser that does not convert to avro first and supports flattenSpec and int96 columns, add support for flattenSpec for parquet-avro conversion parser, much test with a bunch of files lifted from spark-sql * fix avro flattener to support nullable primitives for auto discovery and now only supports primitive arrays instead of all arrays * remove leftover print * convert micro timestamp to millis * checkstyle * add ignore for .parquet and .parq to rat exclude * fix legit test failure from avro flattern behavior change * fix rebase * add exclusions to pom to cut down on redundant jars * refactor tests, add support for unwrapping lists for parquet-avro, review comments * more comment * fix oops * tweak parquet-avro list handling * more docs * fix style * grr styles
82 lines
2.1 KiB
JSON
Executable File
82 lines
2.1 KiB
JSON
Executable File
{
|
|
"type": "index_hadoop",
|
|
"spec": {
|
|
"ioConfig": {
|
|
"type": "hadoop",
|
|
"inputSpec": {
|
|
"type": "static",
|
|
"inputFormat": "%s",
|
|
"paths": "example/wiki/wiki.parquet"
|
|
},
|
|
"metadataUpdateSpec": {
|
|
"type": "postgresql",
|
|
"connectURI": "jdbc:postgresql://localhost/druid",
|
|
"user": "druid",
|
|
"password": "asdf",
|
|
"segmentTable": "druid_segments"
|
|
},
|
|
"segmentOutputPath": "/tmp/segments"
|
|
},
|
|
"dataSchema": {
|
|
"dataSource": "wikipedia",
|
|
"parser": {
|
|
"type": "%s",
|
|
"parseSpec": {
|
|
"format": "timeAndDims",
|
|
"timestampSpec": {
|
|
"column": "timestamp",
|
|
"format": "auto"
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [
|
|
"page",
|
|
"language",
|
|
"user",
|
|
"unpatrolled"
|
|
],
|
|
"dimensionExclusions": [],
|
|
"spatialDimensions": []
|
|
}
|
|
}
|
|
},
|
|
"metricsSpec": [
|
|
{
|
|
"type": "count",
|
|
"name": "count"
|
|
},
|
|
{
|
|
"type": "doubleSum",
|
|
"name": "deleted",
|
|
"fieldName": "deleted"
|
|
},
|
|
{
|
|
"type": "doubleSum",
|
|
"name": "delta",
|
|
"fieldName": "delta"
|
|
}
|
|
],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": "NONE",
|
|
"intervals": [
|
|
"2013-08-30/2013-09-02"
|
|
]
|
|
}
|
|
},
|
|
"tuningConfig": {
|
|
"type": "hadoop",
|
|
"workingPath": "tmp/working_path",
|
|
"partitionsSpec": {
|
|
"targetPartitionSize": 5000000
|
|
},
|
|
"jobProperties": {
|
|
"mapreduce.map.java.opts": "-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps",
|
|
"mapreduce.reduce.java.opts": "-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps",
|
|
"mapred.child.java.opts": "-server -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
|
|
},
|
|
"leaveIntermediate": true
|
|
}
|
|
}
|
|
}
|