Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:metamx/druid

This commit is contained in:
Eric Tschetter 2012-11-19 17:01:33 -06:00
parent 701cc9562b cdda8c2b7c
commit c657969243
6 changed files with 286 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
doc/data_flow_realtime.dot Normal file
View File

@ -0,0 +1,39 @@
digraph DruidDataFlow {
graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB];
node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"];
edge [color="#377d18"];
1 [label = "REST query"];
2 [label = "realtime_data_src"];
4 [label = "indexed segments\nblob_store (S3, HDFS)"];
5 [label = "Realtime.working_set"];
6 [label = "Realtime.spill_disk"];
7 [label = "HDFS_data_src"];
8 [label = "external_data_src"];
9 [label = "Realtime.local_disk"];
10 [label = "metadata_store (mysql)"];
11 [label = "HadoopDruidIndexer.working_set"];
12 [label = "IndexerService.working_set"];
2 -> 5 [label = "Realtime.Firehose"];
5 -> 6 [label = "Realtime.Firehose"];
5 -> 9 [label = "Realtime.Indexer"];
6 -> 9 [label = "Realtime.Indexer"];
5 -> 10 [label = "Realtime.MetadataUpdater"];
7 -> 11 [label = "HadoopDruidIndexer"];
11 -> 4 [label = "HadoopDruidIndexer"];
11 -> 10 [label = "HadoopDruidIndexer"];
8 -> 12 [label = "IndexerService"];
12 -> 4 [label = "IndexerService"];
12 -> 10 [label = "IndexerService"];
9 -> 4 [label = "Realtime.segmentPusher"];
5 -> 1 [label = " results "];
1 -> 5 [label = " query "];
}

BIN
doc/data_flow_realtime.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 96 KiB

128
doc/data_flow_realtime.svg Normal file
View File

@ -0,0 +1,128 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<!-- Generated by dot version 1.13 (v16) (Mon August 23, 2004)
For user: (peb) Paul E Baclace Title: DruidDataFlow Pages: 1 -->
<svg width="950pt" height="404pt"
viewBox = "-1 -1 949 403"
xmlns="http://www.w3.org/2000/svg">
<g id="graph0" class="graph" style="font-family:Times-Roman;font-size:14.00;">
<title>DruidDataFlow</title>
<polygon style="fill:#fffef5;stroke:#fffef5;" points="0,401 0,-1 948,-1 948,401 0,401"/>
<g id="node1" class="node"><title>1</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="177,4 91,4 91,40 177,40 177,4"/>
<text text-anchor="middle" x="134" y="27" style="font-family:Helvetica;">REST query</text>
</g>
<g id="node4" class="node"><title>5</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="272,92 124,92 124,128 272,128 272,92"/>
<text text-anchor="middle" x="198" y="115" style="font-family:Helvetica;">Realtime.working_set</text>
</g>
<g id="edge28" class="edge"><title>1-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M162,40C169,45 175,51 180,58 185,65 189,74 192,82"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="195,82 194,92 189,83 195,82"/>
<text text-anchor="middle" x="205" y="71"> query </text>
</g>
<g id="node2" class="node"><title>2</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="335,4 209,4 209,40 335,40 335,4"/>
<text text-anchor="middle" x="272" y="27" style="font-family:Helvetica;">realtime_data_src</text>
</g>
<g id="edge2" class="edge"><title>2-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M257,40C246,53 232,70 220,84"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="222,87 213,92 217,82 222,87"/>
<text text-anchor="middle" x="293" y="71">Realtime.Firehose</text>
</g>
<g id="node3" class="node"><title>4</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="328,356 172,356 172,396 328,396 328,356"/>
<text text-anchor="middle" x="250" y="373" style="font-family:Helvetica;">indexed segments</text>
<text text-anchor="middle" x="250" y="389" style="font-family:Helvetica;">blob_store (S3, HDFS)</text>
</g>
<g id="edge26" class="edge"><title>5-&gt;1</title>
<path style="fill:none;stroke:#377d18;" d="M150,92C143,87 137,81 133,74 129,67 128,58 128,50"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="125,50 129,40 131,50 125,50"/>
<text text-anchor="middle" x="154" y="71"> results </text>
</g>
<g id="node5" class="node"><title>6</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="136,180 6,180 6,216 136,216 136,180"/>
<text text-anchor="middle" x="71" y="203" style="font-family:Helvetica;">Realtime.spill_disk</text>
</g>
<g id="edge4" class="edge"><title>5-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M123,127C105,133 89,139 82,146 76,152 74,161 72,170"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="75,170 71,180 69,170 75,170"/>
<text text-anchor="middle" x="136" y="159">Realtime.Firehose</text>
</g>
<g id="node8" class="node"><title>9</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="143,268 9,268 9,304 143,304 143,268"/>
<text text-anchor="middle" x="76" y="291" style="font-family:Helvetica;">Realtime.local_disk</text>
</g>
<g id="edge6" class="edge"><title>5-&gt;9</title>
<path style="fill:none;stroke:#377d18;" d="M200,128C203,158 204,215 175,250 171,255 162,260 151,264"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="153,267 142,268 150,261 153,267"/>
<text text-anchor="middle" x="247" y="203">Realtime.Indexer</text>
</g>
<g id="node9" class="node"><title>10</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="726,358 564,358 564,394 726,394 726,358"/>
<text text-anchor="middle" x="645" y="381" style="font-family:Helvetica;">metadata_store (mysql)</text>
</g>
<g id="edge10" class="edge"><title>5-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M273,115C391,125 611,146 682,180 751,214 798,276 754,338 750,344 739,349 727,354"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="728,358 717,358 725,351 728,358"/>
<text text-anchor="middle" x="835" y="247">Realtime.MetadataUpdater</text>
</g>
<g id="edge8" class="edge"><title>6-&gt;9</title>
<path style="fill:none;stroke:#377d18;" d="M72,216C73,228 74,244 74,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="77,258 75,268 71,258 77,258"/>
<text text-anchor="middle" x="123" y="247">Realtime.Indexer</text>
</g>
<g id="node6" class="node"><title>7</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="672,180 558,180 558,216 672,216 672,180"/>
<text text-anchor="middle" x="615" y="203" style="font-family:Helvetica;">HDFS_data_src</text>
</g>
<g id="node10" class="node"><title>11</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="726,268 504,268 504,304 726,304 726,268"/>
<text text-anchor="middle" x="615" y="291" style="font-family:Helvetica;">HadoopDruidIndexer.working_set</text>
</g>
<g id="edge12" class="edge"><title>7-&gt;11</title>
<path style="fill:none;stroke:#377d18;" d="M615,216C615,228 615,244 615,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="619,258 615,268 612,258 619,258"/>
<text text-anchor="middle" x="674" y="247">HadoopDruidIndexer</text>
</g>
<g id="node7" class="node"><title>8</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="438,180 312,180 312,216 438,216 438,180"/>
<text text-anchor="middle" x="375" y="203" style="font-family:Helvetica;">external_data_src</text>
</g>
<g id="node11" class="node"><title>12</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="464,268 276,268 276,304 464,304 464,268"/>
<text text-anchor="middle" x="370" y="291" style="font-family:Helvetica;">IndexerService.working_set</text>
</g>
<g id="edge18" class="edge"><title>8-&gt;12</title>
<path style="fill:none;stroke:#377d18;" d="M374,216C373,228 372,244 372,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="375,258 371,268 369,258 375,258"/>
<text text-anchor="middle" x="414" y="247">IndexerService</text>
</g>
<g id="edge24" class="edge"><title>9-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M85,304C91,316 100,330 112,338 126,348 144,356 161,361"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="162,358 171,364 160,364 162,358"/>
<text text-anchor="middle" x="181" y="335">Realtime.segmentPusher</text>
</g>
<g id="edge14" class="edge"><title>11-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M553,304C536,309 519,315 503,322 488,328 486,334 472,338 430,353 380,362 339,368"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="339,371 329,369 339,365 339,371"/>
<text text-anchor="middle" x="568" y="335">HadoopDruidIndexer</text>
</g>
<g id="edge16" class="edge"><title>11-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M621,304C625,317 631,334 636,348"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="639,347 639,358 633,349 639,347"/>
<text text-anchor="middle" x="690" y="335">HadoopDruidIndexer</text>
</g>
<g id="edge20" class="edge"><title>12-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M300,304C284,310 270,316 264,322 258,328 255,337 253,346"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="256,347 251,356 250,346 256,347"/>
<text text-anchor="middle" x="306" y="335">IndexerService</text>
</g>
<g id="edge22" class="edge"><title>12-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M370,304C371,316 374,330 384,338 396,349 483,361 554,368"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="554,365 564,369 554,371 554,365"/>
<text text-anchor="middle" x="426" y="335">IndexerService</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 7.5 KiB

27
doc/data_flow_simple.dot Normal file
View File

@ -0,0 +1,27 @@
digraph DruidDataFlow {
graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB];
node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"];
edge [color="#377d18"];
1 [label = "REST query"];
10 [label = "Broker.query_cache"];
2 [label = "realtime_data_src"];
3 [label = "Compute.disk_cache"];
4 [label = "indexed segments\nblob_store (S3, HDFS)"];
5 [label = "Realtime.working_set"];
6 [label = "Compute.working_set"];
2 -> 5 [label = "Realtime.Firehose"];
5 -> 4 [label = "Realtime.segmentPusher"];
4 -> 3 [label = "Compute.load"];
3 -> 6 [label = "Compute.map"];
5 -> 10 [label = " results "];
6 -> 10 [label = " results "];
10 -> 1 [label = " results "];
10 -> 5 [label = " query "];
10 -> 6 [label = " query "];
1 -> 10 [label = " query "];
}

BIN
doc/data_flow_simple.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

92
doc/data_flow_simple.svg Normal file
View File

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN"
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
<!-- Generated by dot version 1.13 (v16) (Mon August 23, 2004)
For user: (peb) Paul E Baclace Title: DruidDataFlow Pages: 1 -->
<svg width="405pt" height="492pt"
viewBox = "-1 -1 404 491"
xmlns="http://www.w3.org/2000/svg">
<g id="graph0" class="graph" style="font-family:Times-Roman;font-size:14.00;">
<title>DruidDataFlow</title>
<polygon style="fill:#fffef5;stroke:#fffef5;" points="0,489 0,-1 403,-1 403,489 0,489"/>
<g id="node1" class="node"><title>1</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="163,4 77,4 77,40 163,40 163,4"/>
<text text-anchor="middle" x="120" y="27" style="font-family:Helvetica;">REST query</text>
</g>
<g id="node2" class="node"><title>10</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="189,92 51,92 51,128 189,128 189,92"/>
<text text-anchor="middle" x="120" y="115" style="font-family:Helvetica;">Broker.query_cache</text>
</g>
<g id="edge20" class="edge"><title>1-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M94,40C89,45 84,51 81,58 77,67 81,76 87,84"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="90,82 94,92 85,87 90,82"/>
<text text-anchor="middle" x="104" y="71"> query </text>
</g>
<g id="edge14" class="edge"><title>10-&gt;1</title>
<path style="fill:none;stroke:#377d18;" d="M123,92C123,86 124,80 125,74 125,66 125,58 124,50"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="121,50 123,40 127,50 121,50"/>
<text text-anchor="middle" x="146" y="71"> results </text>
</g>
<g id="node6" class="node"><title>5</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="303,180 155,180 155,216 303,216 303,180"/>
<text text-anchor="middle" x="229" y="203" style="font-family:Helvetica;">Realtime.working_set</text>
</g>
<g id="edge16" class="edge"><title>10-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M181,128C188,132 195,138 204,146 211,153 215,162 219,171"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="222,169 223,180 216,172 222,169"/>
<text text-anchor="middle" x="232" y="159"> query </text>
</g>
<g id="node7" class="node"><title>6</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="172,448 22,448 22,484 172,484 172,448"/>
<text text-anchor="middle" x="97" y="471" style="font-family:Helvetica;">Compute.working_set</text>
</g>
<g id="edge18" class="edge"><title>10-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M78,128C46,144 23,157 23,198 23,198 23,198 23,378 23,404 41,426 60,442"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="62,439 68,448 58,445 62,439"/>
<text text-anchor="middle" x="43" y="293"> query </text>
</g>
<g id="node3" class="node"><title>2</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="359,92 233,92 233,128 359,128 359,92"/>
<text text-anchor="middle" x="296" y="115" style="font-family:Helvetica;">realtime_data_src</text>
</g>
<g id="edge2" class="edge"><title>2-&gt;5</title>
<path style="fill:none;stroke:#377d18;" d="M286,128C280,138 272,151 264,162 261,166 258,169 255,173"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="258,175 248,180 253,170 258,175"/>
<text text-anchor="middle" x="326" y="159">Realtime.Firehose</text>
</g>
<g id="node4" class="node"><title>3</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="280,360 136,360 136,396 280,396 280,360"/>
<text text-anchor="middle" x="208" y="383" style="font-family:Helvetica;">Compute.disk_cache</text>
</g>
<g id="edge8" class="edge"><title>3-&gt;6</title>
<path style="fill:none;stroke:#377d18;" d="M185,396C169,409 146,427 128,442"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="130,445 120,448 126,439 130,445"/>
<text text-anchor="middle" x="200" y="427">Compute.map</text>
</g>
<g id="node5" class="node"><title>4</title>
<polygon style="fill:#fdf4c6;stroke:#0a0701;" points="315,268 159,268 159,308 315,308 315,268"/>
<text text-anchor="middle" x="237" y="285" style="font-family:Helvetica;">indexed segments</text>
<text text-anchor="middle" x="237" y="301" style="font-family:Helvetica;">blob_store (S3, HDFS)</text>
</g>
<g id="edge6" class="edge"><title>4-&gt;3</title>
<path style="fill:none;stroke:#377d18;" d="M230,308C226,321 221,337 217,350"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="220,351 214,360 214,349 220,351"/>
<text text-anchor="middle" x="263" y="339">Compute.load</text>
</g>
<g id="edge10" class="edge"><title>5-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M183,180C174,175 165,169 157,162 149,155 141,146 135,137"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="132,138 130,128 138,135 132,138"/>
<text text-anchor="middle" x="178" y="159"> results </text>
</g>
<g id="edge4" class="edge"><title>5-&gt;4</title>
<path style="fill:none;stroke:#377d18;" d="M231,216C232,228 233,244 234,258"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="237,258 235,268 231,258 237,258"/>
<text text-anchor="middle" x="302" y="247">Realtime.segmentPusher</text>
</g>
<g id="edge12" class="edge"><title>6-&gt;10</title>
<path style="fill:none;stroke:#377d18;" d="M97,448C97,430 98,403 97,378 97,198 97,198 97,198 98,177 104,155 110,138"/>
<polygon style="fill:#377d18;stroke:#377d18;" points="107,137 113,128 113,139 107,137"/>
<text text-anchor="middle" x="119" y="293"> results </text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB