Simplify ITHttpInputSourceTest to mitigate flakiness (#11751)

* Increment retry count to add more time for tests to pass

* Re-enable ITHttpInputSourceTest

* Restore original count

* This test is about input source, hash partitioning takes longer and not required thus changing to dynamic

* Further simplify by removing sketches
This commit is contained in:
Agustin Gonzalez 2021-10-12 09:51:27 -07:00 committed by GitHub
parent adb2237628
commit 887cecf29e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 88 deletions

View File

@ -36,8 +36,6 @@ public class ITHttpInputSourceTest extends AbstractITBatchIndexTest
private static final String INDEX_TASK = "/indexer/wikipedia_http_inputsource_task.json"; private static final String INDEX_TASK = "/indexer/wikipedia_http_inputsource_task.json";
private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_http_inputsource_queries.json"; private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_http_inputsource_queries.json";
// Ignore while we debug...
@Test(enabled = false)
public void doTest() throws IOException public void doTest() throws IOException
{ {
final String indexDatasource = "wikipedia_http_inputsource_test_" + UUID.randomUUID(); final String indexDatasource = "wikipedia_http_inputsource_test_" + UUID.randomUUID();

View File

@ -16,82 +16,31 @@
] ]
}, },
{ {
"description": "timeseries, datasketch aggs, all", "description": "simple aggr",
"query":{ "query":{
"queryType" : "timeseries", "queryType" : "topN",
"dataSource": "%%DATASOURCE%%", "dataSource" : "%%DATASOURCE%%",
"granularity":"day", "intervals" : ["2016-06-27/2016-06-28"],
"intervals":[ "granularity" : "all",
"2016-06-27/P1D" "dimension" : "page",
], "metric" : "count",
"filter":null, "threshold" : 3,
"aggregations":[ "aggregations" : [
{ {
"type": "HLLSketchMerge", "type" : "count",
"name": "approxCountHLL", "name" : "count"
"fieldName": "HLLSketchBuild",
"lgK": 12,
"tgtHllType": "HLL_4",
"round": true
},
{
"type":"thetaSketch",
"name":"approxCountTheta",
"fieldName":"thetaSketch",
"size":16384,
"shouldFinalize":true,
"isInputThetaSketch":false,
"errorBoundsStdDev":null
},
{
"type":"quantilesDoublesSketch",
"name":"quantilesSketch",
"fieldName":"quantilesDoublesSketch",
"k":128
} }
] ]
}, },
"expectedResults":[ "expectedResults":[
{ {
"timestamp" : "2016-06-27T00:00:00.000Z", "timestamp" : "2016-06-27T00:00:11.000Z",
"result" : { "result" :
"quantilesSketch":48866, [
"approxCountTheta":7862.0, {"count":58,"page":"Copa América Centenario"},
"approxCountHLL":7961 {"count":32,"page":"User:Cyde/List of candidates for speedy deletion/Subpage"},
} {"count":32,"page":"Wikipedia:Administrators' noticeboard/Incidents"}
}
] ]
},
{
"description": "timeseries, stringFirst/stringLast aggs, all",
"query":{
"queryType" : "timeseries",
"dataSource": "%%DATASOURCE%%",
"granularity":"day",
"intervals":[
"2016-06-27/P1D"
],
"filter":null,
"aggregations":[
{
"type": "stringFirst",
"name": "first_user",
"fieldName": "user"
},
{
"type":"stringLast",
"name":"last_user",
"fieldName":"user"
}
]
},
"expectedResults":[
{
"timestamp" : "2016-06-27T00:00:00.000Z",
"result" : {
"first_user":"Lsjbot",
"last_user":"EmausBot"
}
} }
] ]
} }

View File

@ -41,21 +41,6 @@
"type": "doubleSum", "type": "doubleSum",
"name": "delta", "name": "delta",
"fieldName": "delta" "fieldName": "delta"
},
{
"name": "thetaSketch",
"type": "thetaSketch",
"fieldName": "user"
},
{
"name": "quantilesDoublesSketch",
"type": "quantilesDoublesSketch",
"fieldName": "delta"
},
{
"name": "HLLSketchBuild",
"type": "HLLSketchBuild",
"fieldName": "user"
} }
], ],
"granularitySpec": { "granularitySpec": {
@ -78,9 +63,8 @@
"type": "index_parallel", "type": "index_parallel",
"maxNumConcurrentSubTasks": 10, "maxNumConcurrentSubTasks": 10,
"partitionsSpec": { "partitionsSpec": {
"type": "hashed" "type": "dynamic"
}, },
"forceGuaranteedRollup": true,
"splitHintSpec": { "splitHintSpec": {
"type": "maxSize", "type": "maxSize",
"maxNumFiles": 1 "maxNumFiles": 1