mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-18 19:05:06 +00:00
bf1550a0b2
This adds a new Rollup module to XPack, which allows users to configure periodic "rollup jobs" to pre-aggregate data. That data is then available later for search through a special RollupSearch API, which mimics the DSL and functionality of regular search. Rollups are used to drastically reduce the on-disk footprint of metric-based data (e.g. timestamped document with numeric and keyword fields). It can also be used to speed up aggregations over large datasets, since the rolled data will be considerably smaller and fewer documents to search. The PR adds seven new endpoints to interact with Rollups; create/get/delete job, start/stop job, a capabilities API similar to field-caps, and a Rollup-enabled search. Original commit: elastic/x-pack-elasticsearch@dcde91aacf
654 lines
27 KiB
Groovy
654 lines
27 KiB
Groovy
import org.elasticsearch.gradle.test.NodeInfo
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
|
|
apply plugin: 'elasticsearch.docs-test'
|
|
|
|
/* List of files that have snippets that probably should be converted to
|
|
* `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and
|
|
* only remove entries from this list. When it is empty we'll remove it
|
|
* entirely and have a party! There will be cake and everything.... */
|
|
buildRestTests.expectedUnconvertedCandidates = [
|
|
'en/ml/functions/count.asciidoc',
|
|
'en/ml/functions/geo.asciidoc',
|
|
'en/ml/functions/info.asciidoc',
|
|
'en/ml/functions/metric.asciidoc',
|
|
'en/ml/functions/rare.asciidoc',
|
|
'en/ml/functions/sum.asciidoc',
|
|
'en/ml/functions/time.asciidoc',
|
|
'en/ml/aggregations.asciidoc',
|
|
'en/ml/customurl.asciidoc',
|
|
'en/monitoring/indices.asciidoc',
|
|
'en/rest-api/security/ssl.asciidoc',
|
|
'en/rest-api/security/users.asciidoc',
|
|
'en/rest-api/security/tokens.asciidoc',
|
|
'en/rest-api/watcher/put-watch.asciidoc',
|
|
'en/security/authentication/user-cache.asciidoc',
|
|
'en/security/authorization/field-and-document-access-control.asciidoc',
|
|
'en/security/authorization/run-as-privilege.asciidoc',
|
|
'en/security/tribe-clients-integrations/http.asciidoc',
|
|
'en/security/tribe-clients-integrations/monitoring.asciidoc',
|
|
'en/security/authorization/custom-roles-provider.asciidoc',
|
|
'en/watcher/actions/email.asciidoc',
|
|
'en/watcher/actions/hipchat.asciidoc',
|
|
'en/watcher/actions/index.asciidoc',
|
|
'en/watcher/actions/logging.asciidoc',
|
|
'en/watcher/actions/pagerduty.asciidoc',
|
|
'en/watcher/actions/slack.asciidoc',
|
|
'en/watcher/actions/jira.asciidoc',
|
|
'en/watcher/actions/webhook.asciidoc',
|
|
'en/watcher/condition/always.asciidoc',
|
|
'en/watcher/condition/array-compare.asciidoc',
|
|
'en/watcher/condition/compare.asciidoc',
|
|
'en/watcher/condition/never.asciidoc',
|
|
'en/watcher/condition/script.asciidoc',
|
|
'en/watcher/customizing-watches.asciidoc',
|
|
'en/watcher/example-watches/example-watch-meetupdata.asciidoc',
|
|
'en/watcher/how-watcher-works.asciidoc',
|
|
'en/watcher/input/chain.asciidoc',
|
|
'en/watcher/input/http.asciidoc',
|
|
'en/watcher/input/search.asciidoc',
|
|
'en/watcher/input/simple.asciidoc',
|
|
'en/watcher/transform.asciidoc',
|
|
'en/watcher/transform/chain.asciidoc',
|
|
'en/watcher/transform/script.asciidoc',
|
|
'en/watcher/transform/search.asciidoc',
|
|
'en/watcher/trigger/schedule/cron.asciidoc',
|
|
'en/watcher/trigger/schedule/daily.asciidoc',
|
|
'en/watcher/trigger/schedule/hourly.asciidoc',
|
|
'en/watcher/trigger/schedule/interval.asciidoc',
|
|
'en/watcher/trigger/schedule/monthly.asciidoc',
|
|
'en/watcher/trigger/schedule/weekly.asciidoc',
|
|
'en/watcher/trigger/schedule/yearly.asciidoc',
|
|
'en/watcher/troubleshooting.asciidoc',
|
|
'en/ml/api-quickref.asciidoc',
|
|
'en/rest-api/ml/delete-calendar-event.asciidoc',
|
|
'en/rest-api/ml/delete-snapshot.asciidoc',
|
|
'en/rest-api/ml/forecast.asciidoc',
|
|
'en/rest-api/ml/get-bucket.asciidoc',
|
|
'en/rest-api/ml/get-job-stats.asciidoc',
|
|
'en/rest-api/ml/get-overall-buckets.asciidoc',
|
|
'en/rest-api/ml/get-category.asciidoc',
|
|
'en/rest-api/ml/get-record.asciidoc',
|
|
'en/rest-api/ml/get-influencer.asciidoc',
|
|
'en/rest-api/ml/get-snapshot.asciidoc',
|
|
'en/rest-api/ml/post-data.asciidoc',
|
|
'en/rest-api/ml/preview-datafeed.asciidoc',
|
|
'en/rest-api/ml/revert-snapshot.asciidoc',
|
|
'en/rest-api/ml/update-snapshot.asciidoc',
|
|
'en/rest-api/ml/validate-detector.asciidoc',
|
|
'en/rest-api/ml/validate-job.asciidoc',
|
|
'en/rest-api/security/authenticate.asciidoc',
|
|
'en/rest-api/watcher/stats.asciidoc',
|
|
'en/security/authorization.asciidoc',
|
|
'en/watcher/example-watches/watching-time-series-data.asciidoc',
|
|
]
|
|
|
|
dependencies {
|
|
testCompile project(path: xpackModule('core'), configuration: 'runtime')
|
|
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
|
|
testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
|
|
}
|
|
|
|
Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
|
|
File tmpFile = new File(node.cwd, 'wait.success')
|
|
// wait up to twenty seconds
|
|
final long stopTime = System.currentTimeMillis() + 20000L;
|
|
Exception lastException = null;
|
|
while (System.currentTimeMillis() < stopTime) {
|
|
lastException = null;
|
|
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
|
|
HttpURLConnection httpURLConnection = null;
|
|
try {
|
|
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection();
|
|
httpURLConnection.setRequestProperty("Authorization", "Basic " +
|
|
Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
|
|
httpURLConnection.setRequestMethod("GET");
|
|
httpURLConnection.setConnectTimeout(1000);
|
|
httpURLConnection.setReadTimeout(30000);
|
|
httpURLConnection.connect();
|
|
if (httpURLConnection.getResponseCode() == 200) {
|
|
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
|
|
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
|
|
}
|
|
break;
|
|
}
|
|
} catch (Exception e) {
|
|
logger.debug("failed to call cluster health", e)
|
|
lastException = e
|
|
} finally {
|
|
if (httpURLConnection != null) {
|
|
httpURLConnection.disconnect();
|
|
}
|
|
}
|
|
|
|
// did not start, so wait a bit before trying again
|
|
Thread.sleep(500L);
|
|
}
|
|
if (tmpFile.exists() == false && lastException != null) {
|
|
logger.error("final attempt of calling cluster health failed", lastException)
|
|
}
|
|
return tmpFile.exists()
|
|
}
|
|
|
|
integTestCluster {
|
|
plugin xpackProject('plugin').path
|
|
setting 'xpack.security.authc.token.enabled', 'true'
|
|
// Disable monitoring exporters for the docs tests
|
|
setting 'xpack.monitoring.exporters._local.type', 'local'
|
|
setting 'xpack.monitoring.exporters._local.enabled', 'false'
|
|
setting 'xpack.license.self_generated.type', 'trial'
|
|
setupCommand 'setupTestAdmin',
|
|
'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
|
|
waitCondition = waitWithAuth
|
|
}
|
|
|
|
|
|
|
|
buildRestTests.docs = fileTree(projectDir) {
|
|
// No snippets in here!
|
|
exclude 'build.gradle'
|
|
// That is where the snippets go, not where they come from!
|
|
exclude 'build'
|
|
// These file simply doesn't pass yet. We should figure out how to fix them.
|
|
exclude 'en/watcher/reference/actions.asciidoc'
|
|
exclude 'en/rest-api/graph/explore.asciidoc'
|
|
}
|
|
|
|
Map<String, String> setups = buildRestTests.setups
|
|
setups['my_inactive_watch'] = '''
|
|
- do:
|
|
xpack.watcher.put_watch:
|
|
id: "my_watch"
|
|
active: false
|
|
body: >
|
|
{
|
|
"trigger": {
|
|
"schedule": {
|
|
"hourly": {
|
|
"minute": [ 0, 5 ]
|
|
}
|
|
}
|
|
},
|
|
"input": {
|
|
"simple": {
|
|
"payload": {
|
|
"send": "yes"
|
|
}
|
|
}
|
|
},
|
|
"condition": {
|
|
"always": {}
|
|
},
|
|
"actions": {
|
|
"test_index": {
|
|
"index": {
|
|
"index": "test",
|
|
"doc_type": "test2"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
- match: { _id: "my_watch" }
|
|
'''
|
|
setups['my_active_watch'] = setups['my_inactive_watch'].replace(
|
|
'active: false', 'active: true')
|
|
|
|
// Used by SQL because it looks SQL-ish
|
|
setups['library'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: library
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 1
|
|
mappings:
|
|
book:
|
|
properties:
|
|
name:
|
|
type: text
|
|
fields:
|
|
keyword:
|
|
type: keyword
|
|
author:
|
|
type: text
|
|
fields:
|
|
keyword:
|
|
type: keyword
|
|
release_date:
|
|
type: date
|
|
page_count:
|
|
type: short
|
|
- do:
|
|
bulk:
|
|
index: library
|
|
type: book
|
|
refresh: true
|
|
body: |
|
|
{"index":{"_id": "Leviathan Wakes"}}
|
|
{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
|
|
{"index":{"_id": "Hyperion"}}
|
|
{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
|
|
{"index":{"_id": "Dune"}}
|
|
{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
|
|
{"index":{"_id": "Dune Messiah"}}
|
|
{"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331}
|
|
{"index":{"_id": "Children of Dune"}}
|
|
{"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408}
|
|
{"index":{"_id": "God Emperor of Dune"}}
|
|
{"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454}
|
|
{"index":{"_id": "Consider Phlebas"}}
|
|
{"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471}
|
|
{"index":{"_id": "Pandora's Star"}}
|
|
{"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768}
|
|
{"index":{"_id": "Revelation Space"}}
|
|
{"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585}
|
|
{"index":{"_id": "A Fire Upon the Deep"}}
|
|
{"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613}
|
|
{"index":{"_id": "Ender's Game"}}
|
|
{"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324}
|
|
{"index":{"_id": "1984"}}
|
|
{"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328}
|
|
{"index":{"_id": "Fahrenheit 451"}}
|
|
{"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227}
|
|
{"index":{"_id": "Brave New World"}}
|
|
{"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268}
|
|
{"index":{"_id": "Foundation"}}
|
|
{"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224}
|
|
{"index":{"_id": "The Giver"}}
|
|
{"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208}
|
|
{"index":{"_id": "Slaughterhouse-Five"}}
|
|
{"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275}
|
|
{"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}}
|
|
{"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180}
|
|
{"index":{"_id": "Snow Crash"}}
|
|
{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470}
|
|
{"index":{"_id": "Neuromancer"}}
|
|
{"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271}
|
|
{"index":{"_id": "The Handmaid's Tale"}}
|
|
{"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}
|
|
{"index":{"_id": "Starship Troopers"}}
|
|
{"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335}
|
|
{"index":{"_id": "The Left Hand of Darkness"}}
|
|
{"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304}
|
|
{"index":{"_id": "The Moon is a Harsh Mistress"}}
|
|
{"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288}
|
|
|
|
'''
|
|
setups['server_metrics_index'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: server-metrics
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 0
|
|
mappings:
|
|
metric:
|
|
properties:
|
|
timestamp:
|
|
type: date
|
|
total:
|
|
type: long
|
|
'''
|
|
setups['server_metrics_data'] = setups['server_metrics_index'] + '''
|
|
- do:
|
|
bulk:
|
|
index: server-metrics
|
|
type: metric
|
|
refresh: true
|
|
body: |
|
|
{"index": {"_id":"1177"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":40476}
|
|
{"index": {"_id":"1178"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":15287}
|
|
{"index": {"_id":"1179"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":-776}
|
|
{"index": {"_id":"1180"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":11366}
|
|
{"index": {"_id":"1181"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":3606}
|
|
{"index": {"_id":"1182"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":19006}
|
|
{"index": {"_id":"1183"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":38613}
|
|
{"index": {"_id":"1184"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":19516}
|
|
{"index": {"_id":"1185"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":-258}
|
|
{"index": {"_id":"1186"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":9551}
|
|
{"index": {"_id":"1187"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":11217}
|
|
{"index": {"_id":"1188"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":22557}
|
|
{"index": {"_id":"1189"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":40508}
|
|
{"index": {"_id":"1190"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":11887}
|
|
{"index": {"_id":"1191"}}
|
|
{"timestamp":"2017-03-23T13:00:00","total":31659}
|
|
'''
|
|
setups['server_metrics_job'] = setups['server_metrics_data'] + '''
|
|
- do:
|
|
xpack.ml.put_job:
|
|
job_id: "total-requests"
|
|
body: >
|
|
{
|
|
"description" : "Total sum of requests",
|
|
"analysis_config" : {
|
|
"bucket_span":"10m",
|
|
"detectors" :[
|
|
{
|
|
"detector_description": "Sum of total",
|
|
"function": "sum",
|
|
"field_name": "total"
|
|
}
|
|
]},
|
|
"data_description" : {
|
|
"time_field":"timestamp",
|
|
"time_format": "epoch_ms"
|
|
}
|
|
}
|
|
'''
|
|
setups['server_metrics_datafeed'] = setups['server_metrics_job'] + '''
|
|
- do:
|
|
xpack.ml.put_datafeed:
|
|
datafeed_id: "datafeed-total-requests"
|
|
body: >
|
|
{
|
|
"job_id":"total-requests",
|
|
"indexes":"server-metrics",
|
|
"types":"metric"
|
|
}
|
|
'''
|
|
setups['server_metrics_openjob'] = setups['server_metrics_datafeed'] + '''
|
|
- do:
|
|
xpack.ml.open_job:
|
|
job_id: "total-requests"
|
|
'''
|
|
setups['server_metrics_startdf'] = setups['server_metrics_openjob'] + '''
|
|
- do:
|
|
xpack.ml.start_datafeed:
|
|
datafeed_id: "datafeed-total-requests"
|
|
'''
|
|
setups['calendar_outages'] = '''
|
|
- do:
|
|
xpack.ml.put_calendar:
|
|
calendar_id: "planned-outages"
|
|
'''
|
|
setups['calendar_outages_addevent'] = setups['calendar_outages'] + '''
|
|
- do:
|
|
xpack.ml.post_calendar_events:
|
|
calendar_id: "planned-outages"
|
|
body: >
|
|
{ "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" }
|
|
|
|
|
|
'''
|
|
setups['calendar_outages_openjob'] = setups['server_metrics_openjob'] + '''
|
|
- do:
|
|
xpack.ml.put_calendar:
|
|
calendar_id: "planned-outages"
|
|
'''
|
|
setups['calendar_outages_addjob'] = setups['server_metrics_openjob'] + '''
|
|
- do:
|
|
xpack.ml.put_calendar:
|
|
calendar_id: "planned-outages"
|
|
body: >
|
|
{
|
|
"job_ids": ["total-requests"]
|
|
}
|
|
'''
|
|
setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + '''
|
|
- do:
|
|
xpack.ml.post_calendar_events:
|
|
calendar_id: "planned-outages"
|
|
body: >
|
|
{ "events" : [
|
|
{ "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"},
|
|
{ "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"},
|
|
{ "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"}
|
|
]}
|
|
'''
|
|
setups['role_mapping'] = '''
|
|
- do:
|
|
xpack.security.put_role_mapping:
|
|
name: "mapping1"
|
|
body: >
|
|
{
|
|
"enabled": true,
|
|
"roles": [ "user" ],
|
|
"rules": { "field": { "username": "*" } }
|
|
}
|
|
'''
|
|
setups['sensor_rollup_job'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: sensor-1
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 0
|
|
mappings:
|
|
metric:
|
|
properties:
|
|
timestamp:
|
|
type: date
|
|
temperature:
|
|
type: long
|
|
voltage:
|
|
type: float
|
|
node:
|
|
type: keyword
|
|
- do:
|
|
xpack.rollup.put_job:
|
|
id: "sensor"
|
|
body: >
|
|
{
|
|
"index_pattern": "sensor-*",
|
|
"rollup_index": "sensor_rollup",
|
|
"cron": "*/30 * * * * ?",
|
|
"size" :1000,
|
|
"groups" : {
|
|
"date_histogram": {
|
|
"field": "timestamp",
|
|
"interval": "1h",
|
|
"delay": "7d"
|
|
},
|
|
"terms": {
|
|
"fields": ["node"]
|
|
}
|
|
},
|
|
"metrics": [
|
|
{
|
|
"field": "temperature",
|
|
"metrics": ["min", "max", "sum"]
|
|
},
|
|
{
|
|
"field": "voltage",
|
|
"metrics": ["avg"]
|
|
}
|
|
]
|
|
}
|
|
'''
|
|
setups['sensor_started_rollup_job'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: sensor-1
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 0
|
|
mappings:
|
|
data:
|
|
properties:
|
|
timestamp:
|
|
type: date
|
|
temperature:
|
|
type: long
|
|
voltage:
|
|
type: float
|
|
node:
|
|
type: keyword
|
|
|
|
- do:
|
|
bulk:
|
|
index: sensor-1
|
|
type: data
|
|
refresh: true
|
|
body: |
|
|
{"index":{}}
|
|
{"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
|
|
{"index":{}}
|
|
{"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
|
|
{"index":{}}
|
|
{"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
|
|
{"index":{}}
|
|
{"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
|
|
{"index":{}}
|
|
{"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
|
|
{"index":{}}
|
|
{"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
|
|
|
|
- do:
|
|
xpack.rollup.put_job:
|
|
id: "sensor"
|
|
body: >
|
|
{
|
|
"index_pattern": "sensor-*",
|
|
"rollup_index": "sensor_rollup",
|
|
"cron": "* * * * * ?",
|
|
"size" :1000,
|
|
"groups" : {
|
|
"date_histogram": {
|
|
"field": "timestamp",
|
|
"interval": "1h",
|
|
"delay": "7d"
|
|
},
|
|
"terms": {
|
|
"fields": ["node"]
|
|
}
|
|
},
|
|
"metrics": [
|
|
{
|
|
"field": "temperature",
|
|
"metrics": ["min", "max", "sum"]
|
|
},
|
|
{
|
|
"field": "voltage",
|
|
"metrics": ["avg"]
|
|
}
|
|
]
|
|
}
|
|
- do:
|
|
xpack.rollup.start_job:
|
|
id: "sensor"
|
|
'''
|
|
|
|
setups['sensor_index'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: sensor-1
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 0
|
|
mappings:
|
|
data:
|
|
properties:
|
|
timestamp:
|
|
type: date
|
|
temperature:
|
|
type: long
|
|
voltage:
|
|
type: float
|
|
node:
|
|
type: keyword
|
|
'''
|
|
|
|
setups['sensor_prefab_data'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: sensor_rollup
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 0
|
|
mappings:
|
|
rollup:
|
|
properties:
|
|
node.terms.value:
|
|
type: keyword
|
|
temperature.sum.value:
|
|
type: double
|
|
temperature.max.value:
|
|
type: double
|
|
temperature.min.value:
|
|
type: double
|
|
timestamp.date_histogram.time_zone:
|
|
type: keyword
|
|
timestamp.date_histogram.interval:
|
|
type: keyword
|
|
timestamp.date_histogram.timestamp:
|
|
type: date
|
|
timestamp.date_histogram._count:
|
|
type: long
|
|
voltage.avg.value:
|
|
type: double
|
|
voltage.avg._count:
|
|
type: long
|
|
_rollup.id:
|
|
type: keyword
|
|
_rollup.version:
|
|
type: long
|
|
_meta:
|
|
_rollup:
|
|
sensor:
|
|
cron: "* * * * * ?"
|
|
rollup_index: "sensor_rollup"
|
|
index_pattern: "sensor-*"
|
|
timeout: "20s"
|
|
size: 1000
|
|
groups:
|
|
date_histogram:
|
|
delay: "7d"
|
|
field: "timestamp"
|
|
interval: "1h"
|
|
time_zone: "UTC"
|
|
terms:
|
|
fields:
|
|
- "node"
|
|
id: sensor
|
|
metrics:
|
|
- field: "temperature"
|
|
metrics:
|
|
- min
|
|
- max
|
|
- sum
|
|
- field: "voltage"
|
|
metrics:
|
|
- avg
|
|
|
|
- do:
|
|
bulk:
|
|
index: sensor_rollup
|
|
type: rollup
|
|
refresh: true
|
|
body: |
|
|
{"index":{}}
|
|
{"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
{"index":{}}
|
|
{"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
{"index":{}}
|
|
{"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
{"index":{}}
|
|
{"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
{"index":{}}
|
|
{"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
{"index":{}}
|
|
{"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"}
|
|
|
|
'''
|