import org.elasticsearch.gradle.info.BuildParams import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ apply plugin: 'elasticsearch.docs-test' apply plugin: 'elasticsearch.rest-resources' /* List of files that have snippets that will not work until platinum tests can occur ... */ buildRestTests.expectedUnconvertedCandidates = [ 'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc', 'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc', 'reference/ml/anomaly-detection/apis/get-bucket.asciidoc', 'reference/ml/anomaly-detection/apis/get-category.asciidoc', 'reference/ml/anomaly-detection/apis/get-influencer.asciidoc', 'reference/ml/anomaly-detection/apis/get-job-stats.asciidoc', 'reference/ml/anomaly-detection/apis/get-job.asciidoc', 'reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc', 'reference/ml/anomaly-detection/apis/get-record.asciidoc', 'reference/ml/anomaly-detection/apis/get-snapshot.asciidoc', 'reference/ml/anomaly-detection/apis/post-data.asciidoc', 'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc', 'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc', 'reference/ml/anomaly-detection/apis/update-job.asciidoc' ] restResources { restApi { includeCore '*' } } testClusters.integTest { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' if (BuildParams.isSnapshotBuild() == false) { systemProperty 'es.autoscaling_feature_flag_registered', 'true' systemProperty 'es.searchable_snapshots_feature_enabled', 'true' } setting 'xpack.autoscaling.enabled', 'true' setting 'xpack.eql.enabled', 'true' keystorePassword 's3cr3t' } // enable regexes in painless so our tests don't complain about example snippets that use them setting 'script.painless.regex.enabled', 'true' setting 'path.repo', "${buildDir}/cluster/shared/repo" Closure configFile = { extraConfigFile it, file("src/test/cluster/config/$it") } configFile 'analysis/example_word_list.txt' configFile 'analysis/hyphenation_patterns.xml' configFile 'analysis/synonym.txt' configFile 'analysis/stemmer_override.txt' configFile 'userdict_ja.txt' configFile 'userdict_ko.txt' configFile 'KeywordTokenizer.rbbi' extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff') extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic') // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 systemProperty 'es.transport.cname_in_publish_address', 'true' } // build the cluster with all plugins project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj -> /* Skip repositories. We just aren't going to be able to test them so it * doesn't make sense to waste time installing them. */ if (subproj.path.startsWith(':plugins:repository-')) { return } // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported if (subproj.path.startsWith(':plugins:ingest-attachment') && BuildParams.inFipsJvm) { return } testClusters.integTest.plugin subproj.path } buildRestTests.docs = fileTree(projectDir) { // No snippets in here! exclude 'build.gradle' // That is where the snippets go, not where they come from! exclude 'build' // Just syntax examples exclude 'README.asciidoc' // Broken code snippet tests exclude 'reference/graph/explore.asciidoc' if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) { // We don't install/support this plugin in FIPS 140 exclude 'plugins/ingest-attachment.asciidoc' // We can't conditionally control output, this would be missing the ingest-attachment plugin exclude 'reference/cat/plugins.asciidoc' } } listSnippets.docs = buildRestTests.docs Closure setupMyIndex = { String name, int count -> buildRestTests.setups[name] = ''' - do: indices.create: index: my-index-000001 body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: "@timestamp": type: date http: properties: request: properties: method: type: keyword message: type: text user: properties: id: type: keyword doc_values: true - do: bulk: index: my-index-000001 refresh: true body: |''' for (int i = 0; i < count; i++) { String ip, user_id if (i == 0) { ip = '127.0.0.1' user_id = 'kimchy' } else { ip = '10.42.42.42' user_id= 'elkbee' } buildRestTests.setups[name] += """ { "index":{"_id": "$i"} } { "@timestamp": "2099-11-15T14:12:12", "http": { "request": { "method": "get" }, "response": { "bytes": 1070000, "status_code": 200 }, "version": "1.1" }, "message": "GET /search HTTP/1.1 200 1070000", "source": { "ip": "$ip" }, "user": { "id": "$user_id" } }""" } } setupMyIndex('my_index', 5) setupMyIndex('my_index_big', 120) setupMyIndex('my_index_huge', 1200) // Used for several full-text search and agg examples buildRestTests.setups['messages'] = ''' - do: indices.create: index: my-index-000001 body: settings: number_of_shards: 1 number_of_replicas: 1 - do: bulk: index: my-index-000001 refresh: true body: | {"index":{"_id": "0"}} {"message": "trying out Elasticsearch"} {"index":{"_id": "1"}} {"message": "some message with the number 1"} {"index":{"_id": "2"}} {"message": "some message with the number 2"} {"index":{"_id": "3"}} {"message": "some message with the number 3"} {"index":{"_id": "4"}} {"message": "some message with the number 4"}''' // Used for EQL buildRestTests.setups['sec_logs'] = ''' - do: indices.create: index: my-index-000001 body: settings: number_of_shards: 1 number_of_replicas: 1 - do: bulk: index: my-index-000001 refresh: true body: | {"index":{}} {"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} {"index":{}} {"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} {"index":{}} {"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } {"index":{}} {"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} {"index":{}} {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }} {"index":{}} {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' buildRestTests.setups['host'] = ''' # Fetch the http host. We use the host of the master because we know there will always be a master. - do: cluster.state: {} - set: { master_node: master } - do: nodes.info: metric: [ http, transport ] - set: {nodes.$master.http.publish_address: host} - set: {nodes.$master.transport.publish_address: transport_host} ''' buildRestTests.setups['node'] = ''' # Fetch the node name. We use the host of the master because we know there will always be a master. - do: cluster.state: {} - is_true: master_node - set: { master_node: node_name } ''' // Used by scripted metric docs buildRestTests.setups['ledger'] = ''' - do: indices.create: index: ledger body: settings: number_of_shards: 2 number_of_replicas: 1 mappings: properties: type: type: keyword amount: type: double - do: bulk: index: ledger refresh: true body: | {"index":{}} {"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"} {"index":{}} {"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "description": "another thing"} {"index":{}} {"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"} {"index":{}} {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"} {"index":{}} {"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}''' // Used by aggregation docs buildRestTests.setups['sales'] = ''' - do: indices.create: index: sales body: settings: number_of_shards: 2 number_of_replicas: 1 mappings: properties: type: type: keyword - do: bulk: index: sales refresh: true body: | {"index":{}} {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} {"index":{}} {"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"} {"index":{}} {"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"} {"index":{}} {"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"} {"index":{}} {"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"} {"index":{}} {"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"} {"index":{}} {"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}''' // Used by cumulative cardinality aggregation docs buildRestTests.setups['user_hits'] = ''' - do: indices.create: index: user_hits body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: user_id: type: keyword timestamp: type: date - do: bulk: index: user_hits refresh: true body: | {"index":{}} {"timestamp": "2019-01-01T13:00:00", "user_id": "1"} {"index":{}} {"timestamp": "2019-01-01T13:00:00", "user_id": "2"} {"index":{}} {"timestamp": "2019-01-02T13:00:00", "user_id": "1"} {"index":{}} {"timestamp": "2019-01-02T13:00:00", "user_id": "3"} {"index":{}} {"timestamp": "2019-01-03T13:00:00", "user_id": "1"} {"index":{}} {"timestamp": "2019-01-03T13:00:00", "user_id": "2"} {"index":{}} {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}''' // Fake bank account data used by getting-started.asciidoc buildRestTests.setups['bank'] = ''' - do: indices.create: index: bank body: settings: number_of_shards: 5 number_of_routing_shards: 5 - do: bulk: index: bank refresh: true body: | #bank_data# ''' /* Load the actual accounts only if we're going to use them. This complicates * dependency checking but that is a small price to pay for not building a * 400kb string every time we start the build. */ File accountsFile = new File("$projectDir/src/test/resources/accounts.json") buildRestTests.inputs.file(accountsFile) buildRestTests.doFirst { String accounts = accountsFile.getText('UTF-8') // Indent like a yaml test needs accounts = accounts.replaceAll('(?m)^', ' ') buildRestTests.setups['bank'] = buildRestTests.setups['bank'].replace('#bank_data#', accounts) } // Used by sampler and diversified-sampler aggregation docs buildRestTests.setups['stackoverflow'] = ''' - do: indices.create: index: stackoverflow body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: author: type: keyword tags: type: keyword - do: bulk: index: stackoverflow refresh: true body: |''' // Make Kibana strongly connected to elasticsearch and logstash // Make Kibana rarer (and therefore higher-ranking) than JavaScript // Make JavaScript strongly connected to jquery and angular // Make Cabana strongly connected to elasticsearch but only as a result of a single author for (int i = 0; i < 150; i++) { buildRestTests.setups['stackoverflow'] += """ {"index":{}} {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}""" } for (int i = 0; i < 50; i++) { buildRestTests.setups['stackoverflow'] += """ {"index":{}} {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}""" } for (int i = 0; i < 200; i++) { buildRestTests.setups['stackoverflow'] += """ {"index":{}} {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}""" } for (int i = 0; i < 200; i++) { buildRestTests.setups['stackoverflow'] += """ {"index":{}} {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}""" } for (int i = 0; i < 50; i++) { buildRestTests.setups['stackoverflow'] += """ {"index":{}} {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}""" } buildRestTests.setups['stackoverflow'] += """ """ // Used by significant_text aggregation docs buildRestTests.setups['news'] = ''' - do: indices.create: index: news body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: source: type: keyword content: type: text - do: bulk: index: news refresh: true body: |''' // Make h5n1 strongly connected to bird flu for (int i = 0; i < 100; i++) { buildRestTests.setups['news'] += """ {"index":{}} {"source": "very_relevant_$i", "content": "bird flu h5n1"}""" } for (int i = 0; i < 100; i++) { buildRestTests.setups['news'] += """ {"index":{}} {"source": "filler_$i", "content": "bird dupFiller "}""" } for (int i = 0; i < 100; i++) { buildRestTests.setups['news'] += """ {"index":{}} {"source": "filler_$i", "content": "flu dupFiller "}""" } for (int i = 0; i < 20; i++) { buildRestTests.setups['news'] += """ {"index":{}} {"source": "partially_relevant_$i", "content": "elasticsearch dupFiller dupFiller dupFiller dupFiller pozmantier"}""" } for (int i = 0; i < 10; i++) { buildRestTests.setups['news'] += """ {"index":{}} {"source": "partially_relevant_$i", "content": "elasticsearch logstash kibana"}""" } buildRestTests.setups['news'] += """ """ // Used by some aggregations buildRestTests.setups['exams'] = ''' - do: indices.create: index: exams body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: grade: type: byte - do: bulk: index: exams refresh: true body: | {"index":{}} {"grade": 100, "weight": 2} {"index":{}} {"grade": 50, "weight": 3}''' buildRestTests.setups['stored_example_script'] = ''' # Simple script to load a field. Not really a good example, but a simple one. - do: put_script: id: "my_script" body: { "script": { "lang": "painless", "source": "doc[params.field].value" } } - match: { acknowledged: true } ''' buildRestTests.setups['stored_scripted_metric_script'] = ''' - do: put_script: id: "my_init_script" body: { "script": { "lang": "painless", "source": "state.transactions = []" } } - match: { acknowledged: true } - do: put_script: id: "my_map_script" body: { "script": { "lang": "painless", "source": "state.transactions.add(doc.type.value == 'sale' ? doc.amount.value : -1 * doc.amount.value)" } } - match: { acknowledged: true } - do: put_script: id: "my_combine_script" body: { "script": { "lang": "painless", "source": "double profit = 0;for (t in state.transactions) { profit += t; } return profit" } } - match: { acknowledged: true } - do: put_script: id: "my_reduce_script" body: { "script": { "lang": "painless", "source": "double profit = 0;for (a in states) { profit += a; } return profit" } } - match: { acknowledged: true } ''' // Used by analyze api buildRestTests.setups['analyze_sample'] = ''' - do: indices.create: index: analyze_sample body: settings: number_of_shards: 1 number_of_replicas: 0 analysis: normalizer: my_normalizer: type: custom filter: [lowercase] mappings: properties: obj1.field1: type: text''' // Used by percentile/percentile-rank aggregations buildRestTests.setups['latency'] = ''' - do: indices.create: index: latency body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: load_time: type: long - do: bulk: index: latency refresh: true body: |''' for (int i = 0; i < 100; i++) { def value = i if (i % 10) { value = i * 10 } buildRestTests.setups['latency'] += """ {"index":{}} {"load_time": "$value"}""" } // Used by t_test aggregations buildRestTests.setups['node_upgrade'] = ''' - do: indices.create: index: node_upgrade body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: group: type: keyword startup_time_before: type: long startup_time_after: type: long - do: bulk: index: node_upgrade refresh: true body: | {"index":{}} {"group": "A", "startup_time_before": 102, "startup_time_after": 89} {"index":{}} {"group": "A", "startup_time_before": 99, "startup_time_after": 93} {"index":{}} {"group": "A", "startup_time_before": 111, "startup_time_after": 72} {"index":{}} {"group": "B", "startup_time_before": 97, "startup_time_after": 98} {"index":{}} {"group": "B", "startup_time_before": 101, "startup_time_after": 102} {"index":{}} {"group": "B", "startup_time_before": 99, "startup_time_after": 98}''' // Used by iprange agg buildRestTests.setups['iprange'] = ''' - do: indices.create: index: ip_addresses body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: ip: type: ip - do: bulk: index: ip_addresses refresh: true body: |''' for (int i = 0; i < 255; i++) { buildRestTests.setups['iprange'] += """ {"index":{}} {"ip": "10.0.0.$i"}""" } for (int i = 0; i < 5; i++) { buildRestTests.setups['iprange'] += """ {"index":{}} {"ip": "9.0.0.$i"}""" buildRestTests.setups['iprange'] += """ {"index":{}} {"ip": "11.0.0.$i"}""" buildRestTests.setups['iprange'] += """ {"index":{}} {"ip": "12.0.0.$i"}""" } // Used by SQL because it looks SQL-ish buildRestTests.setups['library'] = ''' - do: indices.create: include_type_name: true index: library body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: book: properties: name: type: text fields: keyword: type: keyword author: type: text fields: keyword: type: keyword release_date: type: date page_count: type: short - do: bulk: index: library type: book refresh: true body: | {"index":{"_id": "Leviathan Wakes"}} {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} {"index":{"_id": "Hyperion"}} {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} {"index":{"_id": "Dune"}} {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} {"index":{"_id": "Dune Messiah"}} {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} {"index":{"_id": "Children of Dune"}} {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} {"index":{"_id": "God Emperor of Dune"}} {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} {"index":{"_id": "Consider Phlebas"}} {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} {"index":{"_id": "Pandora's Star"}} {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} {"index":{"_id": "Revelation Space"}} {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} {"index":{"_id": "A Fire Upon the Deep"}} {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} {"index":{"_id": "Ender's Game"}} {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} {"index":{"_id": "1984"}} {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} {"index":{"_id": "Fahrenheit 451"}} {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} {"index":{"_id": "Brave New World"}} {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} {"index":{"_id": "Foundation"}} {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} {"index":{"_id": "The Giver"}} {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} {"index":{"_id": "Slaughterhouse-Five"}} {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}} {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} {"index":{"_id": "Snow Crash"}} {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} {"index":{"_id": "Neuromancer"}} {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} {"index":{"_id": "The Handmaid's Tale"}} {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} {"index":{"_id": "Starship Troopers"}} {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} {"index":{"_id": "The Left Hand of Darkness"}} {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} {"index":{"_id": "The Moon is a Harsh Mistress"}} {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ''' buildRestTests.setups['sensor_rollup_job'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: raw: method: PUT path: _rollup/job/sensor body: > { "index_pattern": "sensor-*", "rollup_index": "sensor_rollup", "cron": "*/30 * * * * ?", "page_size" :1000, "groups" : { "date_histogram": { "field": "timestamp", "fixed_interval": "1h", "delay": "7d" }, "terms": { "fields": ["node"] } }, "metrics": [ { "field": "temperature", "metrics": ["min", "max", "sum"] }, { "field": "voltage", "metrics": ["avg"] } ] } ''' buildRestTests.setups['sensor_started_rollup_job'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: bulk: index: sensor-1 refresh: true body: | {"index":{}} {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} {"index":{}} {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} {"index":{}} {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} {"index":{}} {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} {"index":{}} {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} {"index":{}} {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} - do: raw: method: PUT path: _rollup/job/sensor body: > { "index_pattern": "sensor-*", "rollup_index": "sensor_rollup", "cron": "* * * * * ?", "page_size" :1000, "groups" : { "date_histogram": { "field": "timestamp", "fixed_interval": "1h", "delay": "7d" }, "terms": { "fields": ["node"] } }, "metrics": [ { "field": "temperature", "metrics": ["min", "max", "sum"] }, { "field": "voltage", "metrics": ["avg"] } ] } - do: raw: method: POST path: _rollup/job/sensor/_start ''' buildRestTests.setups['sensor_index'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword load: type: double net_in: type: long net_out: type: long hostname: type: keyword datacenter: type: keyword ''' buildRestTests.setups['sensor_prefab_data'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: indices.create: index: sensor_rollup body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: node.terms.value: type: keyword temperature.sum.value: type: double temperature.max.value: type: double temperature.min.value: type: double timestamp.date_histogram.time_zone: type: keyword timestamp.date_histogram.interval: type: keyword timestamp.date_histogram.timestamp: type: date timestamp.date_histogram._count: type: long voltage.avg.value: type: double voltage.avg._count: type: long _rollup.id: type: keyword _rollup.version: type: long _meta: _rollup: sensor: cron: "* * * * * ?" rollup_index: "sensor_rollup" index_pattern: "sensor-*" timeout: "20s" page_size: 1000 groups: date_histogram: delay: "7d" field: "timestamp" fixed_interval: "60m" time_zone: "UTC" terms: fields: - "node" id: sensor metrics: - field: "temperature" metrics: - min - max - sum - field: "voltage" metrics: - avg - do: bulk: index: sensor_rollup refresh: true body: | {"index":{}} {"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} ''' buildRestTests.setups['sample_job'] = ''' - do: ml.put_job: job_id: "sample_job" body: > { "description" : "Very basic job", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "function": "count" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' buildRestTests.setups['farequote_index'] = ''' - do: indices.create: index: farequote body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: time: type: date responsetime: type: float airline: type: keyword doc_count: type: integer ''' buildRestTests.setups['farequote_data'] = buildRestTests.setups['farequote_index'] + ''' - do: bulk: index: farequote type: metric refresh: true body: | {"index": {"_id":"1"}} {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5} {"index": {"_id":"2"}} {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23} {"index": {"_id":"3"}} {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42} ''' buildRestTests.setups['farequote_job'] = buildRestTests.setups['farequote_data'] + ''' - do: ml.put_job: job_id: "farequote" body: > { "analysis_config": { "bucket_span": "60m", "detectors": [{ "function": "mean", "field_name": "responsetime", "by_field_name": "airline" }], "summary_count_field_name": "doc_count" }, "data_description": { "time_field": "time" } } ''' buildRestTests.setups['farequote_datafeed'] = buildRestTests.setups['farequote_job'] + ''' - do: ml.put_datafeed: datafeed_id: "datafeed-farequote" body: > { "job_id":"farequote", "indexes":"farequote" } ''' buildRestTests.setups['server_metrics_index'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: timestamp: type: date total: type: long ''' buildRestTests.setups['server_metrics_data'] = buildRestTests.setups['server_metrics_index'] + ''' - do: bulk: index: server-metrics type: metric refresh: true body: | {"index": {"_id":"1177"}} {"timestamp":"2017-03-23T13:00:00","total":40476} {"index": {"_id":"1178"}} {"timestamp":"2017-03-23T13:00:00","total":15287} {"index": {"_id":"1179"}} {"timestamp":"2017-03-23T13:00:00","total":-776} {"index": {"_id":"1180"}} {"timestamp":"2017-03-23T13:00:00","total":11366} {"index": {"_id":"1181"}} {"timestamp":"2017-03-23T13:00:00","total":3606} {"index": {"_id":"1182"}} {"timestamp":"2017-03-23T13:00:00","total":19006} {"index": {"_id":"1183"}} {"timestamp":"2017-03-23T13:00:00","total":38613} {"index": {"_id":"1184"}} {"timestamp":"2017-03-23T13:00:00","total":19516} {"index": {"_id":"1185"}} {"timestamp":"2017-03-23T13:00:00","total":-258} {"index": {"_id":"1186"}} {"timestamp":"2017-03-23T13:00:00","total":9551} {"index": {"_id":"1187"}} {"timestamp":"2017-03-23T13:00:00","total":11217} {"index": {"_id":"1188"}} {"timestamp":"2017-03-23T13:00:00","total":22557} {"index": {"_id":"1189"}} {"timestamp":"2017-03-23T13:00:00","total":40508} {"index": {"_id":"1190"}} {"timestamp":"2017-03-23T13:00:00","total":11887} {"index": {"_id":"1191"}} {"timestamp":"2017-03-23T13:00:00","total":31659} ''' buildRestTests.setups['server_metrics_job'] = buildRestTests.setups['server_metrics_data'] + ''' - do: ml.put_job: job_id: "total-requests" body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' buildRestTests.setups['server_metrics_job-raw'] = buildRestTests.setups['server_metrics_data'] + ''' - do: raw: method: PUT path: _ml/anomaly_detectors/total-requests body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' buildRestTests.setups['server_metrics_datafeed'] = buildRestTests.setups['server_metrics_job'] + ''' - do: ml.put_datafeed: datafeed_id: "datafeed-total-requests" body: > { "job_id":"total-requests", "indexes":"server-metrics" } ''' buildRestTests.setups['server_metrics_datafeed-raw'] = buildRestTests.setups['server_metrics_job-raw'] + ''' - do: raw: method: PUT path: _ml/datafeeds/datafeed-total-requests body: > { "job_id":"total-requests", "indexes":"server-metrics" } ''' buildRestTests.setups['server_metrics_openjob'] = buildRestTests.setups['server_metrics_datafeed'] + ''' - do: ml.open_job: job_id: "total-requests" ''' buildRestTests.setups['server_metrics_openjob-raw'] = buildRestTests.setups['server_metrics_datafeed-raw'] + ''' - do: raw: method: POST path: _ml/anomaly_detectors/total-requests/_open ''' buildRestTests.setups['server_metrics_startdf'] = buildRestTests.setups['server_metrics_openjob'] + ''' - do: ml.start_datafeed: datafeed_id: "datafeed-total-requests" ''' buildRestTests.setups['calendar_outages'] = ''' - do: ml.put_calendar: calendar_id: "planned-outages" ''' buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['calendar_outages'] + ''' - do: ml.post_calendar_events: calendar_id: "planned-outages" body: > { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" } ''' buildRestTests.setups['calendar_outages_openjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' - do: ml.put_calendar: calendar_id: "planned-outages" ''' buildRestTests.setups['calendar_outages_addjob'] = buildRestTests.setups['server_metrics_openjob'] + ''' - do: ml.put_calendar: calendar_id: "planned-outages" body: > { "job_ids": ["total-requests"] } ''' buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['calendar_outages_addjob'] + ''' - do: ml.post_calendar_events: calendar_id: "planned-outages" body: > { "events" : [ { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"}, { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"}, { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"} ]} ''' // used by median absolute deviation aggregation buildRestTests.setups['reviews'] = ''' - do: indices.create: index: reviews body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: product: type: keyword rating: type: long - do: bulk: index: reviews refresh: true body: | {"index": {"_id": "1"}} {"product": "widget-foo", "rating": 1} {"index": {"_id": "2"}} {"product": "widget-foo", "rating": 5} ''' buildRestTests.setups['remote_cluster'] = buildRestTests.setups['host'] + ''' - do: cluster.put_settings: body: persistent: cluster.remote.remote_cluster.seeds: $transport_host ''' buildRestTests.setups['remote_cluster_and_leader_index'] = buildRestTests.setups['remote_cluster'] + ''' - do: indices.create: index: leader_index body: settings: index.number_of_replicas: 0 index.number_of_shards: 1 index.soft_deletes.enabled: true ''' buildRestTests.setups['seats'] = ''' - do: indices.create: index: seats body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: properties: theatre: type: keyword cost: type: long row: type: long number: type: long sold: type: boolean - do: bulk: index: seats refresh: true body: | {"index":{"_id": "1"}} {"theatre": "Skyline", "cost": 37, "row": 1, "number": 7, "sold": false} {"index":{"_id": "2"}} {"theatre": "Graye", "cost": 30, "row": 3, "number": 5, "sold": false} {"index":{"_id": "3"}} {"theatre": "Graye", "cost": 33, "row": 2, "number": 6, "sold": false} {"index":{"_id": "4"}} {"theatre": "Skyline", "cost": 20, "row": 5, "number": 2, "sold": false}''' buildRestTests.setups['kibana_sample_data_ecommerce'] = ''' - do: indices.create: index: kibana_sample_data_ecommerce body: settings: number_of_shards: 1 number_of_replicas: 0 ''' buildRestTests.setups['add_timestamp_pipeline'] = ''' - do: ingest.put_pipeline: id: "add_timestamp_pipeline" body: > { "processors": [ { "set" : { "field" : "@timestamp", "value" : "{{_ingest.timestamp}}" } } ] } ''' buildRestTests.setups['simple_kibana_continuous_pivot'] = buildRestTests.setups['kibana_sample_data_ecommerce'] + buildRestTests.setups['add_timestamp_pipeline'] + ''' - do: raw: method: PUT path: _transform/simple-kibana-ecomm-pivot body: > { "source": { "index": "kibana_sample_data_ecommerce", "query": { "term": { "geoip.continent_name": { "value": "Asia" } } } }, "pivot": { "group_by": { "customer_id": { "terms": { "field": "customer_id" } } }, "aggregations": { "max_price": { "max": { "field": "taxful_total_price" } } } }, "description": "Maximum priced ecommerce data", "dest": { "index": "kibana_sample_data_ecommerce_transform", "pipeline": "add_timestamp_pipeline" }, "frequency": "5m", "sync": { "time": { "field": "order_date", "delay": "60s" } } } ''' buildRestTests.setups['setup_logdata'] = ''' - do: indices.create: index: logdata body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: properties: grade: type: byte - do: bulk: index: logdata refresh: true body: | {"index":{}} {"grade": 100, "weight": 2} {"index":{}} {"grade": 50, "weight": 3} ''' buildRestTests.setups['logdata_job'] = buildRestTests.setups['setup_logdata'] + ''' - do: ml.put_data_frame_analytics: id: "loganalytics" body: > { "source": { "index": "logdata" }, "dest": { "index": "logdata_out" }, "analysis": { "outlier_detection": {} } } ''' // Used by snapshot lifecycle management docs buildRestTests.setups['setup-repository'] = ''' - do: snapshot.create_repository: repository: my_repository body: type: fs settings: location: buildDir/cluster/shared/repo ''' // Fake sec logs data used by EQL search buildRestTests.setups['atomic_red_regsvr32'] = ''' - do: indices.create: index: my-index-000001 body: settings: number_of_shards: 5 number_of_routing_shards: 5 - do: bulk: index: my-index-000001 refresh: true body: | #atomic_red_data# ''' /* Load the actual events only if we're going to use them. */ File atomicRedRegsvr32File = new File("$projectDir/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json") buildRestTests.inputs.file(atomicRedRegsvr32File) buildRestTests.doFirst { String events = atomicRedRegsvr32File.getText('UTF-8') // Indent like a yaml test needs events = events.replaceAll('(?m)^', ' ') buildRestTests.setups['atomic_red_regsvr32'] = buildRestTests.setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events) }