import org.elasticsearch.gradle.test.NodeInfo import java.nio.charset.StandardCharsets apply plugin: 'elasticsearch.docs-test' /* List of files that have snippets that probably should be converted to * `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and * only remove entries from this list. When it is empty we'll remove it * entirely and have a party! There will be cake and everything.... */ buildRestTests.expectedUnconvertedCandidates = [ 'en/rest-api/watcher/put-watch.asciidoc', 'en/security/authentication/user-cache.asciidoc', 'en/security/authorization/run-as-privilege.asciidoc', 'en/security/ccs-clients-integrations/http.asciidoc', 'en/security/authorization/custom-roles-provider.asciidoc', 'en/rest-api/ml/delete-snapshot.asciidoc', 'en/rest-api/ml/get-bucket.asciidoc', 'en/rest-api/ml/get-job-stats.asciidoc', 'en/rest-api/ml/get-overall-buckets.asciidoc', 'en/rest-api/ml/get-category.asciidoc', 'en/rest-api/ml/get-record.asciidoc', 'en/rest-api/ml/get-influencer.asciidoc', 'en/rest-api/ml/get-snapshot.asciidoc', 'en/rest-api/ml/post-data.asciidoc', 'en/rest-api/ml/revert-snapshot.asciidoc', 'en/rest-api/ml/update-snapshot.asciidoc', 'en/rest-api/watcher/stats.asciidoc', 'en/watcher/example-watches/watching-time-series-data.asciidoc', ] dependencies { testCompile project(path: xpackModule('core'), configuration: 'shadow') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> File tmpFile = new File(node.cwd, 'wait.success') // wait up to twenty seconds final long stopTime = System.currentTimeMillis() + 20000L; Exception lastException = null; while (System.currentTimeMillis() < stopTime) { lastException = null; // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned HttpURLConnection httpURLConnection = null; try { httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection(); httpURLConnection.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); httpURLConnection.setRequestMethod("GET"); httpURLConnection.setConnectTimeout(1000); httpURLConnection.setReadTimeout(30000); httpURLConnection.connect(); if (httpURLConnection.getResponseCode() == 200) { tmpFile.withWriter StandardCharsets.UTF_8.name(), { it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) } break; } } catch (Exception e) { logger.debug("failed to call cluster health", e) lastException = e } finally { if (httpURLConnection != null) { httpURLConnection.disconnect(); } } // did not start, so wait a bit before trying again Thread.sleep(500L); } if (tmpFile.exists() == false && lastException != null) { logger.error("final attempt of calling cluster health failed", lastException) } return tmpFile.exists() } // copy xpack rest api File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') project.copyRestSpec.from(xpackResources) { include 'rest-api-spec/api/**' } integTestCluster { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Disable monitoring exporters for the docs tests setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setupCommand 'setupTestAdmin', 'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser' waitCondition = waitWithAuth } buildRestTests.docs = fileTree(projectDir) { // No snippets in here! exclude 'build.gradle' // That is where the snippets go, not where they come from! exclude 'build' // These file simply doesn't pass yet. We should figure out how to fix them. exclude 'en/watcher/reference/actions.asciidoc' exclude 'en/rest-api/graph/explore.asciidoc' } Map setups = buildRestTests.setups setups['my_inactive_watch'] = ''' - do: xpack.watcher.put_watch: id: "my_watch" active: false body: > { "trigger": { "schedule": { "hourly": { "minute": [ 0, 5 ] } } }, "input": { "simple": { "payload": { "send": "yes" } } }, "condition": { "always": {} }, "actions": { "test_index": { "index": { "index": "test", "doc_type": "test2" } } } } - match: { _id: "my_watch" } ''' setups['my_active_watch'] = setups['my_inactive_watch'].replace( 'active: false', 'active: true') // Used by SQL because it looks SQL-ish setups['library'] = ''' - do: indices.create: index: library body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: book: properties: name: type: text fields: keyword: type: keyword author: type: text fields: keyword: type: keyword release_date: type: date page_count: type: short - do: bulk: index: library type: book refresh: true body: | {"index":{"_id": "Leviathan Wakes"}} {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} {"index":{"_id": "Hyperion"}} {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} {"index":{"_id": "Dune"}} {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} {"index":{"_id": "Dune Messiah"}} {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} {"index":{"_id": "Children of Dune"}} {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} {"index":{"_id": "God Emperor of Dune"}} {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} {"index":{"_id": "Consider Phlebas"}} {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} {"index":{"_id": "Pandora's Star"}} {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} {"index":{"_id": "Revelation Space"}} {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} {"index":{"_id": "A Fire Upon the Deep"}} {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} {"index":{"_id": "Ender's Game"}} {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} {"index":{"_id": "1984"}} {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} {"index":{"_id": "Fahrenheit 451"}} {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} {"index":{"_id": "Brave New World"}} {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} {"index":{"_id": "Foundation"}} {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} {"index":{"_id": "The Giver"}} {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} {"index":{"_id": "Slaughterhouse-Five"}} {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}} {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} {"index":{"_id": "Snow Crash"}} {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} {"index":{"_id": "Neuromancer"}} {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} {"index":{"_id": "The Handmaid's Tale"}} {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} {"index":{"_id": "Starship Troopers"}} {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} {"index":{"_id": "The Left Hand of Darkness"}} {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} {"index":{"_id": "The Moon is a Harsh Mistress"}} {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ''' setups['sample_job'] = ''' - do: xpack.ml.put_job: job_id: "sample_job" body: > { "description" : "Very basic job", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "function": "count" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' setups['farequote_index'] = ''' - do: indices.create: index: farequote body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: time: type: date responsetime: type: float airline: type: keyword doc_count: type: integer ''' setups['farequote_data'] = setups['farequote_index'] + ''' - do: bulk: index: farequote type: metric refresh: true body: | {"index": {"_id":"1"}} {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5} {"index": {"_id":"2"}} {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23} {"index": {"_id":"3"}} {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42} ''' setups['farequote_job'] = setups['farequote_data'] + ''' - do: xpack.ml.put_job: job_id: "farequote" body: > { "analysis_config": { "bucket_span": "60m", "detectors": [{ "function": "mean", "field_name": "responsetime", "by_field_name": "airline" }], "summary_count_field_name": "doc_count" }, "data_description": { "time_field": "time" } } ''' setups['farequote_datafeed'] = setups['farequote_job'] + ''' - do: xpack.ml.put_datafeed: datafeed_id: "datafeed-farequote" body: > { "job_id":"farequote", "indexes":"farequote" } ''' setups['ml_filter_safe_domains'] = ''' - do: xpack.ml.put_filter: filter_id: "safe_domains" body: > { "description": "A list of safe domains", "items": ["*.google.com", "wikipedia.org"] } ''' setups['server_metrics_index'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: timestamp: type: date total: type: long ''' setups['server_metrics_data'] = setups['server_metrics_index'] + ''' - do: bulk: index: server-metrics type: metric refresh: true body: | {"index": {"_id":"1177"}} {"timestamp":"2017-03-23T13:00:00","total":40476} {"index": {"_id":"1178"}} {"timestamp":"2017-03-23T13:00:00","total":15287} {"index": {"_id":"1179"}} {"timestamp":"2017-03-23T13:00:00","total":-776} {"index": {"_id":"1180"}} {"timestamp":"2017-03-23T13:00:00","total":11366} {"index": {"_id":"1181"}} {"timestamp":"2017-03-23T13:00:00","total":3606} {"index": {"_id":"1182"}} {"timestamp":"2017-03-23T13:00:00","total":19006} {"index": {"_id":"1183"}} {"timestamp":"2017-03-23T13:00:00","total":38613} {"index": {"_id":"1184"}} {"timestamp":"2017-03-23T13:00:00","total":19516} {"index": {"_id":"1185"}} {"timestamp":"2017-03-23T13:00:00","total":-258} {"index": {"_id":"1186"}} {"timestamp":"2017-03-23T13:00:00","total":9551} {"index": {"_id":"1187"}} {"timestamp":"2017-03-23T13:00:00","total":11217} {"index": {"_id":"1188"}} {"timestamp":"2017-03-23T13:00:00","total":22557} {"index": {"_id":"1189"}} {"timestamp":"2017-03-23T13:00:00","total":40508} {"index": {"_id":"1190"}} {"timestamp":"2017-03-23T13:00:00","total":11887} {"index": {"_id":"1191"}} {"timestamp":"2017-03-23T13:00:00","total":31659} ''' setups['server_metrics_job'] = setups['server_metrics_data'] + ''' - do: xpack.ml.put_job: job_id: "total-requests" body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' setups['server_metrics_datafeed'] = setups['server_metrics_job'] + ''' - do: xpack.ml.put_datafeed: datafeed_id: "datafeed-total-requests" body: > { "job_id":"total-requests", "indexes":"server-metrics" } ''' setups['server_metrics_openjob'] = setups['server_metrics_datafeed'] + ''' - do: xpack.ml.open_job: job_id: "total-requests" ''' setups['server_metrics_startdf'] = setups['server_metrics_openjob'] + ''' - do: xpack.ml.start_datafeed: datafeed_id: "datafeed-total-requests" ''' setups['calendar_outages'] = ''' - do: xpack.ml.put_calendar: calendar_id: "planned-outages" ''' setups['calendar_outages_addevent'] = setups['calendar_outages'] + ''' - do: xpack.ml.post_calendar_events: calendar_id: "planned-outages" body: > { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" } ''' setups['calendar_outages_openjob'] = setups['server_metrics_openjob'] + ''' - do: xpack.ml.put_calendar: calendar_id: "planned-outages" ''' setups['calendar_outages_addjob'] = setups['server_metrics_openjob'] + ''' - do: xpack.ml.put_calendar: calendar_id: "planned-outages" body: > { "job_ids": ["total-requests"] } ''' setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + ''' - do: xpack.ml.post_calendar_events: calendar_id: "planned-outages" body: > { "events" : [ { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"}, { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"}, { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"} ]} ''' setups['role_mapping'] = ''' - do: xpack.security.put_role_mapping: name: "mapping1" body: > { "enabled": true, "roles": [ "user" ], "rules": { "field": { "username": "*" } } } ''' setups['sensor_rollup_job'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: _doc: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: xpack.rollup.put_job: id: "sensor" body: > { "index_pattern": "sensor-*", "rollup_index": "sensor_rollup", "cron": "*/30 * * * * ?", "page_size" :1000, "groups" : { "date_histogram": { "field": "timestamp", "interval": "1h", "delay": "7d" }, "terms": { "fields": ["node"] } }, "metrics": [ { "field": "temperature", "metrics": ["min", "max", "sum"] }, { "field": "voltage", "metrics": ["avg"] } ] } ''' setups['sensor_started_rollup_job'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: _doc: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: bulk: index: sensor-1 type: _doc refresh: true body: | {"index":{}} {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} {"index":{}} {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} {"index":{}} {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} {"index":{}} {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} {"index":{}} {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} {"index":{}} {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} - do: xpack.rollup.put_job: id: "sensor" body: > { "index_pattern": "sensor-*", "rollup_index": "sensor_rollup", "cron": "* * * * * ?", "page_size" :1000, "groups" : { "date_histogram": { "field": "timestamp", "interval": "1h", "delay": "7d" }, "terms": { "fields": ["node"] } }, "metrics": [ { "field": "temperature", "metrics": ["min", "max", "sum"] }, { "field": "voltage", "metrics": ["avg"] } ] } - do: xpack.rollup.start_job: id: "sensor" ''' setups['sensor_index'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: _doc: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword load: type: double net_in: type: long net_out: type: long hostname: type: keyword datacenter: type: keyword ''' setups['sensor_prefab_data'] = ''' - do: indices.create: index: sensor-1 body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: _doc: properties: timestamp: type: date temperature: type: long voltage: type: float node: type: keyword - do: indices.create: index: sensor_rollup body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: _doc: properties: node.terms.value: type: keyword temperature.sum.value: type: double temperature.max.value: type: double temperature.min.value: type: double timestamp.date_histogram.time_zone: type: keyword timestamp.date_histogram.interval: type: keyword timestamp.date_histogram.timestamp: type: date timestamp.date_histogram._count: type: long voltage.avg.value: type: double voltage.avg._count: type: long _rollup.id: type: keyword _rollup.version: type: long _meta: _rollup: sensor: cron: "* * * * * ?" rollup_index: "sensor_rollup" index_pattern: "sensor-*" timeout: "20s" page_size: 1000 groups: date_histogram: delay: "7d" field: "timestamp" interval: "1h" time_zone: "UTC" terms: fields: - "node" id: sensor metrics: - field: "temperature" metrics: - min - max - sum - field: "voltage" metrics: - avg - do: bulk: index: sensor_rollup type: _doc refresh: true body: | {"index":{}} {"node.terms.value":"b","temperature.sum.value":201.0,"temperature.max.value":201.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":201.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.800000190734863,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516640400000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"c","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516381200000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"a","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.099999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516554000000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"a","temperature.sum.value":200.0,"temperature.max.value":200.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":200.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.199999809265137,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516726800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"b","temperature.sum.value":198.0,"temperature.max.value":198.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":198.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":5.599999904632568,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516467600000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} {"index":{}} {"node.terms.value":"c","temperature.sum.value":202.0,"temperature.max.value":202.0,"timestamp.date_histogram.time_zone":"UTC","temperature.min.value":202.0,"timestamp.date_histogram._count":1,"timestamp.date_histogram.interval":"1h","_rollup.computed":["temperature.sum","temperature.min","voltage.avg","temperature.max","node.terms","timestamp.date_histogram"],"voltage.avg.value":4.0,"node.terms._count":1,"_rollup.version":1,"timestamp.date_histogram.timestamp":1516294800000,"voltage.avg._count":1.0,"_rollup.id":"sensor"} ''' setups['admin_role'] = ''' - do: xpack.security.put_role: name: "my_admin_role" body: > { "cluster": ["all"], "indices": [ {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}} ], "run_as": [ "other_user" ], "metadata" : {"version": 1} } ''' setups['jacknich_user'] = ''' - do: xpack.security.put_user: username: "jacknich" body: > { "password" : "test-password", "roles" : [ "admin", "other_role1" ], "full_name" : "Jack Nicholson", "email" : "jacknich@example.com", "metadata" : { "intelligence" : 7 } } '''