import org.elasticsearch.gradle.test.NodeInfo import java.nio.charset.StandardCharsets apply plugin: 'elasticsearch.docs-test' /* List of files that have snippets that probably should be converted to * `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and * only remove entries from this list. When it is empty we'll remove it * entirely and have a party! There will be cake and everything.... */ buildRestTests.expectedUnconvertedCandidates = [ 'en/ml/getting-started.asciidoc', 'en/ml/functions/count.asciidoc', 'en/ml/functions/geo.asciidoc', 'en/ml/functions/info.asciidoc', 'en/ml/functions/metric.asciidoc', 'en/ml/functions/rare.asciidoc', 'en/ml/functions/sum.asciidoc', 'en/ml/functions/time.asciidoc', 'en/ml/aggregations.asciidoc', 'en/ml/customurl.asciidoc', 'en/rest-api/security/users.asciidoc', 'en/rest-api/security/tokens.asciidoc', 'en/rest-api/watcher/put-watch.asciidoc', 'en/rest-api/ml/post-data.asciidoc', 'en/security/authentication/user-cache.asciidoc', 'en/security/authorization/field-and-document-access-control.asciidoc', 'en/security/authorization/run-as-privilege.asciidoc', 'en/security/tribe-clients-integrations/beats.asciidoc', 'en/security/tribe-clients-integrations/http.asciidoc', 'en/security/tribe-clients-integrations/monitoring.asciidoc', 'en/security/authorization/custom-roles-provider.asciidoc', 'en/watcher/actions/email.asciidoc', 'en/watcher/actions/hipchat.asciidoc', 'en/watcher/actions/index.asciidoc', 'en/watcher/actions/logging.asciidoc', 'en/watcher/actions/pagerduty.asciidoc', 'en/watcher/actions/slack.asciidoc', 'en/watcher/actions/jira.asciidoc', 'en/watcher/actions/webhook.asciidoc', 'en/watcher/condition/always.asciidoc', 'en/watcher/condition/array-compare.asciidoc', 'en/watcher/condition/compare.asciidoc', 'en/watcher/condition/never.asciidoc', 'en/watcher/condition/script.asciidoc', 'en/watcher/customizing-watches.asciidoc', 'en/watcher/example-watches/example-watch-meetupdata.asciidoc', 'en/watcher/how-watcher-works.asciidoc', 'en/watcher/input/chain.asciidoc', 'en/watcher/input/http.asciidoc', 'en/watcher/input/search.asciidoc', 'en/watcher/input/simple.asciidoc', 'en/watcher/transform.asciidoc', 'en/watcher/transform/chain.asciidoc', 'en/watcher/transform/script.asciidoc', 'en/watcher/transform/search.asciidoc', 'en/watcher/trigger/schedule/cron.asciidoc', 'en/watcher/trigger/schedule/daily.asciidoc', 'en/watcher/trigger/schedule/hourly.asciidoc', 'en/watcher/trigger/schedule/interval.asciidoc', 'en/watcher/trigger/schedule/monthly.asciidoc', 'en/watcher/trigger/schedule/weekly.asciidoc', 'en/watcher/trigger/schedule/yearly.asciidoc', 'en/watcher/troubleshooting.asciidoc', 'en/ml/api-quickref.asciidoc', 'en/rest-api/ml/close-job.asciidoc', 'en/rest-api/ml/delete-datafeed.asciidoc', 'en/rest-api/ml/delete-snapshot.asciidoc', 'en/rest-api/ml/flush-job.asciidoc', 'en/rest-api/ml/forecast.asciidoc', 'en/rest-api/ml/get-bucket.asciidoc', 'en/rest-api/ml/get-overall-buckets.asciidoc', 'en/rest-api/ml/get-category.asciidoc', 'en/rest-api/ml/get-datafeed-stats.asciidoc', 'en/rest-api/ml/get-job-stats.asciidoc', 'en/rest-api/ml/get-record.asciidoc', 'en/rest-api/ml/preview-datafeed.asciidoc', 'en/rest-api/ml/update-datafeed.asciidoc', 'en/rest-api/ml/update-job.asciidoc', 'en/rest-api/ml/update-snapshot.asciidoc', 'en/rest-api/ml/validate-detector.asciidoc', 'en/rest-api/ml/delete-job.asciidoc', 'en/rest-api/ml/get-datafeed.asciidoc', 'en/rest-api/ml/get-influencer.asciidoc', 'en/rest-api/ml/get-job.asciidoc', 'en/rest-api/ml/get-snapshot.asciidoc', 'en/rest-api/ml/revert-snapshot.asciidoc', 'en/rest-api/ml/validate-job.asciidoc', 'en/rest-api/security/authenticate.asciidoc', 'en/rest-api/watcher/stats.asciidoc', 'en/security/authorization.asciidoc', 'en/watcher/example-watches/watching-time-series-data.asciidoc', ] dependencies { testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime') testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts') } Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> File tmpFile = new File(node.cwd, 'wait.success') // wait up to twenty seconds final long stopTime = System.currentTimeMillis() + 20000L; Exception lastException = null; while (System.currentTimeMillis() < stopTime) { lastException = null; // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned HttpURLConnection httpURLConnection = null; try { httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection(); httpURLConnection.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); httpURLConnection.setRequestMethod("GET"); httpURLConnection.setConnectTimeout(1000); httpURLConnection.setReadTimeout(30000); httpURLConnection.connect(); if (httpURLConnection.getResponseCode() == 200) { tmpFile.withWriter StandardCharsets.UTF_8.name(), { it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) } break; } } catch (Exception e) { logger.debug("failed to call cluster health", e) lastException = e } finally { if (httpURLConnection != null) { httpURLConnection.disconnect(); } } // did not start, so wait a bit before trying again Thread.sleep(500L); } if (tmpFile.exists() == false && lastException != null) { logger.error("final attempt of calling cluster health failed", lastException) } return tmpFile.exists() } integTestCluster { plugin ':x-pack-elasticsearch:plugin' setting 'xpack.security.authc.token.enabled', 'true' // Disable monitoring exporters for the docs tests setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setupCommand 'setupTestAdmin', 'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser' waitCondition = waitWithAuth } buildRestTests.docs = fileTree(projectDir) { // No snippets in here! exclude 'build.gradle' // That is where the snippets go, not where they come from! exclude 'build' // These file simply doesn't pass yet. We should figure out how to fix them. exclude 'en/watcher/reference/actions.asciidoc' exclude 'en/rest-api/graph/explore.asciidoc' } Map setups = buildRestTests.setups setups['my_inactive_watch'] = ''' - do: xpack.watcher.put_watch: id: "my_watch" active: false body: > { "trigger": { "schedule": { "hourly": { "minute": [ 0, 5 ] } } }, "input": { "simple": { "payload": { "send": "yes" } } }, "condition": { "always": {} }, "actions": { "test_index": { "index": { "index": "test", "doc_type": "test2" } } } } - match: { _id: "my_watch" } ''' setups['my_active_watch'] = setups['my_inactive_watch'].replace( 'active: false', 'active: true') // Used by SQL because it looks SQL-ish setups['library'] = ''' - do: indices.create: index: library body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: book: properties: name: type: keyword author: type: keyword release_date: type: date page_count: type: short - do: bulk: index: library type: book refresh: true body: | {"index":{"_id": "Leviathan Wakes"}} {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} {"index":{"_id": "Hyperion"}} {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} {"index":{"_id": "Dune"}} {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} {"index":{"_id": "Consider Phlebas"}} {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} {"index":{"_id": "Pandora's Star"}} {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} {"index":{"_id": "Revelation Space"}} {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} {"index":{"_id": "A Fire Upon the Deep"}} {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} {"index":{"_id": "Ender's Game"}} {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} {"index":{"_id": "1984"}} {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} {"index":{"_id": "Fahrenheit 451"}} {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} {"index":{"_id": "Brave New World"}} {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} {"index":{"_id": "Foundation"}} {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} {"index":{"_id": "The Giver"}} {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} {"index":{"_id": "Slaughterhouse-Five"}} {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}} {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} {"index":{"_id": "Snow Crash"}} {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} {"index":{"_id": "Neuromancer"}} {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} {"index":{"_id": "The Handmaid's Tale"}} {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} {"index":{"_id": "Starship Troopers"}} {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} {"index":{"_id": "The Left Hand of Darkness"}} {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} {"index":{"_id": "The Moon is a Harsh Mistress"}} {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ''' setups['server_metrics_index'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: timestamp: type: date total: type: long ''' setups['server_metrics_job'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: timestamp: type: date total: type: long - do: xpack.ml.put_job: job_id: "total-requests" body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } ''' setups['server_metrics_openjob'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: timestamp: type: date total: type: long - do: xpack.ml.put_job: job_id: "total-requests" body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } - do: xpack.ml.put_datafeed: datafeed_id: "datafeed-total-requests" body: > { "job_id":"total-requests", "indexes":"server-metrics", "types":"metric" } - do: xpack.ml.open_job: job_id: "total-requests" ''' setups['server_metrics_startdf'] = ''' - do: indices.create: index: server-metrics body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: metric: properties: timestamp: type: date total: type: long - do: xpack.ml.put_job: job_id: "total-requests" body: > { "description" : "Total sum of requests", "analysis_config" : { "bucket_span":"10m", "detectors" :[ { "detector_description": "Sum of total", "function": "sum", "field_name": "total" } ]}, "data_description" : { "time_field":"timestamp", "time_format": "epoch_ms" } } - do: xpack.ml.put_datafeed: datafeed_id: "datafeed-total-requests" body: > { "job_id":"total-requests", "indexes":"server-metrics", "types":"metric" } - do: xpack.ml.open_job: job_id: "total-requests" - do: xpack.ml.start_datafeed: datafeed_id: "datafeed-total-requests" '''