OpenSearch/docs/build.gradle

import org.elasticsearch.gradle.test.NodeInfo

import java.nio.charset.StandardCharsets

apply plugin: 'elasticsearch.docs-test'

/* List of files that have snippets that probably should be converted to
 * `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and
 * only remove entries from this list. When it is empty we'll remove it
 * entirely and have a party! There will be cake and everything.... */
buildRestTests.expectedUnconvertedCandidates = [
        'en/ml/functions/count.asciidoc',
        'en/ml/functions/geo.asciidoc',
        'en/ml/functions/info.asciidoc',
        'en/ml/functions/metric.asciidoc',
        'en/ml/functions/rare.asciidoc',
        'en/ml/functions/sum.asciidoc',
        'en/ml/functions/time.asciidoc',
        'en/ml/aggregations.asciidoc',
        'en/ml/customurl.asciidoc',
        'en/monitoring/indices.asciidoc',
        'en/rest-api/security/ssl.asciidoc',
        'en/rest-api/security/users.asciidoc',
        'en/rest-api/security/tokens.asciidoc',
        'en/rest-api/watcher/put-watch.asciidoc',
        'en/security/authentication/user-cache.asciidoc',
        'en/security/authorization/field-and-document-access-control.asciidoc',
        'en/security/authorization/run-as-privilege.asciidoc',
        'en/security/tribe-clients-integrations/http.asciidoc',
        'en/security/tribe-clients-integrations/monitoring.asciidoc',
        'en/security/authorization/custom-roles-provider.asciidoc',
        'en/watcher/actions/email.asciidoc',
        'en/watcher/actions/hipchat.asciidoc',
        'en/watcher/actions/index.asciidoc',
        'en/watcher/actions/logging.asciidoc',
        'en/watcher/actions/pagerduty.asciidoc',
        'en/watcher/actions/slack.asciidoc',
        'en/watcher/actions/jira.asciidoc',
        'en/watcher/actions/webhook.asciidoc',
        'en/watcher/condition/always.asciidoc',
        'en/watcher/condition/array-compare.asciidoc',
        'en/watcher/condition/compare.asciidoc',
        'en/watcher/condition/never.asciidoc',
        'en/watcher/condition/script.asciidoc',
        'en/watcher/customizing-watches.asciidoc',
        'en/watcher/example-watches/example-watch-meetupdata.asciidoc',
        'en/watcher/how-watcher-works.asciidoc',
        'en/watcher/input/chain.asciidoc',
        'en/watcher/input/http.asciidoc',
        'en/watcher/input/search.asciidoc',
        'en/watcher/input/simple.asciidoc',
        'en/watcher/transform.asciidoc',
        'en/watcher/transform/chain.asciidoc',
        'en/watcher/transform/script.asciidoc',
        'en/watcher/transform/search.asciidoc',
        'en/watcher/trigger/schedule/cron.asciidoc',
        'en/watcher/trigger/schedule/daily.asciidoc',
        'en/watcher/trigger/schedule/hourly.asciidoc',
        'en/watcher/trigger/schedule/interval.asciidoc',
        'en/watcher/trigger/schedule/monthly.asciidoc',
        'en/watcher/trigger/schedule/weekly.asciidoc',
        'en/watcher/trigger/schedule/yearly.asciidoc',
        'en/watcher/troubleshooting.asciidoc',
        'en/ml/api-quickref.asciidoc',
        'en/rest-api/ml/delete-calendar-event.asciidoc',
        'en/rest-api/ml/delete-snapshot.asciidoc',
        'en/rest-api/ml/forecast.asciidoc',
        'en/rest-api/ml/get-bucket.asciidoc',
        'en/rest-api/ml/get-job-stats.asciidoc',
        'en/rest-api/ml/get-overall-buckets.asciidoc',
        'en/rest-api/ml/get-category.asciidoc',
        'en/rest-api/ml/get-record.asciidoc',
        'en/rest-api/ml/get-influencer.asciidoc',
        'en/rest-api/ml/get-snapshot.asciidoc',
        'en/rest-api/ml/post-data.asciidoc',        
        'en/rest-api/ml/preview-datafeed.asciidoc',
        'en/rest-api/ml/revert-snapshot.asciidoc',
        'en/rest-api/ml/update-snapshot.asciidoc',
        'en/rest-api/ml/validate-detector.asciidoc',
        'en/rest-api/ml/validate-job.asciidoc',
        'en/rest-api/security/authenticate.asciidoc',
        'en/rest-api/watcher/stats.asciidoc',
        'en/security/authorization.asciidoc',
        'en/watcher/example-watches/watching-time-series-data.asciidoc',
]

dependencies {
    testCompile project(path: xpackModule('core'), configuration: 'runtime')
    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
    testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}

Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
    File tmpFile = new File(node.cwd, 'wait.success')
    // wait up to twenty seconds
    final long stopTime = System.currentTimeMillis() + 20000L;
    Exception lastException = null;
    while (System.currentTimeMillis() < stopTime) {
        lastException = null;
        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
        HttpURLConnection httpURLConnection = null;
        try {
            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection();
            httpURLConnection.setRequestProperty("Authorization", "Basic " +
                    Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
            httpURLConnection.setRequestMethod("GET");
            httpURLConnection.setConnectTimeout(1000);
            httpURLConnection.setReadTimeout(30000);
            httpURLConnection.connect();
            if (httpURLConnection.getResponseCode() == 200) {
                tmpFile.withWriter StandardCharsets.UTF_8.name(), {
                    it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
                }
                break;
            }
        } catch (Exception e) {
            logger.debug("failed to call cluster health", e)
            lastException = e
        } finally {
            if (httpURLConnection != null) {
                httpURLConnection.disconnect();
            }
        }

        // did not start, so wait a bit before trying again
        Thread.sleep(500L);
    }
    if (tmpFile.exists() == false && lastException != null) {
        logger.error("final attempt of calling cluster health failed", lastException)
    }
    return tmpFile.exists()
}

integTestCluster {
    plugin xpackProject('plugin').path
    setting 'xpack.security.authc.token.enabled', 'true'
    // Disable monitoring exporters for the docs tests
    setting 'xpack.monitoring.exporters._local.type', 'local'
    setting 'xpack.monitoring.exporters._local.enabled', 'false'
    setting 'xpack.license.self_generated.type', 'trial'
    setupCommand 'setupTestAdmin',
            'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
    waitCondition = waitWithAuth
}



buildRestTests.docs = fileTree(projectDir) {
    // No snippets in here!
    exclude 'build.gradle'
    // That is where the snippets go, not where they come from!
    exclude 'build'
    // These file simply doesn't pass yet. We should figure out how to fix them.
    exclude 'en/watcher/reference/actions.asciidoc'
    exclude 'en/rest-api/graph/explore.asciidoc'
}

Map<String, String> setups = buildRestTests.setups
setups['my_inactive_watch'] = '''
  - do:
      xpack.watcher.put_watch:
        id: "my_watch"
        active: false
        body:  >
          {
            "trigger": {
              "schedule": {
                "hourly": {
                  "minute": [ 0, 5 ]
                }
              }
            },
            "input": {
              "simple": {
                "payload": {
                  "send": "yes"
                }
              }
            },
            "condition": {
              "always": {}
            },
            "actions": {
              "test_index": {
                "index": {
                  "index": "test",
                  "doc_type": "test2"
                }
              }
            }
          }
  - match: { _id: "my_watch" }
'''
setups['my_active_watch'] = setups['my_inactive_watch'].replace(
        'active: false', 'active: true')

// Used by SQL because it looks SQL-ish
setups['library'] = '''
  - do:
        indices.create:
          index: library
          body:
            settings:
              number_of_shards: 1
              number_of_replicas: 1
            mappings:
              book:
                properties:
                  name:
                    type: text
                    fields:
                      keyword:
                        type: keyword
                  author:
                    type: text
                    fields:
                      keyword:
                        type: keyword
                  release_date:
                    type: date
                  page_count:
                    type: short
  - do:
        bulk:
          index: library
          type: book
          refresh: true
          body: |
            {"index":{"_id": "Leviathan Wakes"}}
            {"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
            {"index":{"_id": "Hyperion"}}
            {"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
            {"index":{"_id": "Dune"}}
            {"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
            {"index":{"_id": "Dune Messiah"}}
            {"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331}
            {"index":{"_id": "Children of Dune"}}
            {"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408}
            {"index":{"_id": "God Emperor of Dune"}}
            {"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454}
            {"index":{"_id": "Consider Phlebas"}}
            {"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471}
            {"index":{"_id": "Pandora's Star"}}
            {"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768}
            {"index":{"_id": "Revelation Space"}}
            {"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585}
            {"index":{"_id": "A Fire Upon the Deep"}}
            {"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613}
            {"index":{"_id": "Ender's Game"}}
            {"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324}
            {"index":{"_id": "1984"}}
            {"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328}
            {"index":{"_id": "Fahrenheit 451"}}
            {"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227}
            {"index":{"_id": "Brave New World"}}
            {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268}
            {"index":{"_id": "Foundation"}}
            {"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224}
            {"index":{"_id": "The Giver"}}
            {"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208}
            {"index":{"_id": "Slaughterhouse-Five"}}
            {"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275}
            {"index":{"_id": "The Hitchhiker's Guide to the Galaxy"}}
            {"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180}
            {"index":{"_id": "Snow Crash"}}
            {"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470}
            {"index":{"_id": "Neuromancer"}}
            {"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271}
            {"index":{"_id": "The Handmaid's Tale"}}
            {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}
            {"index":{"_id": "Starship Troopers"}}
            {"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335}
            {"index":{"_id": "The Left Hand of Darkness"}}
            {"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304}
            {"index":{"_id": "The Moon is a Harsh Mistress"}}
            {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288}

'''
setups['server_metrics_index'] = '''
  - do:
        indices.create:
          index: server-metrics
          body:
            settings:
              number_of_shards: 1
              number_of_replicas: 0
            mappings:
              metric:
                properties:
                  timestamp:
                    type: date
                  total:
                    type: long
'''
setups['server_metrics_data'] = setups['server_metrics_index'] + '''
  - do:
        bulk:
          index: server-metrics
          type: metric
          refresh: true
          body: |
            {"index": {"_id":"1177"}}
            {"timestamp":"2017-03-23T13:00:00","total":40476}
            {"index": {"_id":"1178"}}
            {"timestamp":"2017-03-23T13:00:00","total":15287}
            {"index": {"_id":"1179"}}
            {"timestamp":"2017-03-23T13:00:00","total":-776}
            {"index": {"_id":"1180"}}
            {"timestamp":"2017-03-23T13:00:00","total":11366}
            {"index": {"_id":"1181"}}
            {"timestamp":"2017-03-23T13:00:00","total":3606}
            {"index": {"_id":"1182"}}
            {"timestamp":"2017-03-23T13:00:00","total":19006}
            {"index": {"_id":"1183"}}
            {"timestamp":"2017-03-23T13:00:00","total":38613}
            {"index": {"_id":"1184"}}
            {"timestamp":"2017-03-23T13:00:00","total":19516}
            {"index": {"_id":"1185"}}
            {"timestamp":"2017-03-23T13:00:00","total":-258}
            {"index": {"_id":"1186"}}
            {"timestamp":"2017-03-23T13:00:00","total":9551}
            {"index": {"_id":"1187"}}
            {"timestamp":"2017-03-23T13:00:00","total":11217}
            {"index": {"_id":"1188"}}
            {"timestamp":"2017-03-23T13:00:00","total":22557}
            {"index": {"_id":"1189"}}
            {"timestamp":"2017-03-23T13:00:00","total":40508}
            {"index": {"_id":"1190"}}
            {"timestamp":"2017-03-23T13:00:00","total":11887}
            {"index": {"_id":"1191"}}
            {"timestamp":"2017-03-23T13:00:00","total":31659}
'''
setups['server_metrics_job'] = setups['server_metrics_data'] + '''
  - do:
      xpack.ml.put_job:
        job_id: "total-requests"
        body:  >
          {
            "description" : "Total sum of requests",
            "analysis_config" : {
              "bucket_span":"10m",
              "detectors" :[
              {
                "detector_description": "Sum of total",
                "function": "sum",
                "field_name": "total"
              }
            ]},
            "data_description" : {
              "time_field":"timestamp",
              "time_format": "epoch_ms"
            }
          }
'''
setups['server_metrics_datafeed'] = setups['server_metrics_job'] + '''
  - do:
      xpack.ml.put_datafeed:
        datafeed_id: "datafeed-total-requests"
        body:  >
          {
          "job_id":"total-requests",
          "indexes":"server-metrics",
          "types":"metric"
          }
'''
setups['server_metrics_openjob'] = setups['server_metrics_datafeed'] + '''
  - do:
      xpack.ml.open_job:
        job_id: "total-requests"
'''
setups['server_metrics_startdf'] = setups['server_metrics_openjob'] + '''
  - do:
      xpack.ml.start_datafeed:
        datafeed_id: "datafeed-total-requests"
'''
setups['calendar_outages'] = '''
  - do:
        xpack.ml.put_calendar:
          calendar_id: "planned-outages"
'''
setups['calendar_outages_addevent'] = setups['calendar_outages'] + '''
  - do:
        xpack.ml.post_calendar_events:
          calendar_id: "planned-outages"
          body: >
            { "description": "event 1", "start_time": "2017-12-01T00:00:00Z", "end_time": "2017-12-02T00:00:00Z", "calendar_id": "planned-outages" }


'''
setups['calendar_outages_openjob'] = setups['server_metrics_openjob'] + '''
  - do:
       xpack.ml.put_calendar:
         calendar_id: "planned-outages"
'''
setups['calendar_outages_addjob'] = setups['server_metrics_openjob'] + '''
  - do:
       xpack.ml.put_calendar:
         calendar_id: "planned-outages"
         body:  >
           {
           "job_ids": ["total-requests"]
           }
'''
setups['calendar_outages_addevent'] = setups['calendar_outages_addjob'] + '''
  - do:
       xpack.ml.post_calendar_events:
         calendar_id: "planned-outages"
         body:  >
           { "events" : [
           { "description": "event 1", "start_time": "1513641600000", "end_time": "1513728000000"},
           { "description": "event 2", "start_time": "1513814400000", "end_time": "1513900800000"},
           { "description": "event 3", "start_time": "1514160000000", "end_time": "1514246400000"}
           ]}
'''
setups['role_mapping'] = '''
  - do:
        xpack.security.put_role_mapping:
         name: "mapping1"
         body:  >
            {
               "enabled": true,
               "roles": [ "user" ],
               "rules": { "field": { "username": "*" } }
            }
'''