mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-31 04:18:39 +00:00
175bda64a0
Gradle's finalizedBy on tasks only ensures one task runs after another, but not immediately after. This is problematic for our integration tests since it allows multiple project's integ test clusters to be simultaneously. While this has not been a problem thus far (gradle 2.13 happened to keep the finalizedBy tasks close enough that no clusters were running in parallel), with gradle 3.3 the task graph generation has changed, and numerous clusters may be running simultaneously, causing memory pressure, and thus generally slower tests, or even failure if the system has a limited amount of memory (eg in a vagrant host). This commit reworks how integ tests are configured. It adds an `integTestCluster` extension to gradle which is equivalent to the current `integTest.cluster` and moves the rest test runner task to `integTestRunner`. The `integTest` task is then just a dummy task, which depends on the cluster runner task, as well as the cluster stop task. This means running `integTest` in one project will both run the rest tests, and shut down the cluster, before running `integTest` in another project.
440 lines
16 KiB
Groovy
440 lines
16 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
apply plugin: 'elasticsearch.docs-test'
|
|
|
|
/* List of files that have snippets that probably should be converted to
|
|
* `// CONSOLE` and `// TESTRESPONSE` but have yet to be converted. Try and
|
|
* only remove entries from this list. When it is empty we'll remove it
|
|
* entirely and have a party! There will be cake and everything.... */
|
|
buildRestTests.expectedUnconvertedCandidates = [
|
|
'reference/aggregations/bucket/geodistance-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/geohashgrid-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/iprange-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/missing-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/nested-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/range-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/reverse-nested-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/significantterms-aggregation.asciidoc',
|
|
'reference/aggregations/bucket/terms-aggregation.asciidoc',
|
|
'reference/aggregations/matrix/stats-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/cardinality-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/extendedstats-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/geobounds-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/geocentroid-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/percentile-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/percentile-rank-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/scripted-metric-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/stats-aggregation.asciidoc',
|
|
'reference/aggregations/metrics/tophits-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline.asciidoc',
|
|
'reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/bucket-script-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/cumulative-sum-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/derivative-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/extended-stats-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/max-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/min-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/movavg-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/serial-diff-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/stats-bucket-aggregation.asciidoc',
|
|
'reference/aggregations/pipeline/sum-bucket-aggregation.asciidoc',
|
|
'reference/analysis/analyzers/lang-analyzer.asciidoc',
|
|
'reference/analysis/analyzers/pattern-analyzer.asciidoc',
|
|
'reference/analysis/charfilters/htmlstrip-charfilter.asciidoc',
|
|
'reference/analysis/charfilters/pattern-replace-charfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/asciifolding-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/elision-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/keep-words-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/keyword-marker-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/keyword-repeat-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/limit-token-count-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/lowercase-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/stop-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc',
|
|
'reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc',
|
|
'reference/cat/snapshots.asciidoc',
|
|
'reference/cat/templates.asciidoc',
|
|
'reference/cat/thread_pool.asciidoc',
|
|
'reference/cluster/allocation-explain.asciidoc',
|
|
'reference/cluster/nodes-info.asciidoc',
|
|
'reference/cluster/nodes-stats.asciidoc',
|
|
'reference/cluster/pending.asciidoc',
|
|
'reference/cluster/state.asciidoc',
|
|
'reference/cluster/stats.asciidoc',
|
|
'reference/cluster/tasks.asciidoc',
|
|
'reference/docs/delete-by-query.asciidoc',
|
|
'reference/docs/delete.asciidoc',
|
|
'reference/docs/index_.asciidoc',
|
|
'reference/docs/reindex.asciidoc',
|
|
'reference/docs/update-by-query.asciidoc',
|
|
'reference/docs/update.asciidoc',
|
|
'reference/index-modules/similarity.asciidoc',
|
|
'reference/index-modules/store.asciidoc',
|
|
'reference/index-modules/translog.asciidoc',
|
|
'reference/indices/flush.asciidoc',
|
|
'reference/indices/get-settings.asciidoc',
|
|
'reference/indices/put-mapping.asciidoc',
|
|
'reference/indices/recovery.asciidoc',
|
|
'reference/indices/segments.asciidoc',
|
|
'reference/indices/shard-stores.asciidoc',
|
|
'reference/ingest/ingest-node.asciidoc',
|
|
'reference/mapping/dynamic/templates.asciidoc',
|
|
'reference/mapping/fields/all-field.asciidoc',
|
|
'reference/mapping/params/analyzer.asciidoc',
|
|
'reference/mapping/types/binary.asciidoc',
|
|
'reference/mapping/types/geo-point.asciidoc',
|
|
'reference/mapping/types/geo-shape.asciidoc',
|
|
'reference/mapping/types/ip.asciidoc',
|
|
'reference/mapping/types/nested.asciidoc',
|
|
'reference/mapping/types/object.asciidoc',
|
|
'reference/mapping/types/percolator.asciidoc',
|
|
'reference/modules/scripting/security.asciidoc',
|
|
'reference/modules/scripting/using.asciidoc',
|
|
'reference/modules/cross-cluster-search.asciidoc', // this is hard to test since we need 2 clusters -- maybe we can trick it into referencing itself...
|
|
'reference/query-dsl/exists-query.asciidoc',
|
|
'reference/query-dsl/function-score-query.asciidoc',
|
|
'reference/query-dsl/geo-shape-query.asciidoc',
|
|
'reference/search/field-stats.asciidoc',
|
|
'reference/search/profile.asciidoc',
|
|
'reference/search/request/highlighting.asciidoc',
|
|
'reference/search/request/inner-hits.asciidoc',
|
|
]
|
|
|
|
integTestCluster {
|
|
setting 'script.inline', 'true'
|
|
setting 'script.stored', 'true'
|
|
setting 'script.max_compilations_per_minute', '1000'
|
|
/* Enable regexes in painless so our tests don't complain about example
|
|
* snippets that use them. */
|
|
setting 'script.painless.regex.enabled', 'true'
|
|
Closure configFile = {
|
|
extraConfigFile it, "src/test/cluster/config/$it"
|
|
}
|
|
configFile 'scripts/my_script.painless'
|
|
configFile 'scripts/my_init_script.painless'
|
|
configFile 'scripts/my_map_script.painless'
|
|
configFile 'scripts/my_combine_script.painless'
|
|
configFile 'scripts/my_reduce_script.painless'
|
|
configFile 'userdict_ja.txt'
|
|
configFile 'KeywordTokenizer.rbbi'
|
|
// Whitelist reindexing from the local node so we can test it.
|
|
setting 'reindex.remote.whitelist', '127.0.0.1:*'
|
|
}
|
|
|
|
// Build the cluster with all plugins
|
|
|
|
project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
|
|
/* Skip repositories. We just aren't going to be able to test them so it
|
|
* doesn't make sense to waste time installing them. */
|
|
if (subproj.path.startsWith(':plugins:repository-')) {
|
|
return
|
|
}
|
|
subproj.afterEvaluate { // need to wait until the project has been configured
|
|
integTestCluster {
|
|
plugin subproj.path
|
|
}
|
|
}
|
|
}
|
|
|
|
buildRestTests.docs = fileTree(projectDir) {
|
|
// No snippets in here!
|
|
exclude 'build.gradle'
|
|
// That is where the snippets go, not where they come from!
|
|
exclude 'build'
|
|
}
|
|
|
|
Closure setupTwitter = { String name, int count ->
|
|
buildRestTests.setups[name] = '''
|
|
- do:
|
|
indices.create:
|
|
index: twitter
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 1
|
|
mappings:
|
|
tweet:
|
|
properties:
|
|
user:
|
|
type: keyword
|
|
doc_values: true
|
|
date:
|
|
type: date
|
|
likes:
|
|
type: long
|
|
- do:
|
|
bulk:
|
|
index: twitter
|
|
type: tweet
|
|
refresh: true
|
|
body: |'''
|
|
for (int i = 0; i < count; i++) {
|
|
String user, text
|
|
if (i == 0) {
|
|
user = 'kimchy'
|
|
text = 'trying out Elasticsearch'
|
|
} else {
|
|
user = 'test'
|
|
text = "some message with the number $i"
|
|
}
|
|
buildRestTests.setups[name] += """
|
|
{"index":{"_id": "$i"}}
|
|
{"user": "$user", "message": "$text", "date": "2009-11-15T14:12:12", "likes": $i}"""
|
|
}
|
|
}
|
|
setupTwitter('twitter', 5)
|
|
setupTwitter('big_twitter', 120)
|
|
setupTwitter('huge_twitter', 1200)
|
|
|
|
buildRestTests.setups['host'] = '''
|
|
# Fetch the http host. We use the host of the master because we know there will always be a master.
|
|
- do:
|
|
cluster.state: {}
|
|
- set: { master_node: master }
|
|
- do:
|
|
nodes.info:
|
|
metric: [ http ]
|
|
- is_true: nodes.$master.http.publish_address
|
|
- set: {nodes.$master.http.publish_address: host}
|
|
'''
|
|
|
|
// Used by scripted metric docs
|
|
buildRestTests.setups['ledger'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: ledger
|
|
body:
|
|
settings:
|
|
number_of_shards: 2
|
|
number_of_replicas: 1
|
|
mappings:
|
|
sale:
|
|
properties:
|
|
type:
|
|
type: keyword
|
|
amount:
|
|
type: double
|
|
- do:
|
|
bulk:
|
|
index: ledger
|
|
type: item
|
|
refresh: true
|
|
body: |
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "amount": 200, "type": "sale", "description": "something"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "amount": 10, "type": "expense", "decription": "another thing"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "amount": 150, "type": "sale", "description": "blah"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "cost of blah"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "amount": 50, "type": "expense", "description": "advertisement"}'''
|
|
|
|
// Used by aggregation docs
|
|
buildRestTests.setups['sales'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: sales
|
|
body:
|
|
settings:
|
|
number_of_shards: 2
|
|
number_of_replicas: 1
|
|
mappings:
|
|
sale:
|
|
properties:
|
|
type:
|
|
type: keyword
|
|
- do:
|
|
bulk:
|
|
index: sales
|
|
type: sale
|
|
refresh: true
|
|
body: |
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "t-shirt"}
|
|
{"index":{}}
|
|
{"date": "2015/01/01 00:00:00", "price": 150, "promoted": true, "rating": 5, "type": "bag"}
|
|
{"index":{}}
|
|
{"date": "2015/02/01 00:00:00", "price": 50, "promoted": false, "rating": 1, "type": "hat"}
|
|
{"index":{}}
|
|
{"date": "2015/02/01 00:00:00", "price": 10, "promoted": true, "rating": 4, "type": "t-shirt"}
|
|
{"index":{}}
|
|
{"date": "2015/03/01 00:00:00", "price": 200, "promoted": true, "rating": 1, "type": "hat"}
|
|
{"index":{}}
|
|
{"date": "2015/03/01 00:00:00", "price": 175, "promoted": false, "rating": 2, "type": "t-shirt"}'''
|
|
|
|
// Dummy bank account data used by getting-started.asciidoc
|
|
buildRestTests.setups['bank'] = '''
|
|
- do:
|
|
bulk:
|
|
index: bank
|
|
type: account
|
|
refresh: true
|
|
body: |
|
|
#bank_data#
|
|
'''
|
|
/* Load the actual accounts only if we're going to use them. This complicates
|
|
* dependency checking but that is a small price to pay for not building a
|
|
* 400kb string every time we start the build. */
|
|
File accountsFile = new File("$projectDir/src/test/resources/accounts.json")
|
|
buildRestTests.inputs.file(accountsFile)
|
|
buildRestTests.doFirst {
|
|
String accounts = accountsFile.getText('UTF-8')
|
|
// Indent like a yaml test needs
|
|
accounts = accounts.replaceAll('(?m)^', ' ')
|
|
buildRestTests.setups['bank'] =
|
|
buildRestTests.setups['bank'].replace('#bank_data#', accounts)
|
|
}
|
|
|
|
buildRestTests.setups['range_index'] = '''
|
|
- do :
|
|
indices.create:
|
|
index: range_index
|
|
body:
|
|
settings:
|
|
number_of_shards: 2
|
|
number_of_replicas: 1
|
|
mappings:
|
|
my_type:
|
|
properties:
|
|
expected_attendees:
|
|
type: integer_range
|
|
time_frame:
|
|
type: date_range
|
|
format: yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis
|
|
- do:
|
|
bulk:
|
|
index: range_index
|
|
type: my_type
|
|
refresh: true
|
|
body: |
|
|
{"index":{"_id": 1}}
|
|
{"expected_attendees": {"gte": 10, "lte": 20}, "time_frame": {"gte": "2015-10-31 12:00:00", "lte": "2015-11-01"}}'''
|
|
|
|
// Used by index boost doc
|
|
buildRestTests.setups['index_boost'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: index1
|
|
- do:
|
|
indices.create:
|
|
index: index2
|
|
|
|
- do:
|
|
indices.put_alias:
|
|
index: index1
|
|
name: alias1
|
|
'''
|
|
// Used by sampler and diversified-sampler aggregation docs
|
|
buildRestTests.setups['stackoverflow'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: stackoverflow
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 1
|
|
mappings:
|
|
question:
|
|
properties:
|
|
author:
|
|
type: keyword
|
|
tags:
|
|
type: keyword
|
|
- do:
|
|
bulk:
|
|
index: stackoverflow
|
|
type: question
|
|
refresh: true
|
|
body: |'''
|
|
|
|
// Make Kibana strongly connected to elasticsearch and logstash
|
|
// Make Kibana rarer (and therefore higher-ranking) than Javascript
|
|
// Make Javascript strongly connected to jquery and angular
|
|
// Make Cabana strongly connected to elasticsearch but only as a result of a single author
|
|
|
|
for (int i = 0; i < 150; i++) {
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
{"index":{}}
|
|
{"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}"""
|
|
}
|
|
for (int i = 0; i < 50; i++) {
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
{"index":{}}
|
|
{"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}"""
|
|
}
|
|
for (int i = 0; i < 200; i++) {
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
{"index":{}}
|
|
{"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}"""
|
|
}
|
|
for (int i = 0; i < 200; i++) {
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
{"index":{}}
|
|
{"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}"""
|
|
}
|
|
for (int i = 0; i < 50; i++) {
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
{"index":{}}
|
|
{"author": "noisy author", "tags": ["elasticsearch", "cabana"]}"""
|
|
}
|
|
buildRestTests.setups['stackoverflow'] += """
|
|
"""
|
|
|
|
// Used by some aggregations
|
|
buildRestTests.setups['exams'] = '''
|
|
- do:
|
|
indices.create:
|
|
index: exams
|
|
body:
|
|
settings:
|
|
number_of_shards: 1
|
|
number_of_replicas: 1
|
|
mappings:
|
|
exam:
|
|
properties:
|
|
grade:
|
|
type: byte
|
|
- do:
|
|
bulk:
|
|
index: exams
|
|
type: exam
|
|
refresh: true
|
|
body: |
|
|
{"index":{}}
|
|
{"grade": 100}
|
|
{"index":{}}
|
|
{"grade": 50}'''
|