OpenSearch/settings.gradle

157 lines
5.1 KiB
Groovy
Raw Normal View History

String dirName = rootProject.projectDir.name
rootProject.name = dirName
List projects = [
'build-tools',
'rest-api-spec',
'core',
'docs',
'client:rest',
'client:rest-high-level',
'client:sniffer',
'client:transport',
'client:test',
'client:client-benchmark-noop-api-plugin',
'client:benchmark',
'benchmarks',
2015-12-03 17:52:51 -05:00
'distribution:integ-test-zip',
'distribution:bwc-release-snapshot',
'distribution:bwc-stable-snapshot',
'distribution:zip',
'distribution:tar',
'distribution:deb',
'distribution:rpm',
'distribution:tools:java-version-checker',
'distribution:tools:plugin-cli',
'test:framework',
'test:fixtures:example-fixture',
'test:fixtures:hdfs-fixture',
'test:fixtures:krb5kdc-fixture',
'test:fixtures:old-elasticsearch',
'test:logger-usage',
'modules:aggs-matrix-stats',
Add composite aggregator (#26800) * This change adds a module called `aggs-composite` that defines a new aggregation named `composite`. The `composite` aggregation is a multi-buckets aggregation that creates composite buckets made of multiple sources. The sources for each bucket can be defined as: * A `terms` source, values are extracted from a field or a script. * A `date_histogram` source, values are extracted from a date field and rounded to the provided interval. This aggregation can be used to retrieve all buckets of a deeply nested aggregation by flattening the nested aggregation in composite buckets. A composite buckets is composed of one value per source and is built for each document as the combinations of values in the provided sources. For instance the following aggregation: ```` "test_agg": { "terms": { "field": "field1" }, "aggs": { "nested_test_agg": "terms": { "field": "field2" } } } ```` ... which retrieves the top N terms for `field1` and for each top term in `field1` the top N terms for `field2`, can be replaced by a `composite` aggregation in order to retrieve **all** the combinations of `field1`, `field2` in the matching documents: ```` "composite_agg": { "composite": { "sources": [ { "field1": { "terms": { "field": "field1" } } }, { "field2": { "terms": { "field": "field2" } } }, } } ```` The response of the aggregation looks like this: ```` "aggregations": { "composite_agg": { "buckets": [ { "key": { "field1": "alabama", "field2": "almanach" }, "doc_count": 100 }, { "key": { "field1": "alabama", "field2": "calendar" }, "doc_count": 1 }, { "key": { "field1": "arizona", "field2": "calendar" }, "doc_count": 1 } ] } } ```` By default this aggregation returns 10 buckets sorted in ascending order of the composite key. Pagination can be achieved by providing `after` values, the values of the composite key to aggregate after. For instance the following aggregation will aggregate all composite keys that sorts after `arizona, calendar`: ```` "composite_agg": { "composite": { "after": {"field1": "alabama", "field2": "calendar"}, "size": 100, "sources": [ { "field1": { "terms": { "field": "field1" } } }, { "field2": { "terms": { "field": "field2" } } } } } ```` This aggregation is optimized for indices that set an index sorting that match the composite source definition. For instance the aggregation above could run faster on indices that defines an index sorting like this: ```` "settings": { "index.sort.field": ["field1", "field2"] } ```` In this case the `composite` aggregation can early terminate on each segment. This aggregation also accepts multi-valued field but disables early termination for these fields even if index sorting matches the sources definition. This is mandatory because index sorting picks only one value per document to perform the sort.
2017-11-16 09:13:36 -05:00
'modules:aggs-composite',
'modules:analysis-common',
'modules:ingest-common',
'modules:lang-expression',
'modules:lang-mustache',
'modules:lang-painless',
'modules:mapper-extras',
'modules:parent-join',
'modules:percolator',
'modules:reindex',
'modules:repository-url',
'modules:transport-netty4',
'modules:tribe',
'plugins:analysis-icu',
'plugins:analysis-kuromoji',
'plugins:analysis-phonetic',
'plugins:analysis-smartcn',
'plugins:analysis-stempel',
'plugins:analysis-ukrainian',
'plugins:discovery-azure-classic',
'plugins:discovery-ec2',
'plugins:discovery-file',
'plugins:discovery-gce',
'plugins:ingest-geoip',
'plugins:ingest-attachment',
'plugins:ingest-user-agent',
'plugins:mapper-murmur3',
'plugins:mapper-size',
'plugins:repository-azure',
'plugins:repository-gcs',
'plugins:repository-hdfs',
'plugins:repository-s3',
'plugins:jvm-example',
'plugins:store-smb',
'qa:auto-create-index',
'qa:evil-tests',
'qa:full-cluster-restart',
'qa:integration-bwc',
'qa:mixed-cluster',
'qa:multi-cluster-search',
'qa:no-bootstrap-tests',
'qa:reindex-from-old',
2016-08-31 19:45:03 -04:00
'qa:rolling-upgrade',
'qa:smoke-test-client',
'qa:smoke-test-http',
'qa:smoke-test-ingest-with-all-dependencies',
'qa:smoke-test-ingest-disabled',
'qa:smoke-test-multinode',
'qa:smoke-test-plugins',
'qa:smoke-test-reindex-with-all-modules',
'qa:smoke-test-tribe-node',
[test] Gradle-ify vagrant tests This gets the tar and tar_plugins tests working in gradle. It does so by adding a subproject, qa/vagrant, which adds the following tasks: Verification ------------ checkPackages - Check the packages against a representative sample of the linux distributions we have in our Vagrantfile checkPackagesAllDistros - Check the packages against all the linux distributions we have in our Vagrantfile Package Verification -------------------- checkCentos6 - Run packaging tests against centos-6 checkCentos7 - Run packaging tests against centos-7 checkDebian8 - Run packaging tests against debian-8 checkFedora22 - Run packaging tests against fedora-22 checkOel7 - Run packaging tests against oel-7 checkOpensuse13 - Run packaging tests against opensuse-13 checkSles12 - Run packaging tests against sles-12 checkUbuntu1204 - Run packaging tests against ubuntu-1204 checkUbuntu1404 - Run packaging tests against ubuntu-1404 checkUbuntu1504 - Run packaging tests against ubuntu-1504 Vagrant ------- smokeTestCentos6 - Smoke test the centos-6 VM smokeTestCentos7 - Smoke test the centos-7 VM smokeTestDebian8 - Smoke test the debian-8 VM smokeTestFedora22 - Smoke test the fedora-22 VM smokeTestOel7 - Smoke test the oel-7 VM smokeTestOpensuse13 - Smoke test the opensuse-13 VM smokeTestSles12 - Smoke test the sles-12 VM smokeTestUbuntu1204 - Smoke test the ubuntu-1204 VM smokeTestUbuntu1404 - Smoke test the ubuntu-1404 VM smokeTestUbuntu1504 - Smoke test the ubuntu-1504 VM vagrantHaltCentos6 - Shutdown the vagrant VM running centos-6 vagrantHaltCentos7 - Shutdown the vagrant VM running centos-7 vagrantHaltDebian8 - Shutdown the vagrant VM running debian-8 vagrantHaltFedora22 - Shutdown the vagrant VM running fedora-22 vagrantHaltOel7 - Shutdown the vagrant VM running oel-7 vagrantHaltOpensuse13 - Shutdown the vagrant VM running opensuse-13 vagrantHaltSles12 - Shutdown the vagrant VM running sles-12 vagrantHaltUbuntu1204 - Shutdown the vagrant VM running ubuntu-1204 vagrantHaltUbuntu1404 - Shutdown the vagrant VM running ubuntu-1404 vagrantHaltUbuntu1504 - Shutdown the vagrant VM running ubuntu-1504 vagrantSmokeTest - Smoke test some representative distros from the Vagrantfile vagrantSmokeTestAllDistros - Smoke test all distros from the Vagrantfile vagrantUpCentos6 - Startup a vagrant VM running centos-6 vagrantUpCentos7 - Startup a vagrant VM running centos-7 vagrantUpDebian8 - Startup a vagrant VM running debian-8 vagrantUpFedora22 - Startup a vagrant VM running fedora-22 vagrantUpOel7 - Startup a vagrant VM running oel-7 vagrantUpOpensuse13 - Startup a vagrant VM running opensuse-13 vagrantUpSles12 - Startup a vagrant VM running sles-12 vagrantUpUbuntu1204 - Startup a vagrant VM running ubuntu-1204 vagrantUpUbuntu1404 - Startup a vagrant VM running ubuntu-1404 vagrantUpUbuntu1504 - Startup a vagrant VM running ubuntu-1504 It does not make the "check" task depend on "checkPackages" so running the vagrant tests is still optional. They are slow and depend on vagrant and virtualbox. The Package Verification tasks are useful for testing individual distros. The Vagrant tasks are listed in `gradle tasks` primarily for discoverability.
2015-11-02 12:40:47 -05:00
'qa:vagrant',
'qa:verify-version-constants',
'qa:wildfly',
'qa:query-builder-bwc'
]
File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
List<String> examplePlugins = []
for (File example : examplePluginsDir.listFiles()) {
if (example.isDirectory() == false) continue;
if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
projects.add("example-plugins:${example.name}".toString())
examplePlugins.add(example.name)
}
boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
if (isEclipse) {
// eclipse cannot handle an intermediate dependency between main and test, so we must create separate projects
// for core-src and core-tests
projects << 'core-tests'
}
include projects.toArray(new String[0])
project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
for (String example : examplePlugins) {
project(":example-plugins:${example}").projectDir = new File(rootProject.projectDir, "plugins/examples/${example}")
}
/* bwc and bwc-unreleased share the same build directory and build file, but
* apply to different backwards compatibility branches. */
project(':distribution:bwc-release-snapshot').projectDir =
new File(rootProject.projectDir, 'distribution/bwc')
project(':distribution:bwc-stable-snapshot').projectDir =
new File(rootProject.projectDir, 'distribution/bwc')
if (isEclipse) {
2015-11-06 14:45:32 -05:00
project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main')
project(":core").buildFileName = 'eclipse-build.gradle'
project(":core-tests").projectDir = new File(rootProject.projectDir, 'core/src/test')
project(":core-tests").buildFileName = 'eclipse-build.gradle'
}
/**
* Iterates over sub directories, looking for build.gradle, and adds a project if found
* for that dir with the given path prefix. Note that this requires each level
* of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
* all files/directories in the source tree to find all projects.
*/
void addSubProjects(String path, File dir) {
if (dir.isDirectory() == false) return;
if (dir.name == 'buildSrc') return;
if (new File(dir, 'build.gradle').exists() == false) return;
String projectName = "${path}:${dir.name}"
include projectName
if (path.isEmpty()) {
project(projectName).projectDir = dir
}
for (File subdir : dir.listFiles()) {
addSubProjects(projectName, subdir)
}
}
// look for extra plugins for elasticsearch
File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
if (extraProjects.exists()) {
for (File extraProjectDir : extraProjects.listFiles()) {
addSubProjects('', extraProjectDir)
}
}