diff --git a/build.gradle b/build.gradle index 1326c8225e1..dac8232b172 100644 --- a/build.gradle +++ b/build.gradle @@ -109,6 +109,7 @@ subprojects { "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec', "org.elasticsearch:elasticsearch:${version}": ':core', "org.elasticsearch:test-framework:${version}": ':test-framework', + "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip', "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip', "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar', "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm', diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index fbbdb242d45..eea2041052a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -23,40 +23,41 @@ import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.test.RunTask import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.bundling.Zip /** * Encapsulates build configuration for an Elasticsearch plugin. */ -class PluginBuildPlugin extends BuildPlugin { +public class PluginBuildPlugin extends BuildPlugin { @Override - void apply(Project project) { + public void apply(Project project) { super.apply(project) configureDependencies(project) - // this afterEvaluate must happen before the afterEvaluate added by integTest configure, + // this afterEvaluate must happen before the afterEvaluate added by integTest creation, // so that the file name resolution for installing the plugin will be setup project.afterEvaluate { String name = project.pluginProperties.extension.name project.jar.baseName = name project.bundlePlugin.baseName = name + project.integTest.dependsOn(project.bundlePlugin) - project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) project.tasks.run.dependsOn(project.bundlePlugin) - project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) - } - RestIntegTestTask.configure(project) - RunTask.configure(project) - Task bundle = configureBundleTask(project) - project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar } - project.configurations.getByName('default').extendsFrom = [] - project.artifacts { - archives bundle - 'default' bundle + if (project.path.startsWith(':modules:')) { + project.integTest.clusterConfig.module(project) + project.tasks.run.clusterConfig.module(project) + } else { + project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) + project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files) + } } + createIntegTestTask(project) + createBundleTask(project) + project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build } - static void configureDependencies(Project project) { + private static void configureDependencies(Project project) { project.dependencies { provided "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}" testCompile "org.elasticsearch:test-framework:${project.versions.elasticsearch}" @@ -72,21 +73,36 @@ class PluginBuildPlugin extends BuildPlugin { } } - static Task configureBundleTask(Project project) { - PluginPropertiesTask buildProperties = project.tasks.create(name: 'pluginProperties', type: PluginPropertiesTask) - File pluginMetadata = project.file("src/main/plugin-metadata") - project.sourceSets.test { - output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties') - resources { - srcDir pluginMetadata - } - } - Task bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) - bundle.configure { - from buildProperties - from pluginMetadata - from project.jar - from bundle.project.configurations.runtime - bundle.project.configurations.provided + /** Adds an integTest task which runs rest tests */ + private static void createIntegTestTask(Project project) { + RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) + integTest.mustRunAfter(project.precommit, project.test) + project.check.dependsOn(integTest) + } + + /** + * Adds a bundlePlugin task which builds the zip containing the plugin jars, + * metadata, properties, and packaging files + */ + private static void createBundleTask(Project project) { + File pluginMetadata = project.file('src/main/plugin-metadata') + + // create a task to build the properties file for this plugin + PluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', PluginPropertiesTask.class) + + // add the plugin properties and metadata to test resources, so unit tests can + // know about the plugin (used by test security code to statically initialize the plugin in unit tests) + SourceSet testSourceSet = project.sourceSets.test + testSourceSet.output.dir(buildProperties.generatedResourcesDir, builtBy: 'pluginProperties') + testSourceSet.resources.srcDir(pluginMetadata) + + // create the actual bundle task, which zips up all the files for the plugin + Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) { + from buildProperties // plugin properties file + from pluginMetadata // metadata (eg custom security policy) + from project.jar // this plugin's jar + from project.configurations.runtime - project.configurations.provided // the dep jars + // extra files for the plugin to go into the zip from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging from('src/main') { include 'config/**' @@ -97,6 +113,13 @@ class PluginBuildPlugin extends BuildPlugin { } } project.assemble.dependsOn(bundle) - return bundle + + // remove jar from the archives (things that will be published), and set it to the zip + project.configurations.archives.artifacts.removeAll { it.archiveTask.is project.jar } + project.artifacts.add('archives', bundle) + + // also make the zip the default artifact (used when depending on this project) + project.configurations.getByName('default').extendsFrom = [] + project.artifacts.add('default', bundle) } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 79a199e98e4..8bc80da74b5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -27,7 +27,7 @@ import org.gradle.api.tasks.Input class ClusterConfiguration { @Input - String distribution = 'zip' + String distribution = 'integ-test-zip' @Input int numNodes = 1 @@ -71,6 +71,8 @@ class ClusterConfiguration { LinkedHashMap plugins = new LinkedHashMap<>() + List modules = new ArrayList<>() + LinkedHashMap setupCommands = new LinkedHashMap<>() @Input @@ -93,6 +95,12 @@ class ClusterConfiguration { plugins.put(name, pluginProject) } + /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */ + @Input + void module(Project moduleProject) { + modules.add(moduleProject) + } + @Input void setupCommand(String name, Object... args) { setupCommands.put(name, args) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 355939d88f6..fff6082b3e5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -60,7 +60,12 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro) { String elasticsearchVersion = VersionProperties.elasticsearch - String packaging = distro == 'tar' ? 'tar.gz' : distro + String packaging = distro + if (distro == 'tar') { + packaging = 'tar.gz' + } else if (distro == 'integ-test-zip') { + packaging = 'zip' + } project.configurations { elasticsearchDistro } @@ -103,6 +108,12 @@ class ClusterFormationTasks { setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node) setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node) + // install modules + for (Project module : node.config.modules) { + String actionName = pluginTaskName('install', module.name, 'Module') + setup = configureInstallModuleTask(taskName(task, node, actionName), project, setup, node, module) + } + // install plugins for (Map.Entry plugin : node.config.plugins.entrySet()) { String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin') @@ -138,6 +149,7 @@ class ClusterFormationTasks { by the source tree. If it isn't then Bad Things(TM) will happen. */ Task extract switch (node.config.distribution) { + case 'integ-test-zip': case 'zip': extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) } @@ -286,6 +298,20 @@ class ClusterFormationTasks { return copyPlugins } + static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) { + if (node.config.distribution != 'integ-test-zip') { + throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!") + } + if (module.plugins.hasPlugin(PluginBuildPlugin) == false) { + throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin") + } + Copy installModule = project.tasks.create(name, Copy.class) + installModule.dependsOn(setup) + installModule.into(new File(node.homeDir, "modules/${module.name}")) + installModule.from({ project.zipTree(module.tasks.bundlePlugin.outputs.files.singleFile) }) + return installModule + } + static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Object plugin) { FileCollection pluginZip if (plugin instanceof Project) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 2247c894dce..337eea3de97 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -173,6 +173,7 @@ class NodeInfo { static File homeDir(File baseDir, String distro) { String path switch (distro) { + case 'integ-test-zip': case 'zip': case 'tar': path = "elasticsearch-${VersionProperties.elasticsearch}" @@ -188,8 +189,8 @@ class NodeInfo { } static File confDir(File baseDir, String distro) { - String Path switch (distro) { + case 'integ-test-zip': case 'zip': case 'tar': return new File(homeDir(baseDir, distro), 'config') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index 4dc94c40108..cd43cd2ca67 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -31,55 +31,38 @@ import org.gradle.util.ConfigureUtil * Runs integration tests, but first starts an ES cluster, * and passes the ES cluster info as parameters to the tests. */ -class RestIntegTestTask extends RandomizedTestingTask { +public class RestIntegTestTask extends RandomizedTestingTask { ClusterConfiguration clusterConfig = new ClusterConfiguration() + /** Flag indicating whether the rest tests in the rest spec should be run. */ @Input boolean includePackaged = false - static RestIntegTestTask configure(Project project) { - Map integTestOptions = [ - name: 'integTest', - type: RestIntegTestTask, - dependsOn: 'testClasses', - group: JavaBasePlugin.VERIFICATION_GROUP, - description: 'Runs rest tests against an elasticsearch cluster.' - ] - RestIntegTestTask integTest = project.tasks.create(integTestOptions) - integTest.configure(BuildPlugin.commonTestConfig(project)) - integTest.configure { - include '**/*IT.class' - systemProperty 'tests.rest.load_packaged', 'false' - } - RandomizedTestingTask test = project.tasks.findByName('test') - if (test != null) { - integTest.classpath = test.classpath - integTest.testClassesDir = test.testClassesDir - integTest.mustRunAfter(test) - } - integTest.mustRunAfter(project.precommit) - project.check.dependsOn(integTest) + public RestIntegTestTask() { + description = 'Runs rest tests against an elasticsearch cluster.' + group = JavaBasePlugin.VERIFICATION_GROUP + dependsOn(project.testClasses) + classpath = project.sourceSets.test.runtimeClasspath + testClassesDir = project.sourceSets.test.output.classesDir + + // start with the common test configuration + configure(BuildPlugin.commonTestConfig(project)) + // override/add more for rest tests + parallelism = '1' + include('**/*IT.class') + systemProperty('tests.rest.load_packaged', 'false') + + // copy the rest spec/tests into the test resources RestSpecHack.configureDependencies(project) project.afterEvaluate { - integTest.dependsOn(RestSpecHack.configureTask(project, integTest.includePackaged)) + dependsOn(RestSpecHack.configureTask(project, includePackaged)) + systemProperty('tests.cluster', "localhost:${clusterConfig.baseTransportPort}") } - return integTest - } - - RestIntegTestTask() { // this must run after all projects have been configured, so we know any project // references can be accessed as a fully configured project.gradle.projectsEvaluated { - Task test = project.tasks.findByName('test') - if (test != null) { - mustRunAfter(test) - } ClusterFormationTasks.setup(project, this, clusterConfig) - configure { - parallelism '1' - systemProperty 'tests.cluster', "localhost:${clusterConfig.baseTransportPort}" - } } } @@ -92,11 +75,11 @@ class RestIntegTestTask extends RandomizedTestingTask { } @Input - void cluster(Closure closure) { + public void cluster(Closure closure) { ConfigureUtil.configure(closure, clusterConfig) } - ClusterConfiguration getCluster() { + public ClusterConfiguration getCluster() { return clusterConfig } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy index e0af8f4cc8e..43b5c2f6f38 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy @@ -28,12 +28,12 @@ import org.gradle.api.tasks.Copy * currently must be available on the local filesystem. This class encapsulates * setting up tasks to copy the rest spec api to test resources. */ -class RestSpecHack { +public class RestSpecHack { /** * Sets dependencies needed to copy the rest spec. * @param project The project to add rest spec dependency to */ - static void configureDependencies(Project project) { + public static void configureDependencies(Project project) { project.configurations { restSpec } @@ -48,7 +48,7 @@ class RestSpecHack { * @param project The project to add the copy task to * @param includePackagedTests true if the packaged tests should be copied, false otherwise */ - static Task configureTask(Project project, boolean includePackagedTests) { + public static Task configureTask(Project project, boolean includePackagedTests) { Map copyRestSpecProps = [ name : 'copyRestSpec', type : Copy, @@ -65,7 +65,6 @@ class RestSpecHack { project.idea { module { if (scopes.TEST != null) { - // TODO: need to add the TEST scope somehow for rest test plugin... scopes.TEST.plus.add(project.configurations.restSpec) } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy index 2be80ca005a..dc9aa769388 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy @@ -18,22 +18,19 @@ */ package org.elasticsearch.gradle.test -import com.carrotsearch.gradle.junit4.RandomizedTestingTask import org.gradle.api.Plugin import org.gradle.api.Project -/** Configures the build to have a rest integration test. */ -class RestTestPlugin implements Plugin { +/** A plugin to add rest integration tests. Used for qa projects. */ +public class RestTestPlugin implements Plugin { @Override - void apply(Project project) { + public void apply(Project project) { project.pluginManager.apply(StandaloneTestBasePlugin) - RandomizedTestingTask integTest = RestIntegTestTask.configure(project) - RestSpecHack.configureDependencies(project) - integTest.configure { - classpath = project.sourceSets.test.runtimeClasspath - testClassesDir project.sourceSets.test.output.classesDir - } + RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class) + integTest.cluster.distribution = 'zip' // rest tests should run with the real zip + integTest.mustRunAfter(project.precommit) + project.check.dependsOn(integTest) } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy index 37f65c88703..32469218a5b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy @@ -2,13 +2,17 @@ package org.elasticsearch.gradle.test import org.gradle.api.DefaultTask import org.gradle.api.Project +import org.gradle.api.Task import org.gradle.api.internal.tasks.options.Option +import org.gradle.util.ConfigureUtil -class RunTask extends DefaultTask { +public class RunTask extends DefaultTask { ClusterConfiguration clusterConfig = new ClusterConfiguration(baseHttpPort: 9200, baseTransportPort: 9300, daemonize: false) - RunTask() { + public RunTask() { + description = "Runs elasticsearch with '${project.path}'" + group = 'Verification' project.afterEvaluate { ClusterFormationTasks.setup(project, this, clusterConfig) } @@ -22,11 +26,10 @@ class RunTask extends DefaultTask { clusterConfig.debug = enabled; } - static void configure(Project project) { - RunTask task = project.tasks.create( - name: 'run', - type: RunTask, - description: "Runs elasticsearch with '${project.path}'", - group: 'Verification') + /** Configure the cluster that will be run. */ + @Override + public Task configure(Closure closure) { + ConfigureUtil.configure(closure, clusterConfig) + return this } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy index 62f6bd553a4..8c3b5f754cc 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy @@ -27,35 +27,26 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.plugins.ide.eclipse.model.EclipseClasspath /** Configures the build to have a rest integration test. */ -class StandaloneTestBasePlugin implements Plugin { +public class StandaloneTestBasePlugin implements Plugin { @Override - void apply(Project project) { + public void apply(Project project) { project.pluginManager.apply(JavaBasePlugin) project.pluginManager.apply(RandomizedTestingPlugin) BuildPlugin.globalBuildInfo(project) BuildPlugin.configureRepositories(project) - // remove some unnecessary tasks for a qa test - project.tasks.removeAll { it.name in ['assemble', 'buildDependents'] } - // only setup tests to build - project.sourceSets { - test - } - project.dependencies { - testCompile "org.elasticsearch:test-framework:${VersionProperties.elasticsearch}" - } + project.sourceSets.create('test') + project.dependencies.add('testCompile', "org.elasticsearch:test-framework:${VersionProperties.elasticsearch}") + + project.eclipse.classpath.sourceSets = [project.sourceSets.test] + project.eclipse.classpath.plusConfigurations = [project.configurations.testRuntime] - project.eclipse { - classpath { - sourceSets = [project.sourceSets.test] - plusConfigurations = [project.configurations.testRuntime] - } - } PrecommitTasks.create(project, false) project.check.dependsOn(project.precommit) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy index b560500aae3..0a2cc841282 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy @@ -25,11 +25,11 @@ import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.JavaBasePlugin -/** Configures the build to have only unit tests. */ -class StandaloneTestPlugin implements Plugin { +/** A plugin to add tests only. Used for QA tests that run arbitrary unit tests. */ +public class StandaloneTestPlugin implements Plugin { @Override - void apply(Project project) { + public void apply(Project project) { project.pluginManager.apply(StandaloneTestBasePlugin) Map testOptions = [ @@ -41,10 +41,8 @@ class StandaloneTestPlugin implements Plugin { ] RandomizedTestingTask test = project.tasks.create(testOptions) test.configure(BuildPlugin.commonTestConfig(project)) - test.configure { - classpath = project.sourceSets.test.runtimeClasspath - testClassesDir project.sourceSets.test.output.classesDir - } + test.classpath = project.sourceSets.test.runtimeClasspath + test.testClassesDir project.sourceSets.test.output.classesDir test.mustRunAfter(project.precommit) project.check.dependsOn(test) } diff --git a/core/build.gradle b/core/build.gradle index eb65c67cf56..3db5097ea7a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -129,8 +129,4 @@ if (isEclipse == false || project.path == ":core-tests") { } check.dependsOn integTest integTest.mustRunAfter test - - RestSpecHack.configureDependencies(project) - Task copyRestSpec = RestSpecHack.configureTask(project, true) - integTest.dependsOn copyRestSpec } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 2d683852012..1fa64d5e7b7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -72,14 +72,14 @@ public class NodeInfo extends BaseNodeResponse { private HttpInfo http; @Nullable - private PluginsInfo plugins; + private PluginsAndModules plugins; NodeInfo() { } public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map serviceAttributes, @Nullable Settings settings, @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool, - @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsInfo plugins) { + @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins) { super(node); this.version = version; this.build = build; @@ -172,7 +172,7 @@ public class NodeInfo extends BaseNodeResponse { } @Nullable - public PluginsInfo getPlugins() { + public PluginsAndModules getPlugins() { return this.plugins; } @@ -217,7 +217,8 @@ public class NodeInfo extends BaseNodeResponse { http = HttpInfo.readHttpInfo(in); } if (in.readBoolean()) { - plugins = PluginsInfo.readPluginsInfo(in); + plugins = new PluginsAndModules(); + plugins.readFrom(in); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java new file mode 100644 index 00000000000..3831fd24f3e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.info; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.plugins.PluginInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Information about plugins and modules + */ +public class PluginsAndModules implements Streamable, ToXContent { + private List plugins; + private List modules; + + public PluginsAndModules() { + plugins = new ArrayList<>(); + modules = new ArrayList<>(); + } + + /** + * Returns an ordered list based on plugins name + */ + public List getPluginInfos() { + List plugins = new ArrayList<>(this.plugins); + Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName())); + return plugins; + } + + /** + * Returns an ordered list based on modules name + */ + public List getModuleInfos() { + List modules = new ArrayList<>(this.modules); + Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName())); + return modules; + } + + public void addPlugin(PluginInfo info) { + plugins.add(info); + } + + public void addModule(PluginInfo info) { + modules.add(info); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + if (plugins.isEmpty() == false || modules.isEmpty() == false) { + throw new IllegalStateException("instance is already populated"); + } + int plugins_size = in.readInt(); + for (int i = 0; i < plugins_size; i++) { + plugins.add(PluginInfo.readFromStream(in)); + } + int modules_size = in.readInt(); + for (int i = 0; i < modules_size; i++) { + modules.add(PluginInfo.readFromStream(in)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(plugins.size()); + for (PluginInfo plugin : getPluginInfos()) { + plugin.writeTo(out); + } + out.writeInt(modules.size()); + for (PluginInfo module : getModuleInfos()) { + module.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("plugins"); + for (PluginInfo pluginInfo : getPluginInfos()) { + pluginInfo.toXContent(builder, params); + } + builder.endArray(); + // TODO: not ideal, make a better api for this (e.g. with jar metadata, and so on) + builder.startArray("modules"); + for (PluginInfo moduleInfo : getModuleInfos()) { + moduleInfo.toXContent(builder, params); + } + builder.endArray(); + + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java deleted file mode 100644 index 927a79b6639..00000000000 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.node.info; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.elasticsearch.plugins.PluginInfo; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -public class PluginsInfo implements Streamable, ToXContent { - static final class Fields { - static final XContentBuilderString PLUGINS = new XContentBuilderString("plugins"); - } - - private List infos; - - public PluginsInfo() { - infos = new ArrayList<>(); - } - - public PluginsInfo(int size) { - infos = new ArrayList<>(size); - } - - /** - * @return an ordered list based on plugins name - */ - public List getInfos() { - Collections.sort(infos, new Comparator() { - @Override - public int compare(final PluginInfo o1, final PluginInfo o2) { - return o1.getName().compareTo(o2.getName()); - } - }); - - return infos; - } - - public void add(PluginInfo info) { - infos.add(info); - } - - public static PluginsInfo readPluginsInfo(StreamInput in) throws IOException { - PluginsInfo infos = new PluginsInfo(); - infos.readFrom(in); - return infos; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - int plugins_size = in.readInt(); - for (int i = 0; i < plugins_size; i++) { - infos.add(PluginInfo.readFromStream(in)); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(infos.size()); - for (PluginInfo plugin : getInfos()) { - plugin.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(Fields.PLUGINS); - for (PluginInfo pluginInfo : getInfos()) { - pluginInfo.toXContent(builder, params); - } - builder.endArray(); - - return builder; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index fe8d4121c75..d8f2a5bbd20 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -74,7 +74,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { versions.add(nodeResponse.nodeInfo().getVersion()); process.addNodeStats(nodeResponse.nodeStats()); jvm.addNodeInfoStats(nodeResponse.nodeInfo(), nodeResponse.nodeStats()); - plugins.addAll(nodeResponse.nodeInfo().getPlugins().getInfos()); + plugins.addAll(nodeResponse.nodeInfo().getPlugins().getPluginInfos()); // now do the stats that should be deduped by hardware (implemented by ip deduping) TransportAddress publishAddress = nodeResponse.nodeInfo().getTransport().address().publishAddress(); diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index 3d7ce9a1fca..2d342eb5743 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -131,34 +131,48 @@ final class Security { @SuppressForbidden(reason = "proper use of URL") static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException { Map map = new HashMap<>(); + // collect up lists of plugins and modules + List pluginsAndModules = new ArrayList<>(); if (Files.exists(environment.pluginsFile())) { try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) { for (Path plugin : stream) { - Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY); - if (Files.exists(policyFile)) { - // first get a list of URLs for the plugins' jars: - // we resolve symlinks so map is keyed on the normalize codebase name - List codebases = new ArrayList<>(); - try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { - for (Path jar : jarStream) { - codebases.add(jar.toRealPath().toUri().toURL()); - } - } - - // parse the plugin's policy file into a set of permissions - Policy policy = readPolicy(policyFile.toUri().toURL(), codebases.toArray(new URL[codebases.size()])); - - // consult this policy for each of the plugin's jars: - for (URL url : codebases) { - if (map.put(url.getFile(), policy) != null) { - // just be paranoid ok? - throw new IllegalStateException("per-plugin permissions already granted for jar file: " + url); - } - } + pluginsAndModules.add(plugin); + } + } + } + if (Files.exists(environment.modulesFile())) { + try (DirectoryStream stream = Files.newDirectoryStream(environment.modulesFile())) { + for (Path plugin : stream) { + pluginsAndModules.add(plugin); + } + } + } + // now process each one + for (Path plugin : pluginsAndModules) { + Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY); + if (Files.exists(policyFile)) { + // first get a list of URLs for the plugins' jars: + // we resolve symlinks so map is keyed on the normalize codebase name + List codebases = new ArrayList<>(); + try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { + for (Path jar : jarStream) { + codebases.add(jar.toRealPath().toUri().toURL()); + } + } + + // parse the plugin's policy file into a set of permissions + Policy policy = readPolicy(policyFile.toUri().toURL(), codebases.toArray(new URL[codebases.size()])); + + // consult this policy for each of the plugin's jars: + for (URL url : codebases) { + if (map.put(url.getFile(), policy) != null) { + // just be paranoid ok? + throw new IllegalStateException("per-plugin permissions already granted for jar file: " + url); } } } } + return Collections.unmodifiableMap(map); } @@ -228,6 +242,7 @@ final class Security { // read-only dirs addPath(policy, "path.home", environment.binFile(), "read,readlink"); addPath(policy, "path.home", environment.libFile(), "read,readlink"); + addPath(policy, "path.home", environment.modulesFile(), "read,readlink"); addPath(policy, "path.plugins", environment.pluginsFile(), "read,readlink"); addPath(policy, "path.conf", environment.configFile(), "read,readlink"); addPath(policy, "path.scripts", environment.scriptsFile(), "read,readlink"); diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 73c950e8bf4..33cf3479419 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -125,7 +125,7 @@ public class TransportClient extends AbstractClient { .put(CLIENT_TYPE_SETTING, CLIENT_TYPE) .build(); - PluginsService pluginsService = new PluginsService(settings, null, pluginClasses); + PluginsService pluginsService = new PluginsService(settings, null, null, pluginClasses); this.settings = pluginsService.updatedSettings(); Version version = Version.CURRENT; diff --git a/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java b/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java index 7952cbe2892..feb4707eac8 100644 --- a/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java +++ b/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java @@ -65,8 +65,8 @@ public class MacAddressProvider { byte[] address = null; try { address = getMacAddress(); - } catch( SocketException se ) { - logger.warn("Unable to get mac address, will use a dummy address", se); + } catch (Throwable t) { + logger.warn("Unable to get mac address, will use a dummy address", t); // address will be set below } diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java index 53081f55d21..2abe730b1e8 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; @@ -41,7 +42,7 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new public class NodesFaultDetection extends FaultDetection { public static final String PING_ACTION_NAME = "internal:discovery/zen/fd/ping"; - + public abstract static class Listener { public void onNodeFailure(DiscoveryNode node, String reason) {} @@ -145,14 +146,18 @@ public class NodesFaultDetection extends FaultDetection { } private void notifyNodeFailure(final DiscoveryNode node, final String reason) { - threadPool.generic().execute(new Runnable() { - @Override - public void run() { - for (Listener listener : listeners) { - listener.onNodeFailure(node, reason); + try { + threadPool.generic().execute(new Runnable() { + @Override + public void run() { + for (Listener listener : listeners) { + listener.onNodeFailure(node, reason); + } } - } - }); + }); + } catch (EsRejectedExecutionException ex) { + logger.trace("[node ] [{}] ignoring node failure (reason [{}]). Local node is shutting down", ex, node, reason); + } } private void notifyPingReceived(final PingRequest pingRequest) { diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index c5c769a73c2..7982c2f35ea 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -58,6 +58,8 @@ public class Environment { private final Path pluginsFile; + private final Path modulesFile; + private final Path sharedDataFile; /** location of bin/, used by plugin manager */ @@ -157,6 +159,7 @@ public class Environment { binFile = homeFile.resolve("bin"); libFile = homeFile.resolve("lib"); + modulesFile = homeFile.resolve("modules"); } /** @@ -275,6 +278,10 @@ public class Environment { return libFile; } + public Path modulesFile() { + return modulesFile; + } + public Path logsFile() { return logsFile; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 53e875cea91..c4fec8cf095 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -336,8 +336,6 @@ public class DocumentMapper implements ToXContent { private void addMappers(Collection objectMappers, Collection fieldMappers, boolean updateAllTypes) { assert mappingLock.isWriteLockedByCurrentThread(); - // first ensure we don't have any incompatible new fields - mapperService.checkNewMappersCompatibility(objectMappers, fieldMappers, updateAllTypes); // update mappers for this document type Map builder = new HashMap<>(this.objectMappers); @@ -356,6 +354,7 @@ public class DocumentMapper implements ToXContent { public MergeResult merge(Mapping mapping, boolean simulate, boolean updateAllTypes) { try (ReleasableLock lock = mappingWriteLock.acquire()) { + mapperService.checkMappersCompatibility(type, mapping, updateAllTypes); final MergeResult mergeResult = new MergeResult(simulate, updateAllTypes); this.mapping.merge(mapping, mergeResult); if (simulate == false) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index aef8d474a6e..e5e3387950e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -28,8 +28,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ReleasableLock; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.core.DateFieldMapper.DateFieldType; @@ -47,7 +45,6 @@ import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; /** A parser for documents, given mappings from a DocumentMapper */ @@ -716,37 +713,64 @@ class DocumentParser implements Closeable { // The path of the dest field might be completely different from the current one so we need to reset it context = context.overridePath(new ContentPath(0)); + String[] paths = Strings.splitStringToArray(field, '.'); + String fieldName = paths[paths.length-1]; ObjectMapper mapper = context.root(); - String objectPath = ""; - String fieldPath = field; - int posDot = field.lastIndexOf('.'); - if (posDot > 0) { - objectPath = field.substring(0, posDot); - context.path().add(objectPath); - mapper = context.docMapper().objectMappers().get(objectPath); - fieldPath = field.substring(posDot + 1); + ObjectMapper[] mappers = new ObjectMapper[paths.length-1]; + if (paths.length > 1) { + ObjectMapper parent = context.root(); + for (int i = 0; i < paths.length-1; i++) { + mapper = context.docMapper().objectMappers().get(context.path().fullPathAsText(paths[i])); + if (mapper == null) { + // One mapping is missing, check if we are allowed to create a dynamic one. + ObjectMapper.Dynamic dynamic = parent.dynamic(); + if (dynamic == null) { + dynamic = dynamicOrDefault(context.root().dynamic()); + } + + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException(parent.fullPath(), paths[i]); + case TRUE: + Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object"); + if (builder == null) { + // if this is a non root object, then explicitly set the dynamic behavior if set + if (!(parent instanceof RootObjectMapper) && parent.dynamic() != ObjectMapper.Defaults.DYNAMIC) { + ((ObjectMapper.Builder) builder).dynamic(parent.dynamic()); + } + builder = MapperBuilders.object(paths[i]).enabled(true).pathType(parent.pathType()); + } + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path()); + mapper = (ObjectMapper) builder.build(builderContext); + if (mapper.nested() != ObjectMapper.Nested.NO) { + throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().fullPathAsText(paths[i]) + "]) through `copy_to`"); + } + break; + case FALSE: + // Maybe we should log something to tell the user that the copy_to is ignored in this case. + break; + default: + throw new AssertionError("Unexpected dynamic type " + dynamic); + + } + } + context.path().add(paths[i]); + mappers[i] = mapper; + parent = mapper; + } } - if (mapper == null) { - //TODO: Create an object dynamically? - throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]"); - } - ObjectMapper update = parseDynamicValue(context, mapper, fieldPath, context.parser().currentToken()); + ObjectMapper update = parseDynamicValue(context, mapper, fieldName, context.parser().currentToken()); assert update != null; // we are parsing a dynamic value so we necessarily created a new mapping - // propagate the update to the root - while (objectPath.length() > 0) { - String parentPath = ""; - ObjectMapper parent = context.root(); - posDot = objectPath.lastIndexOf('.'); - if (posDot > 0) { - parentPath = objectPath.substring(0, posDot); - parent = context.docMapper().objectMappers().get(parentPath); + if (paths.length > 1) { + for (int i = paths.length - 2; i >= 0; i--) { + ObjectMapper parent = context.root(); + if (i > 0) { + parent = mappers[i-1]; + } + assert parent != null; + update = parent.mappingUpdate(update); } - if (parent == null) { - throw new IllegalStateException("[" + objectPath + "] has no parent for path [" + parentPath + "]"); - } - update = parent.mappingUpdate(update); - objectPath = parentPath; } context.addDynamicMappingsUpdate(update); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 552067ac337..c277cdc4728 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -307,7 +307,6 @@ public abstract class FieldMapper extends Mapper { if (ref.get().equals(fieldType()) == false) { throw new IllegalStateException("Cannot overwrite field type reference to unequal reference"); } - ref.incrementAssociatedMappers(); this.fieldTypeRef = ref; } @@ -380,11 +379,6 @@ public abstract class FieldMapper extends Mapper { return; } - boolean strict = this.fieldTypeRef.getNumAssociatedMappers() > 1 && mergeResult.updateAllTypes() == false; - fieldType().checkCompatibility(fieldMergeWith.fieldType(), subConflicts, strict); - for (String conflict : subConflicts) { - mergeResult.addConflict(conflict); - } multiFields.merge(mergeWith, mergeResult); if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 3fad73ebba6..eaaa47c3bd2 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.regex.Regex; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -38,18 +39,49 @@ class FieldTypeLookup implements Iterable { /** Full field name to field type */ private final CopyOnWriteHashMap fullNameToFieldType; + /** Full field name to types containing a mapping for this full name. */ + private final CopyOnWriteHashMap> fullNameToTypes; + /** Index field name to field type */ private final CopyOnWriteHashMap indexNameToFieldType; + /** Index field name to types containing a mapping for this index name. */ + private final CopyOnWriteHashMap> indexNameToTypes; + /** Create a new empty instance. */ public FieldTypeLookup() { fullNameToFieldType = new CopyOnWriteHashMap<>(); + fullNameToTypes = new CopyOnWriteHashMap<>(); indexNameToFieldType = new CopyOnWriteHashMap<>(); + indexNameToTypes = new CopyOnWriteHashMap<>(); } - private FieldTypeLookup(CopyOnWriteHashMap fullName, CopyOnWriteHashMap indexName) { - fullNameToFieldType = fullName; - indexNameToFieldType = indexName; + private FieldTypeLookup( + CopyOnWriteHashMap fullName, + CopyOnWriteHashMap> fullNameToTypes, + CopyOnWriteHashMap indexName, + CopyOnWriteHashMap> indexNameToTypes) { + this.fullNameToFieldType = fullName; + this.fullNameToTypes = fullNameToTypes; + this.indexNameToFieldType = indexName; + this.indexNameToTypes = indexNameToTypes; + } + + private static CopyOnWriteHashMap> addType(CopyOnWriteHashMap> map, String key, String type) { + Set types = map.get(key); + if (types == null) { + return map.copyAndPut(key, Collections.singleton(type)); + } else if (types.contains(type)) { + // noting to do + return map; + } else { + Set newTypes = new HashSet<>(types.size() + 1); + newTypes.addAll(types); + newTypes.add(type); + assert newTypes.size() == types.size() + 1; + newTypes = Collections.unmodifiableSet(newTypes); + return map.copyAndPut(key, newTypes); + } } /** @@ -63,7 +95,9 @@ class FieldTypeLookup implements Iterable { throw new IllegalArgumentException("Default mappings should not be added to the lookup"); } CopyOnWriteHashMap fullName = this.fullNameToFieldType; + CopyOnWriteHashMap> fullNameToTypes = this.fullNameToTypes; CopyOnWriteHashMap indexName = this.indexNameToFieldType; + CopyOnWriteHashMap> indexNameToTypes = this.indexNameToTypes; for (FieldMapper fieldMapper : newFieldMappers) { MappedFieldType fieldType = fieldMapper.fieldType(); @@ -91,8 +125,23 @@ class FieldTypeLookup implements Iterable { // this new field bridges between two existing field names (a full and index name), which we cannot support throw new IllegalStateException("insane mappings found. field " + fieldType.names().fullName() + " maps across types to field " + fieldType.names().indexName()); } + + fullNameToTypes = addType(fullNameToTypes, fieldType.names().fullName(), type); + indexNameToTypes = addType(indexNameToTypes, fieldType.names().indexName(), type); + } + return new FieldTypeLookup(fullName, fullNameToTypes, indexName, indexNameToTypes); + } + + private static boolean beStrict(String type, Set types, boolean updateAllTypes) { + assert types.size() >= 1; + if (updateAllTypes) { + return false; + } else if (types.size() == 1 && types.contains(type)) { + // we are implicitly updating all types + return false; + } else { + return true; } - return new FieldTypeLookup(fullName, indexName); } /** @@ -100,14 +149,15 @@ class FieldTypeLookup implements Iterable { * If any are not compatible, an IllegalArgumentException is thrown. * If updateAllTypes is true, only basic compatibility is checked. */ - public void checkCompatibility(Collection newFieldMappers, boolean updateAllTypes) { - for (FieldMapper fieldMapper : newFieldMappers) { + public void checkCompatibility(String type, Collection fieldMappers, boolean updateAllTypes) { + for (FieldMapper fieldMapper : fieldMappers) { MappedFieldTypeReference ref = fullNameToFieldType.get(fieldMapper.fieldType().names().fullName()); if (ref != null) { List conflicts = new ArrayList<>(); ref.get().checkTypeName(fieldMapper.fieldType(), conflicts); if (conflicts.isEmpty()) { // only check compat if they are the same type - boolean strict = updateAllTypes == false; + final Set types = fullNameToTypes.get(fieldMapper.fieldType().names().fullName()); + boolean strict = beStrict(type, types, updateAllTypes); ref.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); } if (conflicts.isEmpty() == false) { @@ -121,7 +171,8 @@ class FieldTypeLookup implements Iterable { List conflicts = new ArrayList<>(); indexNameRef.get().checkTypeName(fieldMapper.fieldType(), conflicts); if (conflicts.isEmpty()) { // only check compat if they are the same type - boolean strict = updateAllTypes == false; + final Set types = indexNameToTypes.get(fieldMapper.fieldType().names().indexName()); + boolean strict = beStrict(type, types, updateAllTypes); indexNameRef.get().checkCompatibility(fieldMapper.fieldType(), conflicts, strict); } if (conflicts.isEmpty() == false) { @@ -138,6 +189,15 @@ class FieldTypeLookup implements Iterable { return ref.get(); } + /** Get the set of types that have a mapping for the given field. */ + public Set getTypes(String field) { + Set types = fullNameToTypes.get(field); + if (types == null) { + types = Collections.emptySet(); + } + return types; + } + /** Returns the field type for the given index name */ public MappedFieldType getByIndexName(String field) { MappedFieldTypeReference ref = indexNameToFieldType.get(field); @@ -145,6 +205,15 @@ class FieldTypeLookup implements Iterable { return ref.get(); } + /** Get the set of types that have a mapping for the given field. */ + public Set getTypesByIndexName(String field) { + Set types = indexNameToTypes.get(field); + if (types == null) { + types = Collections.emptySet(); + } + return types; + } + /** * Returns a list of the index names of a simple match regex like pattern against full name and index name. */ diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java index d3c6b83a6a3..1a9d0b70b37 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldTypeReference.java @@ -23,12 +23,10 @@ package org.elasticsearch.index.mapper; */ public class MappedFieldTypeReference { private MappedFieldType fieldType; // the current field type this reference points to - private int numAssociatedMappers; public MappedFieldTypeReference(MappedFieldType fieldType) { fieldType.freeze(); // ensure frozen this.fieldType = fieldType; - this.numAssociatedMappers = 1; } public MappedFieldType get() { @@ -40,11 +38,4 @@ public class MappedFieldTypeReference { this.fieldType = fieldType; } - public int getNumAssociatedMappers() { - return numAssociatedMappers; - } - - public void incrementAssociatedMappers() { - ++numAssociatedMappers; - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 7a908a1238b..938f610d6db 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -33,6 +33,7 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; @@ -260,13 +261,10 @@ public class MapperService extends AbstractIndexComponent implements Closeable { assert result.hasConflicts() == false; // we already simulated return oldMapper; } else { - List newObjectMappers = new ArrayList<>(); - List newFieldMappers = new ArrayList<>(); - for (MetadataFieldMapper metadataMapper : mapper.mapping().metadataMappers) { - newFieldMappers.add(metadataMapper); - } - MapperUtils.collect(mapper.mapping().root, newObjectMappers, newFieldMappers); - checkNewMappersCompatibility(newObjectMappers, newFieldMappers, updateAllTypes); + Tuple, Collection> newMappers = checkMappersCompatibility( + mapper.type(), mapper.mapping(), updateAllTypes); + Collection newObjectMappers = newMappers.v1(); + Collection newFieldMappers = newMappers.v2(); addMappers(mapper.type(), newObjectMappers, newFieldMappers); for (DocumentTypeListener typeListener : typeListeners) { @@ -302,9 +300,9 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return true; } - protected void checkNewMappersCompatibility(Collection newObjectMappers, Collection newFieldMappers, boolean updateAllTypes) { + protected void checkMappersCompatibility(String type, Collection objectMappers, Collection fieldMappers, boolean updateAllTypes) { assert mappingLock.isWriteLockedByCurrentThread(); - for (ObjectMapper newObjectMapper : newObjectMappers) { + for (ObjectMapper newObjectMapper : objectMappers) { ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath()); if (existingObjectMapper != null) { MergeResult result = new MergeResult(true, updateAllTypes); @@ -315,7 +313,19 @@ public class MapperService extends AbstractIndexComponent implements Closeable { } } } - fieldTypes.checkCompatibility(newFieldMappers, updateAllTypes); + fieldTypes.checkCompatibility(type, fieldMappers, updateAllTypes); + } + + protected Tuple, Collection> checkMappersCompatibility( + String type, Mapping mapping, boolean updateAllTypes) { + List objectMappers = new ArrayList<>(); + List fieldMappers = new ArrayList<>(); + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + fieldMappers.add(metadataMapper); + } + MapperUtils.collect(mapping.root, objectMappers, fieldMappers); + checkMappersCompatibility(type, objectMappers, fieldMappers, updateAllTypes); + return new Tuple<>(objectMappers, fieldMappers); } protected void addMappers(String type, Collection objectMappers, Collection fieldMappers) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 7045ec2e58b..87a63de99ec 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -135,6 +135,15 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM super(ref); } + @Override + public void checkCompatibility(MappedFieldType other, + List conflicts, boolean strict) { + super.checkCompatibility(other, conflicts, strict); + if (numericPrecisionStep() != other.numericPrecisionStep()) { + conflicts.add("mapper [" + names().fullName() + "] has different [precision_step] values"); + } + } + public abstract NumberFieldType clone(); @Override @@ -251,11 +260,6 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM return; } NumberFieldMapper nfmMergeWith = (NumberFieldMapper) mergeWith; - if (this.fieldTypeRef.getNumAssociatedMappers() > 1 && mergeResult.updateAllTypes() == false) { - if (fieldType().numericPrecisionStep() != nfmMergeWith.fieldType().numericPrecisionStep()) { - mergeResult.addConflict("mapper [" + fieldType().names().fullName() + "] is used by multiple types. Set update_all_types to true to update precision_step across all types."); - } - } if (mergeResult.simulate() == false && mergeResult.hasConflicts() == false) { this.includeInAll = nfmMergeWith.includeInAll; diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index b8bbb65c495..7804b50c390 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1034,7 +1034,7 @@ public class IndexShard extends AbstractIndexShardComponent { boolean wasActive = active.getAndSet(false); if (wasActive) { updateBufferSize(IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER, IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER); - logger.debug("shard is now inactive"); + logger.debug("marking shard as inactive (inactive_time=[{}]) indexing wise", inactiveTime); indexEventListener.onShardInactive(this); } } diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index bb6f85bc0ba..9a78d8b2c1f 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices.memory; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -33,7 +32,6 @@ import org.elasticsearch.index.engine.FlushNotAllowedEngineException; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.threadpool.ThreadPool; @@ -200,159 +198,57 @@ public class IndexingMemoryController extends AbstractLifecycleComponent availableShards() { - ArrayList list = new ArrayList<>(); + protected List availableShards() { + List availableShards = new ArrayList<>(); for (IndexService indexService : indicesService) { - for (IndexShard indexShard : indexService) { - if (shardAvailable(indexShard)) { - list.add(indexShard.shardId()); + for (IndexShard shard : indexService) { + if (shardAvailable(shard)) { + availableShards.add(shard); } } } - return list; + return availableShards; } /** returns true if shard exists and is availabe for updates */ - protected boolean shardAvailable(ShardId shardId) { - return shardAvailable(getShard(shardId)); - } - - /** returns true if shard exists and is availabe for updates */ - protected boolean shardAvailable(@Nullable IndexShard shard) { + protected boolean shardAvailable(IndexShard shard) { // shadow replica doesn't have an indexing buffer - return shard != null && shard.canIndex() && CAN_UPDATE_INDEX_BUFFER_STATES.contains(shard.state()); - } - - /** gets an {@link IndexShard} instance for the given shard. returns null if the shard doesn't exist */ - protected IndexShard getShard(ShardId shardId) { - IndexService indexService = indicesService.indexService(shardId.index().name()); - if (indexService != null) { - IndexShard indexShard = indexService.getShardOrNull(shardId.id()); - return indexShard; - } - return null; + return shard.canIndex() && CAN_UPDATE_INDEX_BUFFER_STATES.contains(shard.state()); } /** set new indexing and translog buffers on this shard. this may cause the shard to refresh to free up heap. */ - protected void updateShardBuffers(ShardId shardId, ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { - final IndexShard shard = getShard(shardId); - if (shard != null) { - try { - shard.updateBufferSize(shardIndexingBufferSize, shardTranslogBufferSize); - } catch (EngineClosedException e) { - // ignore - } catch (FlushNotAllowedEngineException e) { - // ignore - } catch (Exception e) { - logger.warn("failed to set shard {} index buffer to [{}]", e, shardId, shardIndexingBufferSize); - } + protected void updateShardBuffers(IndexShard shard, ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { + try { + shard.updateBufferSize(shardIndexingBufferSize, shardTranslogBufferSize); + } catch (EngineClosedException | FlushNotAllowedEngineException e) { + // ignore + } catch (Exception e) { + logger.warn("failed to set shard {} index buffer to [{}]", e, shard.shardId(), shardIndexingBufferSize); } } - /** returns {@link IndexShard#getActive} if the shard exists, else null */ - protected Boolean getShardActive(ShardId shardId) { - final IndexShard indexShard = getShard(shardId); - if (indexShard == null) { - return null; - } - return indexShard.getActive(); - } - /** check if any shards active status changed, now. */ public void forceCheck() { statusChecker.run(); } class ShardsIndicesStatusChecker implements Runnable { - - // True if the shard was active last time we checked - private final Map shardWasActive = new HashMap<>(); - @Override public synchronized void run() { - EnumSet changes = purgeDeletedAndClosedShards(); - - updateShardStatuses(changes); - - if (changes.isEmpty() == false) { - // Something changed: recompute indexing buffers: - calcAndSetShardBuffers("[" + changes + "]"); - } - } - - /** - * goes through all existing shards and check whether there are changes in their active status - */ - private void updateShardStatuses(EnumSet changes) { - for (ShardId shardId : availableShards()) { - - // Is the shard active now? - Boolean isActive = getShardActive(shardId); - - if (isActive == null) { - // shard was closed.. - continue; - } - - // Was the shard active last time we checked? - Boolean wasActive = shardWasActive.get(shardId); - if (wasActive == null) { - // First time we are seeing this shard - shardWasActive.put(shardId, isActive); - changes.add(ShardStatusChangeType.ADDED); - } else if (isActive) { - // Shard is active now - if (wasActive == false) { - // Shard became active itself, since we last checked (due to new indexing op arriving) - changes.add(ShardStatusChangeType.BECAME_ACTIVE); - logger.debug("marking shard {} as active indexing wise", shardId); - shardWasActive.put(shardId, true); - } else if (checkIdle(shardId) == Boolean.TRUE) { - // Make shard inactive now - changes.add(ShardStatusChangeType.BECAME_INACTIVE); - - shardWasActive.put(shardId, false); - } - } - } - } - - /** - * purge any existing statuses that are no longer updated - * - * @return the changes applied - */ - private EnumSet purgeDeletedAndClosedShards() { - EnumSet changes = EnumSet.noneOf(ShardStatusChangeType.class); - - Iterator statusShardIdIterator = shardWasActive.keySet().iterator(); - while (statusShardIdIterator.hasNext()) { - ShardId shardId = statusShardIdIterator.next(); - if (shardAvailable(shardId) == false) { - changes.add(ShardStatusChangeType.DELETED); - statusShardIdIterator.remove(); - } - } - return changes; - } - - private void calcAndSetShardBuffers(String reason) { - - // Count how many shards are now active: - int activeShardCount = 0; - for (Map.Entry ent : shardWasActive.entrySet()) { - if (ent.getValue()) { - activeShardCount++; + List availableShards = availableShards(); + List activeShards = new ArrayList<>(); + for (IndexShard shard : availableShards) { + if (!checkIdle(shard)) { + activeShards.add(shard); } } + int activeShardCount = activeShards.size(); // TODO: we could be smarter here by taking into account how RAM the IndexWriter on each shard // is actually using (using IW.ramBytesUsed), so that small indices (e.g. Marvel) would not // get the same indexing buffer as large indices. But it quickly gets tricky... if (activeShardCount == 0) { - logger.debug("no active shards (reason={})", reason); return; } @@ -372,13 +268,10 @@ public class IndexingMemoryController extends AbstractLifecycleComponent ent : shardWasActive.entrySet()) { - if (ent.getValue()) { - // This shard is active - updateShardBuffers(ent.getKey(), shardIndexingBufferSize, shardTranslogBufferSize); - } + for (IndexShard shard : activeShards) { + updateShardBuffers(shard, shardIndexingBufferSize, shardTranslogBufferSize); } } } @@ -387,38 +280,17 @@ public class IndexingMemoryController extends AbstractLifecycleComponent MODULES = unmodifiableSet(newHashSet( + "lang-expression", + "lang-groovy")); static final Set OFFICIAL_PLUGINS = unmodifiableSet(newHashSet( "analysis-icu", @@ -78,8 +82,6 @@ public class PluginManager { "discovery-ec2", "discovery-gce", "discovery-multicast", - "lang-expression", - "lang-groovy", "lang-javascript", "lang-python", "mapper-attachments", @@ -221,6 +223,12 @@ public class PluginManager { PluginInfo info = PluginInfo.readFromProperties(root); terminal.println(VERBOSE, "%s", info); + // don't let luser install plugin as a module... + // they might be unavoidably in maven central and are packaged up the same way) + if (MODULES.contains(info.getName())) { + throw new IOException("plugin '" + info.getName() + "' cannot be installed like this, it is a system module"); + } + // update name in handle based on 'name' property found in descriptor file pluginHandle = new PluginHandle(info.getName(), pluginHandle.version, pluginHandle.user); final Path extractLocation = pluginHandle.extractedDir(environment); diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 1db54458cef..4cd5f114616 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -25,9 +25,8 @@ import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.bootstrap.JarHell; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -39,10 +38,7 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexEventListener; -import java.io.Closeable; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -50,6 +46,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.nio.file.DirectoryStream; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -69,10 +66,10 @@ import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; public class PluginsService extends AbstractComponent { /** - * We keep around a list of plugins + * We keep around a list of plugins and modules */ private final List> plugins; - private final PluginsInfo info; + private final PluginsAndModules info; private final Map> onModuleReferences; @@ -89,13 +86,15 @@ public class PluginsService extends AbstractComponent { /** * Constructs a new PluginService * @param settings The settings of the system + * @param modulesDirectory The directory modules exist in, or null if modules should not be loaded from the filesystem * @param pluginsDirectory The directory plugins exist in, or null if plugins should not be loaded from the filesystem * @param classpathPlugins Plugins that exist in the classpath which should be loaded */ - public PluginsService(Settings settings, Path pluginsDirectory, Collection> classpathPlugins) { + public PluginsService(Settings settings, Path modulesDirectory, Path pluginsDirectory, Collection> classpathPlugins) { super(settings); + info = new PluginsAndModules(); - List> tupleBuilder = new ArrayList<>(); + List> pluginsLoaded = new ArrayList<>(); // first we load plugins that are on the classpath. this is for tests and transport clients for (Class pluginClass : classpathPlugins) { @@ -104,24 +103,39 @@ public class PluginsService extends AbstractComponent { if (logger.isTraceEnabled()) { logger.trace("plugin loaded from classpath [{}]", pluginInfo); } - tupleBuilder.add(new Tuple<>(pluginInfo, plugin)); + pluginsLoaded.add(new Tuple<>(pluginInfo, plugin)); + info.addPlugin(pluginInfo); + } + + // load modules + if (modulesDirectory != null) { + try { + List bundles = getModuleBundles(modulesDirectory); + List> loaded = loadBundles(bundles); + pluginsLoaded.addAll(loaded); + for (Tuple module : loaded) { + info.addModule(module.v1()); + } + } catch (IOException ex) { + throw new IllegalStateException("Unable to initialize modules", ex); + } } // now, find all the ones that are in plugins/ if (pluginsDirectory != null) { try { List bundles = getPluginBundles(pluginsDirectory); - tupleBuilder.addAll(loadBundles(bundles)); + List> loaded = loadBundles(bundles); + pluginsLoaded.addAll(loaded); + for (Tuple plugin : loaded) { + info.addPlugin(plugin.v1()); + } } catch (IOException ex) { throw new IllegalStateException("Unable to initialize plugins", ex); } } - plugins = Collections.unmodifiableList(tupleBuilder); - info = new PluginsInfo(); - for (Tuple tuple : plugins) { - info.add(tuple.v1()); - } + plugins = Collections.unmodifiableList(pluginsLoaded); // We need to build a List of jvm and site plugins for checking mandatory plugins Map jvmPlugins = new HashMap<>(); @@ -151,7 +165,18 @@ public class PluginsService extends AbstractComponent { } } - logger.info("loaded {}, sites {}", jvmPlugins.keySet(), sitePlugins); + // we don't log jars in lib/ we really shouldnt log modules, + // but for now: just be transparent so we can debug any potential issues + Set moduleNames = new HashSet<>(); + Set jvmPluginNames = new HashSet<>(); + for (PluginInfo moduleInfo : info.getModuleInfos()) { + moduleNames.add(moduleInfo.getName()); + } + for (PluginInfo pluginInfo : info.getPluginInfos()) { + jvmPluginNames.add(pluginInfo.getName()); + } + + logger.info("modules {}, plugins {}, sites {}", moduleNames, jvmPluginNames, sitePlugins); Map> onModuleReferences = new HashMap<>(); for (Plugin plugin : jvmPlugins.values()) { @@ -160,6 +185,10 @@ public class PluginsService extends AbstractComponent { if (!method.getName().equals("onModule")) { continue; } + // this is a deprecated final method, so all Plugin subclasses have it + if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0].equals(IndexModule.class)) { + continue; + } if (method.getParameterTypes().length == 0 || method.getParameterTypes().length > 1) { logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", plugin.name()); continue; @@ -178,7 +207,7 @@ public class PluginsService extends AbstractComponent { this.onModuleReferences = Collections.unmodifiableMap(onModuleReferences); } - public List> plugins() { + private List> plugins() { return plugins; } @@ -249,12 +278,12 @@ public class PluginsService extends AbstractComponent { } } /** - * Get information about plugins (jvm and site plugins). + * Get information about plugins and modules */ - public PluginsInfo info() { + public PluginsAndModules info() { return info; } - + // a "bundle" is a group of plugins in a single classloader // really should be 1-1, but we are not so fortunate static class Bundle { @@ -262,6 +291,40 @@ public class PluginsService extends AbstractComponent { List urls = new ArrayList<>(); } + // similar in impl to getPluginBundles, but DO NOT try to make them share code. + // we don't need to inherit all the leniency, and things are different enough. + static List getModuleBundles(Path modulesDirectory) throws IOException { + // damn leniency + if (Files.notExists(modulesDirectory)) { + return Collections.emptyList(); + } + List bundles = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(modulesDirectory)) { + for (Path module : stream) { + if (FileSystemUtils.isHidden(module)) { + continue; // skip over .DS_Store etc + } + PluginInfo info = PluginInfo.readFromProperties(module); + if (!info.isJvm()) { + throw new IllegalStateException("modules must be jvm plugins: " + info); + } + if (!info.isIsolated()) { + throw new IllegalStateException("modules must be isolated: " + info); + } + Bundle bundle = new Bundle(); + bundle.plugins.add(info); + // gather urls for jar files + try (DirectoryStream jarStream = Files.newDirectoryStream(module, "*.jar")) { + for (Path jar : jarStream) { + bundle.urls.add(jar.toUri().toURL()); + } + } + bundles.add(bundle); + } + } + return bundles; + } + static List getPluginBundles(Path pluginsDirectory) throws IOException { ESLogger logger = Loggers.getLogger(PluginsService.class); @@ -269,7 +332,7 @@ public class PluginsService extends AbstractComponent { if (!isAccessibleDirectory(pluginsDirectory, logger)) { return Collections.emptyList(); } - + List bundles = new ArrayList<>(); // a special purgatory for plugins that directly depend on each other bundles.add(new Bundle()); @@ -281,7 +344,14 @@ public class PluginsService extends AbstractComponent { continue; } logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); - PluginInfo info = PluginInfo.readFromProperties(plugin); + final PluginInfo info; + try { + info = PluginInfo.readFromProperties(plugin); + } catch (IOException e) { + throw new IllegalStateException("Could not load plugin descriptor for existing plugin [" + + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); + } + List urls = new ArrayList<>(); if (info.isJvm()) { // a jvm plugin: gather urls for jar files @@ -302,7 +372,7 @@ public class PluginsService extends AbstractComponent { bundle.urls.addAll(urls); } } - + return bundles; } @@ -320,7 +390,7 @@ public class PluginsService extends AbstractComponent { } catch (Exception e) { throw new IllegalStateException("failed to load bundle " + bundle.urls + " due to jar hell", e); } - + // create a child to load the plugins in this bundle ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), getClass().getClassLoader()); for (PluginInfo pluginInfo : bundle.plugins) { diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java index 4bec04845d0..730276c1a2b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java @@ -45,9 +45,6 @@ public class RestForceMergeAction extends BaseRestHandler { super(settings, controller, client); controller.registerHandler(POST, "/_forcemerge", this); controller.registerHandler(POST, "/{index}/_forcemerge", this); - - controller.registerHandler(GET, "/_forcemerge", this); - controller.registerHandler(GET, "/{index}/_forcemerge", this); } @Override diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java index 058a93bf6b9..b52f8e6fc10 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java @@ -95,7 +95,7 @@ public class RestPluginsAction extends AbstractCatAction { for (DiscoveryNode node : nodes) { NodeInfo info = nodesInfo.getNodesMap().get(node.id()); - for (PluginInfo pluginInfo : info.getPlugins().getInfos()) { + for (PluginInfo pluginInfo : info.getPlugins().getPluginInfos()) { table.startRow(); table.addCell(node.id()); table.addCell(node.name()); diff --git a/core/src/main/java/org/elasticsearch/script/ClassPermission.java b/core/src/main/java/org/elasticsearch/script/ClassPermission.java new file mode 100644 index 00000000000..eb580bac3ea --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/ClassPermission.java @@ -0,0 +1,171 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import java.security.BasicPermission; +import java.security.Permission; +import java.security.PermissionCollection; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Set; + +/** + * Checked by scripting engines to allow loading a java class. + *

+ * Examples: + *

+ * Allow permission to {@code java.util.List} + * 

permission org.elasticsearch.script.ClassPermission "java.util.List";
+ * Allow permission to classes underneath {@code java.util} (and its subpackages such as {@code java.util.zip}) + * 
permission org.elasticsearch.script.ClassPermission "java.util.*";
+ * Allow permission to standard predefined list of basic classes (see list below) + * 
permission org.elasticsearch.script.ClassPermission "<<STANDARD>>";
+ * Allow permission to all classes + * 
permission org.elasticsearch.script.ClassPermission "*";
+ *

+ * Set of classes (allowed by special value <<STANDARD>>): + *

    + *
  • {@link java.lang.Boolean}
    • + *
  • {@link java.lang.Byte}
    • + *
  • {@link java.lang.Character}
    • + *
  • {@link java.lang.Double}
    • + *
  • {@link java.lang.Integer}
    • + *
  • {@link java.lang.Long}
    • + *
  • {@link java.lang.Math}
    • + *
  • {@link java.lang.Object}
    • + *
  • {@link java.lang.Short}
    • + *
  • {@link java.lang.String}
    • + *
  • {@link java.math.BigDecimal}
    • + *
  • {@link java.util.ArrayList}
    • + *
  • {@link java.util.Arrays}
    • + *
  • {@link java.util.Date}
    • + *
  • {@link java.util.HashMap}
    • + *
  • {@link java.util.HashSet}
    • + *
  • {@link java.util.Iterator}
    • + *
  • {@link java.util.List}
    • + *
  • {@link java.util.Map}
    • + *
  • {@link java.util.Set}
    • + *
  • {@link java.util.UUID}
    • + *
  • {@link org.joda.time.DateTime}
    • + *
  • {@link org.joda.time.DateTimeUtils}
    • + *
  • {@link org.joda.time.DateTimeZone}
    • + *
  • {@link org.joda.time.Instant}
    • + *
+ */ +public final class ClassPermission extends BasicPermission { + private static final long serialVersionUID = 3530711429252193884L; + + public static final String STANDARD = "<>"; + /** Typical set of classes for scripting: basic data types, math, dates, and simple collections */ + // this is the list from the old grovy sandbox impl (+ some things like String, Iterator, etc that were missing) + public static final Set STANDARD_CLASSES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + // jdk classes + java.lang.Boolean.class.getName(), + java.lang.Byte.class.getName(), + java.lang.Character.class.getName(), + java.lang.Double.class.getName(), + java.lang.Integer.class.getName(), + java.lang.Long.class.getName(), + java.lang.Math.class.getName(), + java.lang.Object.class.getName(), + java.lang.Short.class.getName(), + java.lang.String.class.getName(), + java.math.BigDecimal.class.getName(), + java.util.ArrayList.class.getName(), + java.util.Arrays.class.getName(), + java.util.Date.class.getName(), + java.util.HashMap.class.getName(), + java.util.HashSet.class.getName(), + java.util.Iterator.class.getName(), + java.util.List.class.getName(), + java.util.Map.class.getName(), + java.util.Set.class.getName(), + java.util.UUID.class.getName(), + // joda-time + org.joda.time.DateTime.class.getName(), + org.joda.time.DateTimeUtils.class.getName(), + org.joda.time.DateTimeZone.class.getName(), + org.joda.time.Instant.class.getName() + ))); + + /** + * Creates a new ClassPermission object. + * + * @param name class to grant permission to + */ + public ClassPermission(String name) { + super(name); + } + + /** + * Creates a new ClassPermission object. + * This constructor exists for use by the {@code Policy} object to instantiate new Permission objects. + * + * @param name class to grant permission to + * @param actions ignored + */ + public ClassPermission(String name, String actions) { + this(name); + } + + @Override + public boolean implies(Permission p) { + // check for a special value of STANDARD to imply the basic set + if (p != null && p.getClass() == getClass()) { + ClassPermission other = (ClassPermission) p; + if (STANDARD.equals(getName()) && STANDARD_CLASSES.contains(other.getName())) { + return true; + } + } + return super.implies(p); + } + + @Override + public PermissionCollection newPermissionCollection() { + // BasicPermissionCollection only handles wildcards, we expand <> here + PermissionCollection impl = super.newPermissionCollection(); + return new PermissionCollection() { + private static final long serialVersionUID = 6792220143549780002L; + + @Override + public void add(Permission permission) { + if (permission instanceof ClassPermission && STANDARD.equals(permission.getName())) { + for (String clazz : STANDARD_CLASSES) { + impl.add(new ClassPermission(clazz)); + } + } else { + impl.add(permission); + } + } + + @Override + public boolean implies(Permission permission) { + return impl.implies(permission); + } + + @Override + public Enumeration elements() { + return impl.elements(); + } + }; + } +} diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index b620a32fd0f..7e060cd6d90 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -84,9 +84,6 @@ grant { // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; - // needed by lucene SPI currently - permission java.lang.RuntimePermission "getClassLoader"; - // needed by Settings permission java.lang.RuntimePermission "getenv.*"; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy b/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy index d32ea6a2435..8e7ca8d8b6e 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/untrusted.policy @@ -34,5 +34,6 @@ grant { permission java.util.PropertyPermission "rhino.stack.style", "read"; // needed IndyInterface selectMethod (setCallSiteTarget) + // TODO: clean this up / only give it to engines that really must have it permission java.lang.RuntimePermission "getClassLoader"; }; diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help index 389f6d9caa1..e6622e2743a 100644 --- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help +++ b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help @@ -43,8 +43,6 @@ OFFICIAL PLUGINS - discovery-ec2 - discovery-gce - discovery-multicast - - lang-expression - - lang-groovy - lang-javascript - lang-python - mapper-attachments diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java index 947ac475d61..9e842a38722 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java @@ -766,11 +766,11 @@ public class ClusterServiceIT extends ESIntegTestCase { return false; } } - int numberOfThreads = randomIntBetween(2, 256); + int numberOfThreads = randomIntBetween(2, 8); int tasksSubmittedPerThread = randomIntBetween(1, 1024); ConcurrentMap counters = new ConcurrentHashMap<>(); - CountDownLatch latch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread); + CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread); ClusterStateTaskListener listener = new ClusterStateTaskListener() { @Override public void onFailure(String source, Throwable t) { @@ -780,7 +780,7 @@ public class ClusterServiceIT extends ESIntegTestCase { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { counters.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet(); - latch.countDown(); + updateLatch.countDown(); } }; @@ -814,7 +814,7 @@ public class ClusterServiceIT extends ESIntegTestCase { clusterService.submitStateUpdateTask( Thread.currentThread().getName(), new Task(), - ClusterStateTaskConfig.build(Priority.NORMAL), + ClusterStateTaskConfig.build(randomFrom(Priority.values())), executor, listener); } @@ -829,14 +829,16 @@ public class ClusterServiceIT extends ESIntegTestCase { } // wait until all the cluster state updates have been processed - latch.await(); + updateLatch.await(); // assert the number of executed tasks is correct assertEquals(numberOfThreads * tasksSubmittedPerThread, counter.get()); // assert each executor executed the correct number of tasks for (TaskExecutor executor : executors) { - assertEquals((int)counts.get(executor), executor.counter.get()); + if (counts.containsKey(executor)) { + assertEquals((int) counts.get(executor), executor.counter.get()); + } } // assert the correct number of clusterStateProcessed events were triggered diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 8d6a0800461..5a31618f14e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -37,6 +37,8 @@ public class FieldTypeLookupTests extends ESTestCase { FieldTypeLookup lookup = new FieldTypeLookup(); assertNull(lookup.get("foo")); assertNull(lookup.getByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); assertTrue(names.isEmpty()); @@ -70,6 +72,14 @@ public class FieldTypeLookupTests extends ESTestCase { assertNull(lookup.get("bar")); assertEquals(f.fieldType(), lookup2.getByIndexName("bar")); assertNull(lookup.getByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup.getTypes("bar")); + assertEquals(Collections.emptySet(), lookup.getTypesByIndexName("bar")); + assertEquals(Collections.singleton("type"), lookup2.getTypes("foo")); + assertEquals(Collections.emptySet(), lookup2.getTypesByIndexName("foo")); + assertEquals(Collections.emptySet(), lookup2.getTypes("bar")); + assertEquals(Collections.singleton("type"), lookup2.getTypesByIndexName("bar")); assertEquals(1, size(lookup2.iterator())); } @@ -144,7 +154,7 @@ public class FieldTypeLookupTests extends ESTestCase { public void testCheckCompatibilityNewField() { FakeFieldMapper f1 = new FakeFieldMapper("foo", "bar"); FieldTypeLookup lookup = new FieldTypeLookup(); - lookup.checkCompatibility(newList(f1), false); + lookup.checkCompatibility("type", newList(f1), false); } public void testCheckCompatibilityMismatchedTypes() { @@ -155,14 +165,14 @@ public class FieldTypeLookupTests extends ESTestCase { MappedFieldType ft2 = FakeFieldMapper.makeOtherFieldType("foo", "foo"); FieldMapper f2 = new FakeFieldMapper("foo", ft2); try { - lookup.checkCompatibility(newList(f2), false); + lookup.checkCompatibility("type2", newList(f2), false); fail("expected type mismatch"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("cannot be changed from type [faketype] to [otherfaketype]")); } // fails even if updateAllTypes == true try { - lookup.checkCompatibility(newList(f2), true); + lookup.checkCompatibility("type2", newList(f2), true); fail("expected type mismatch"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("cannot be changed from type [faketype] to [otherfaketype]")); @@ -178,25 +188,27 @@ public class FieldTypeLookupTests extends ESTestCase { ft2.setBoost(2.0f); FieldMapper f2 = new FakeFieldMapper("foo", ft2); try { - lookup.checkCompatibility(newList(f2), false); + // different type + lookup.checkCompatibility("type2", newList(f2), false); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("to update [boost] across all types")); } - lookup.checkCompatibility(newList(f2), true); // boost is updateable, so ok if forcing + lookup.checkCompatibility("type", newList(f2), false); // boost is updateable, so ok since we are implicitly updating all types + lookup.checkCompatibility("type2", newList(f2), true); // boost is updateable, so ok if forcing // now with a non changeable setting MappedFieldType ft3 = FakeFieldMapper.makeFieldType("foo", "bar"); ft3.setStored(true); FieldMapper f3 = new FakeFieldMapper("foo", ft3); try { - lookup.checkCompatibility(newList(f3), false); + lookup.checkCompatibility("type2", newList(f3), false); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("has different [store] values")); } // even with updateAllTypes == true, incompatible try { - lookup.checkCompatibility(newList(f3), true); + lookup.checkCompatibility("type2", newList(f3), true); fail("expected conflict"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("has different [store] values")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java index 1d6e72834c3..4a010747624 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -68,6 +69,25 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase { } + public void testDynamicObjectCopyTo() throws Exception { + String mapping = jsonBuilder().startObject().startObject("doc").startObject("properties") + .startObject("foo") + .field("type", "string") + .field("copy_to", "root.top.child") + .endObject() + .endObject().endObject().endObject().string(); + assertAcked( + client().admin().indices().prepareCreate("test-idx") + .addMapping("doc", mapping) + ); + client().prepareIndex("test-idx", "doc", "1") + .setSource("foo", "bar") + .get(); + client().admin().indices().prepareRefresh("test-idx").execute().actionGet(); + SearchResponse response = client().prepareSearch("test-idx") + .setQuery(QueryBuilders.termQuery("root.top.child", "bar")).get(); + assertThat(response.getHits().totalHits(), equalTo(1L)); + } private XContentBuilder createDynamicTemplateMapping() throws IOException { return XContentFactory.jsonBuilder().startObject().startObject("doc") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java index 301d6b13e3b..d94ae2b6735 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperTests.java @@ -167,27 +167,126 @@ public class CopyToMapperTests extends ESSingleNodeTestCase { } - public void testCopyToFieldsNonExistingInnerObjectParsing() throws Exception { - String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties") - + public void testCopyToDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") .startObject("copy_test") - .field("type", "string") - .field("copy_to", "very.inner.field") + .field("type", "string") + .field("copy_to", "very.inner.field") .endObject() - - .endObject().endObject().endObject().string(); + .endObject() + .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); BytesReference json = jsonBuilder().startObject() .field("copy_test", "foo") + .field("new_field", "bar") .endObject().bytes(); + ParseContext.Document doc = docMapper.parse("test", "type1", "1", json).rootDoc(); + assertThat(doc.getFields("copy_test").length, equalTo(1)); + assertThat(doc.getFields("copy_test")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("very.inner.field").length, equalTo(1)); + assertThat(doc.getFields("very.inner.field")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("new_field").length, equalTo(1)); + assertThat(doc.getFields("new_field")[0].stringValue(), equalTo("bar")); + } + + public void testCopyToDynamicInnerInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.far.inner.field") + .endObject() + .startObject("very") + .field("type", "object") + .startObject("properties") + .startObject("far") + .field("type", "object") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .field("new_field", "bar") + .endObject().bytes(); + + ParseContext.Document doc = docMapper.parse("test", "type1", "1", json).rootDoc(); + assertThat(doc.getFields("copy_test").length, equalTo(1)); + assertThat(doc.getFields("copy_test")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("very.far.inner.field").length, equalTo(1)); + assertThat(doc.getFields("very.far.inner.field")[0].stringValue(), equalTo("foo")); + + assertThat(doc.getFields("new_field").length, equalTo(1)); + assertThat(doc.getFields("new_field")[0].stringValue(), equalTo("bar")); + } + + public void testCopyToStrictDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .field("dynamic", "strict") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.inner.field") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .endObject().bytes(); + try { docMapper.parse("test", "type1", "1", json).rootDoc(); fail(); } catch (MapperParsingException ex) { - assertThat(ex.getMessage(), startsWith("attempt to copy value to non-existing object")); + assertThat(ex.getMessage(), startsWith("mapping set to strict, dynamic introduction of [very] within [type1] is not allowed")); + } + } + + public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.far.field") + .endObject() + .startObject("very") + .field("type", "object") + .startObject("properties") + .startObject("far") + .field("type", "object") + .field("dynamic", "strict") + .endObject() + .endObject() + .endObject() + + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .endObject().bytes(); + + try { + docMapper.parse("test", "type1", "1", json).rootDoc(); + fail(); + } catch (MapperParsingException ex) { + assertThat(ex.getMessage(), startsWith("mapping set to strict, dynamic introduction of [field] within [very.far] is not allowed")); } } @@ -337,6 +436,41 @@ public class CopyToMapperTests extends ESSingleNodeTestCase { } } + public void testCopyToDynamicNestedObjectParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startArray("dynamic_templates") + .startObject() + .startObject("objects") + .field("match_mapping_type", "object") + .startObject("mapping") + .field("type", "nested") + .endObject() + .endObject() + .endObject() + .endArray() + .startObject("properties") + .startObject("copy_test") + .field("type", "string") + .field("copy_to", "very.inner.field") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + BytesReference json = jsonBuilder().startObject() + .field("copy_test", "foo") + .field("new_field", "bar") + .endObject().bytes(); + + try { + docMapper.parse("test", "type1", "1", json).rootDoc(); + fail(); + } catch (MapperParsingException ex) { + assertThat(ex.getMessage(), startsWith("It is forbidden to create dynamic nested objects ([very]) through `copy_to`")); + } + } + private void assertFieldValue(Document doc, String field, Number... expected) { IndexableField[] values = doc.getFields(field); if (values == null) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java index 9741b82aad6..93fd71599c4 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java @@ -25,12 +25,14 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.search.SearchHitField; @@ -715,28 +717,25 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { String stage1Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) .field("geohash", true).endObject().endObject().endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test", settings).mapperService().documentMapperParser(); - DocumentMapper stage1 = parser.parse(stage1Mapping); + MapperService mapperService = createIndex("test", settings).mapperService(); + DocumentMapper stage1 = mapperService.merge("type", new CompressedXContent(stage1Mapping), true, false); String stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", false) .field("geohash", false).endObject().endObject().endObject().endObject().string(); - DocumentMapper stage2 = parser.parse(stage2Mapping); - - MergeResult mergeResult = stage1.merge(stage2.mapping(), false, false); - assertThat(mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts().length, equalTo(3)); - // todo better way of checking conflict? - assertThat("mapper [point] has different [lat_lon]", isIn(new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())))); - assertThat("mapper [point] has different [geohash]", isIn(new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())))); - assertThat("mapper [point] has different [geohash_precision]", isIn(new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())))); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [point] has different [lat_lon]")); + assertThat(e.getMessage(), containsString("mapper [point] has different [geohash]")); + assertThat(e.getMessage(), containsString("mapper [point] has different [geohash_precision]")); + } // correct mapping and ensure no failures stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("point").field("type", "geo_point").field("lat_lon", true) .field("geohash", true).endObject().endObject().endObject().endObject().string(); - stage2 = parser.parse(stage2Mapping); - mergeResult = stage1.merge(stage2.mapping(), false, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); } public void testGeoHashSearch() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java index c00bd3101ae..54e9e96f8ad 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapperTests.java @@ -22,12 +22,14 @@ import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -35,6 +37,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.isIn; @@ -376,23 +379,21 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { .startObject("shape").field("type", "geo_shape").field("tree", "geohash").field("strategy", "recursive") .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01).field("orientation", "ccw") .endObject().endObject().endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - DocumentMapper stage1 = parser.parse(stage1Mapping); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper stage1 = mapperService.merge("type", new CompressedXContent(stage1Mapping), true, false); String stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("shape").field("type", "geo_shape").field("tree", "quadtree") .field("strategy", "term").field("precision", "1km").field("tree_levels", 26).field("distance_error_pct", 26) .field("orientation", "cw").endObject().endObject().endObject().endObject().string(); - DocumentMapper stage2 = parser.parse(stage2Mapping); - - MergeResult mergeResult = stage1.merge(stage2.mapping(), false, false); - // check correct conflicts - assertThat(mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts().length, equalTo(4)); - ArrayList conflicts = new ArrayList<>(Arrays.asList(mergeResult.buildConflicts())); - assertThat("mapper [shape] has different [strategy]", isIn(conflicts)); - assertThat("mapper [shape] has different [tree]", isIn(conflicts)); - assertThat("mapper [shape] has different [tree_levels]", isIn(conflicts)); - assertThat("mapper [shape] has different [precision]", isIn(conflicts)); + try { + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [shape] has different [strategy]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [tree_levels]")); + assertThat(e.getMessage(), containsString("mapper [shape] has different [precision]")); + } // verify nothing changed FieldMapper fieldMapper = stage1.mappers().getMapper("shape"); @@ -411,11 +412,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { stage2Mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m") .field("tree_levels", 8).field("distance_error_pct", 0.001).field("orientation", "cw").endObject().endObject().endObject().endObject().string(); - stage2 = parser.parse(stage2Mapping); - mergeResult = stage1.merge(stage2.mapping(), false, false); - - // verify mapping changes, and ensure no failures - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); + mapperService.merge("type", new CompressedXContent(stage2Mapping), false, false); fieldMapper = stage1.mappers().getMapper("shape"); assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java index 07671a2d4b0..30890dcd22a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/merge/JavaMultiFieldMergeTests.java @@ -22,9 +22,11 @@ package org.elasticsearch.index.mapper.multifield.merge; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -32,6 +34,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.Arrays; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -113,9 +116,9 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { public void testUpgradeFromMultiFieldTypeToMultiFields() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping1.json"); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper docMapper = parser.parse(mapping); + DocumentMapper docMapper = mapperService.merge("person", new CompressedXContent(mapping), true, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); assertThat(docMapper.mappers().getMapper("name.indexed"), nullValue()); @@ -129,12 +132,7 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade1.json"); - DocumentMapper docMapper2 = parser.parse(mapping); - - MergeResult mergeResult = docMapper.merge(docMapper2.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); - - docMapper.merge(docMapper2.mapping(), false, false); + mapperService.merge("person", new CompressedXContent(mapping), false, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); @@ -151,12 +149,7 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { assertThat(f, notNullValue()); mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade2.json"); - DocumentMapper docMapper3 = parser.parse(mapping); - - mergeResult = docMapper.merge(docMapper3.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(false)); - - docMapper.merge(docMapper3.mapping(), false, false); + mapperService.merge("person", new CompressedXContent(mapping), false, false); assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); @@ -168,24 +161,19 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase { mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade3.json"); - DocumentMapper docMapper4 = parser.parse(mapping); - mergeResult = docMapper.merge(docMapper4.mapping(), true, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(true)); - assertThat(mergeResult.buildConflicts()[0], equalTo("mapper [name] has different [index] values")); - assertThat(mergeResult.buildConflicts()[1], equalTo("mapper [name] has different [store] values")); + try { + mapperService.merge("person", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [name] has different [index] values")); + assertThat(e.getMessage(), containsString("mapper [name] has different [store] values")); + } - mergeResult = docMapper.merge(docMapper4.mapping(), false, false); - assertThat(Arrays.toString(mergeResult.buildConflicts()), mergeResult.hasConflicts(), equalTo(true)); - - assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); - assertThat(mergeResult.buildConflicts()[0], equalTo("mapper [name] has different [index] values")); - assertThat(mergeResult.buildConflicts()[1], equalTo("mapper [name] has different [store] values")); - - // There are conflicts, but the `name.not_indexed3` has been added, b/c that field has no conflicts + // There are conflicts, so the `name.not_indexed3` has not been added assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions()); assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue()); assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue()); assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue()); - assertThat(docMapper.mappers().getMapper("name.not_indexed3"), notNullValue()); + assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java index 121f100ffa6..9ac039a49fb 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -478,7 +479,7 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { .startObject("properties").startObject("field").field("type", "string").endObject().endObject() .endObject().endObject().string(); - DocumentMapper defaultMapper = parser.parse(mapping); + DocumentMapper defaultMapper = indexService.mapperService().merge("type", new CompressedXContent(mapping), true, false); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() @@ -507,10 +508,12 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { updatedMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "string").startObject("norms").field("enabled", true).endObject() .endObject().endObject().endObject().endObject().string(); - mergeResult = defaultMapper.merge(parser.parse(updatedMapping).mapping(), true, false); - assertTrue(mergeResult.hasConflicts()); - assertEquals(1, mergeResult.buildConflicts().length); - assertTrue(mergeResult.buildConflicts()[0].contains("different [omit_norms]")); + try { + defaultMapper.merge(parser.parse(updatedMapping).mapping(), true, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("different [omit_norms]")); + } } /** diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index df9cc10d8c2..53a3bf7bb6e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -557,7 +558,6 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { public void testMergingConflicts() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true) - .startObject("fielddata").field("format", "doc_values").endObject() .field("store", "yes") .field("index", "analyzed") .field("path", "foo") @@ -565,9 +565,9 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { .endObject() .endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - DocumentMapperParser parser = createIndex("test", indexSettings).mapperService().documentMapperParser(); + MapperService mapperService = createIndex("test", indexSettings).mapperService(); - DocumentMapper docMapper = parser.parse(mapping); + DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(mapping), true, false); assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getLoading(), equalTo(MappedFieldType.Loading.LAZY)); mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", false) @@ -579,20 +579,32 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { .endObject() .endObject().endObject().string(); - MergeResult mergeResult = docMapper.merge(parser.parse(mapping).mapping(), true, false); - List expectedConflicts = new ArrayList<>(Arrays.asList( - "mapper [_timestamp] has different [index] values", - "mapper [_timestamp] has different [store] values", - "Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02", - "Cannot update path in _timestamp value. Value is foo path in merged mapping is bar")); - - for (String conflict : mergeResult.buildConflicts()) { - assertTrue("found unexpected conflict [" + conflict + "]", expectedConflicts.remove(conflict)); + try { + mapperService.merge("type", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("mapper [_timestamp] has different [index] values")); + assertThat(e.getMessage(), containsString("mapper [_timestamp] has different [store] values")); } - assertTrue("missing conflicts: " + Arrays.toString(expectedConflicts.toArray()), expectedConflicts.isEmpty()); + assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getLoading(), equalTo(MappedFieldType.Loading.LAZY)); assertTrue(docMapper.timestampFieldMapper().enabled()); - assertThat(docMapper.timestampFieldMapper().fieldType().fieldDataType().getFormat(indexSettings), equalTo("doc_values")); + + mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp").field("enabled", true) + .field("store", "yes") + .field("index", "analyzed") + .field("path", "bar") + .field("default", "1970-01-02") + .endObject() + .endObject().endObject().string(); + try { + mapperService.merge("type", new CompressedXContent(mapping), false, false); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02")); + assertThat(e.getMessage(), containsString("Cannot update path in _timestamp value. Value is foo path in merged mapping is bar")); + } } public void testBackcompatMergingConflictsForIndexValues() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java index 1edc2bb131a..35034dfd911 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java @@ -48,7 +48,7 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { public void testAllConflicts() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_create_index.json"); String mappingUpdate = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json"); - String[] errorMessage = {"[_all] enabled is true now encountering false", + String[] errorMessage = { "[_all] has different [omit_norms] values", "[_all] has different [store] values", "[_all] has different [store_term_vector] values", @@ -61,6 +61,13 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase { testConflict(mapping, mappingUpdate, errorMessage); } + public void testAllDisabled() throws Exception { + XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", true).endObject().endObject().endObject().endObject(); + XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "string").endObject().endObject().endObject(); + String errorMessage = "[_all] enabled is true now encountering false"; + testConflict(mapping.string(), mappingUpdate.string(), errorMessage); + } + public void testAllWithDefault() throws Exception { String defaultMapping = jsonBuilder().startObject().startObject("_default_") .startObject("_all") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index b44081fc910..f0a8f5d079d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -123,14 +123,14 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { mapperService.merge("type", new CompressedXContent(update.string()), false, false); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("Merge failed")); + assertThat(e.getMessage(), containsString("mapper [foo] cannot be changed from type [long] to [double]")); } try { mapperService.merge("type", new CompressedXContent(update.string()), false, false); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("Merge failed")); + assertThat(e.getMessage(), containsString("mapper [foo] cannot be changed from type [long] to [double]")); } assertTrue(mapperService.documentMapper("type").mapping().root().getMapper("foo") instanceof LongFieldMapper); @@ -167,7 +167,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase { } // same as the testConflictNewType except that the mapping update is on an existing type - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/15049") public void testConflictNewTypeUpdate() throws Exception { XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("foo").field("type", "long").endObject() diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 57ba469a357..ed4b95c03d8 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -140,7 +140,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"integer\"}}}}").execute().actionGet(); fail("Expected MergeMappingException"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("mapper [body] of different type")); + assertThat(e.getMessage(), containsString("mapper [body] cannot be changed from type [string] to [int]")); } } diff --git a/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java index 4dddc4b788f..6d8dda3afb9 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java @@ -22,54 +22,51 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESSingleNodeTestCase; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -public class IndexingMemoryControllerTests extends ESTestCase { +public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { static class MockController extends IndexingMemoryController { final static ByteSizeValue INACTIVE = new ByteSizeValue(-1); - final Map indexingBuffers = new HashMap<>(); - final Map translogBuffers = new HashMap<>(); + final Map indexingBuffers = new HashMap<>(); + final Map translogBuffers = new HashMap<>(); - final Map lastIndexTimeNanos = new HashMap<>(); - final Set activeShards = new HashSet<>(); + final Map lastIndexTimeNanos = new HashMap<>(); + final Set activeShards = new HashSet<>(); long currentTimeSec = TimeValue.timeValueNanos(System.nanoTime()).seconds(); public MockController(Settings settings) { super(Settings.builder() - .put(SHARD_INACTIVE_INTERVAL_TIME_SETTING, "200h") // disable it - .put(IndexShard.INDEX_SHARD_INACTIVE_TIME_SETTING, "1ms") // nearly immediate - .put(settings) - .build(), - null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb + .put(SHARD_INACTIVE_INTERVAL_TIME_SETTING, "200h") // disable it + .put(IndexShard.INDEX_SHARD_INACTIVE_TIME_SETTING, "1ms") // nearly immediate + .put(settings) + .build(), + null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb } - public void deleteShard(ShardId id) { + public void deleteShard(IndexShard id) { indexingBuffers.remove(id); translogBuffers.remove(id); } - public void assertBuffers(ShardId id, ByteSizeValue indexing, ByteSizeValue translog) { + public void assertBuffers(IndexShard id, ByteSizeValue indexing, ByteSizeValue translog) { assertThat(indexingBuffers.get(id), equalTo(indexing)); assertThat(translogBuffers.get(id), equalTo(translog)); } - public void assertInActive(ShardId id) { + public void assertInactive(IndexShard id) { assertThat(indexingBuffers.get(id), equalTo(INACTIVE)); assertThat(translogBuffers.get(id), equalTo(INACTIVE)); } @@ -80,36 +77,31 @@ public class IndexingMemoryControllerTests extends ESTestCase { } @Override - protected List availableShards() { + protected List availableShards() { return new ArrayList<>(indexingBuffers.keySet()); } @Override - protected boolean shardAvailable(ShardId shardId) { - return indexingBuffers.containsKey(shardId); + protected boolean shardAvailable(IndexShard shard) { + return indexingBuffers.containsKey(shard); } @Override - protected Boolean getShardActive(ShardId shardId) { - return activeShards.contains(shardId); + protected void updateShardBuffers(IndexShard shard, ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { + indexingBuffers.put(shard, shardIndexingBufferSize); + translogBuffers.put(shard, shardTranslogBufferSize); } @Override - protected void updateShardBuffers(ShardId shardId, ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { - indexingBuffers.put(shardId, shardIndexingBufferSize); - translogBuffers.put(shardId, shardTranslogBufferSize); - } - - @Override - protected Boolean checkIdle(ShardId shardId) { + protected boolean checkIdle(IndexShard shard) { final TimeValue inactiveTime = settings.getAsTime(IndexShard.INDEX_SHARD_INACTIVE_TIME_SETTING, TimeValue.timeValueMinutes(5)); - Long ns = lastIndexTimeNanos.get(shardId); + Long ns = lastIndexTimeNanos.get(shard); if (ns == null) { - return null; + return true; } else if (currentTimeInNanos() - ns >= inactiveTime.nanos()) { - indexingBuffers.put(shardId, INACTIVE); - translogBuffers.put(shardId, INACTIVE); - activeShards.remove(shardId); + indexingBuffers.put(shard, INACTIVE); + translogBuffers.put(shard, INACTIVE); + activeShards.remove(shard); return true; } else { return false; @@ -120,118 +112,126 @@ public class IndexingMemoryControllerTests extends ESTestCase { currentTimeSec += sec; } - public void simulateIndexing(ShardId shardId) { - lastIndexTimeNanos.put(shardId, currentTimeInNanos()); - if (indexingBuffers.containsKey(shardId) == false) { + public void simulateIndexing(IndexShard shard) { + lastIndexTimeNanos.put(shard, currentTimeInNanos()); + if (indexingBuffers.containsKey(shard) == false) { // First time we are seeing this shard; start it off with inactive buffers as IndexShard does: - indexingBuffers.put(shardId, IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER); - translogBuffers.put(shardId, IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER); + indexingBuffers.put(shard, IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER); + translogBuffers.put(shard, IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER); } - activeShards.add(shardId); + activeShards.add(shard); forceCheck(); } } public void testShardAdditionAndRemoval() { + createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "100kb").build()); - final ShardId shard1 = new ShardId("test", 1); - controller.simulateIndexing(shard1); - controller.assertBuffers(shard1, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "100kb").build()); + IndexShard shard0 = test.getShard(0); + controller.simulateIndexing(shard0); + controller.assertBuffers(shard0, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K // add another shard - final ShardId shard2 = new ShardId("test", 2); - controller.simulateIndexing(shard2); + IndexShard shard1 = test.getShard(1); + controller.simulateIndexing(shard1); + controller.assertBuffers(shard0, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); controller.assertBuffers(shard1, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); - controller.assertBuffers(shard2, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); // remove first shard - controller.deleteShard(shard1); + controller.deleteShard(shard0); controller.forceCheck(); - controller.assertBuffers(shard2, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K + controller.assertBuffers(shard1, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K // remove second shard - controller.deleteShard(shard2); + controller.deleteShard(shard1); controller.forceCheck(); // add a new one - final ShardId shard3 = new ShardId("test", 3); - controller.simulateIndexing(shard3); - controller.assertBuffers(shard3, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K + IndexShard shard2 = test.getShard(2); + controller.simulateIndexing(shard2); + controller.assertBuffers(shard2, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); // translog is maxed at 64K } public void testActiveInactive() { - MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "100kb") - .put(IndexShard.INDEX_SHARD_INACTIVE_TIME_SETTING, "5s") - .build()); + createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); - final ShardId shard1 = new ShardId("test", 1); + MockController controller = new MockController(Settings.builder() + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "100kb") + .put(IndexShard.INDEX_SHARD_INACTIVE_TIME_SETTING, "5s") + .build()); + + IndexShard shard0 = test.getShard(0); + controller.simulateIndexing(shard0); + IndexShard shard1 = test.getShard(1); controller.simulateIndexing(shard1); - final ShardId shard2 = new ShardId("test", 2); - controller.simulateIndexing(shard2); + controller.assertBuffers(shard0, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); controller.assertBuffers(shard1, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); - controller.assertBuffers(shard2, new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(50, ByteSizeUnit.KB)); // index into both shards, move the clock and see that they are still active + controller.simulateIndexing(shard0); controller.simulateIndexing(shard1); - controller.simulateIndexing(shard2); controller.incrementTimeSec(10); controller.forceCheck(); // both shards now inactive - controller.assertInActive(shard1); - controller.assertInActive(shard2); + controller.assertInactive(shard0); + controller.assertInactive(shard1); // index into one shard only, see it becomes active - controller.simulateIndexing(shard1); - controller.assertBuffers(shard1, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); - controller.assertInActive(shard2); + controller.simulateIndexing(shard0); + controller.assertBuffers(shard0, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); + controller.assertInactive(shard1); controller.incrementTimeSec(3); // increment but not enough to become inactive controller.forceCheck(); - controller.assertBuffers(shard1, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); - controller.assertInActive(shard2); + controller.assertBuffers(shard0, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); + controller.assertInactive(shard1); controller.incrementTimeSec(3); // increment some more controller.forceCheck(); - controller.assertInActive(shard1); - controller.assertInActive(shard2); + controller.assertInactive(shard0); + controller.assertInactive(shard1); // index some and shard becomes immediately active - controller.simulateIndexing(shard2); - controller.assertInActive(shard1); - controller.assertBuffers(shard2, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); + controller.simulateIndexing(shard1); + controller.assertInactive(shard0); + controller.assertBuffers(shard1, new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(64, ByteSizeUnit.KB)); } public void testMinShardBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "50kb") - .put(IndexingMemoryController.MIN_SHARD_INDEX_BUFFER_SIZE_SETTING, "6mb") - .put(IndexingMemoryController.MIN_SHARD_TRANSLOG_BUFFER_SIZE_SETTING, "40kb").build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "50kb") + .put(IndexingMemoryController.MIN_SHARD_INDEX_BUFFER_SIZE_SETTING, "6mb") + .put(IndexingMemoryController.MIN_SHARD_TRANSLOG_BUFFER_SIZE_SETTING, "40kb").build()); assertTwoActiveShards(controller, new ByteSizeValue(6, ByteSizeUnit.MB), new ByteSizeValue(40, ByteSizeUnit.KB)); } public void testMaxShardBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "50kb") - .put(IndexingMemoryController.MAX_SHARD_INDEX_BUFFER_SIZE_SETTING, "3mb") - .put(IndexingMemoryController.MAX_SHARD_TRANSLOG_BUFFER_SIZE_SETTING, "10kb").build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "10mb") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "50kb") + .put(IndexingMemoryController.MAX_SHARD_INDEX_BUFFER_SIZE_SETTING, "3mb") + .put(IndexingMemoryController.MAX_SHARD_TRANSLOG_BUFFER_SIZE_SETTING, "10kb").build()); assertTwoActiveShards(controller, new ByteSizeValue(3, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.KB)); } public void testRelativeBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "50%") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "0.5%") - .build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "50%") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "0.5%") + .build()); assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(50, ByteSizeUnit.MB))); assertThat(controller.translogBufferSize(), equalTo(new ByteSizeValue(512, ByteSizeUnit.KB))); @@ -240,10 +240,10 @@ public class IndexingMemoryControllerTests extends ESTestCase { public void testMinBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "0.001%") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "0.001%") - .put(IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING, "6mb") - .put(IndexingMemoryController.MIN_TRANSLOG_BUFFER_SIZE_SETTING, "512kb").build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "0.001%") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "0.001%") + .put(IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING, "6mb") + .put(IndexingMemoryController.MIN_TRANSLOG_BUFFER_SIZE_SETTING, "512kb").build()); assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB))); assertThat(controller.translogBufferSize(), equalTo(new ByteSizeValue(512, ByteSizeUnit.KB))); @@ -251,23 +251,24 @@ public class IndexingMemoryControllerTests extends ESTestCase { public void testMaxBufferSizes() { MockController controller = new MockController(Settings.builder() - .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "90%") - .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "90%") - .put(IndexingMemoryController.MAX_INDEX_BUFFER_SIZE_SETTING, "6mb") - .put(IndexingMemoryController.MAX_TRANSLOG_BUFFER_SIZE_SETTING, "512kb").build()); + .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "90%") + .put(IndexingMemoryController.TRANSLOG_BUFFER_SIZE_SETTING, "90%") + .put(IndexingMemoryController.MAX_INDEX_BUFFER_SIZE_SETTING, "6mb") + .put(IndexingMemoryController.MAX_TRANSLOG_BUFFER_SIZE_SETTING, "512kb").build()); assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB))); assertThat(controller.translogBufferSize(), equalTo(new ByteSizeValue(512, ByteSizeUnit.KB))); } protected void assertTwoActiveShards(MockController controller, ByteSizeValue indexBufferSize, ByteSizeValue translogBufferSize) { - final ShardId shard1 = new ShardId("test", 1); + createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + IndexShard shard0 = test.getShard(0); + controller.simulateIndexing(shard0); + IndexShard shard1 = test.getShard(1); controller.simulateIndexing(shard1); - final ShardId shard2 = new ShardId("test", 2); - controller.simulateIndexing(shard2); + controller.assertBuffers(shard0, indexBufferSize, translogBufferSize); controller.assertBuffers(shard1, indexBufferSize, translogBufferSize); - controller.assertBuffers(shard2, indexBufferSize, translogBufferSize); - } - } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java index b236f9e2795..000365f6a20 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.plugins; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -259,14 +259,14 @@ public class PluginInfoTests extends ESTestCase { } public void testPluginListSorted() { - PluginsInfo pluginsInfo = new PluginsInfo(5); - pluginsInfo.add(new PluginInfo("c", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("b", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("e", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("a", "foo", true, "dummy", true, "dummyclass", true)); - pluginsInfo.add(new PluginInfo("d", "foo", true, "dummy", true, "dummyclass", true)); + PluginsAndModules pluginsInfo = new PluginsAndModules(); + pluginsInfo.addPlugin(new PluginInfo("c", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("b", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("e", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("a", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("d", "foo", true, "dummy", true, "dummyclass", true)); - final List infos = pluginsInfo.getInfos(); + final List infos = pluginsInfo.getPluginInfos(); List names = infos.stream().map((input) -> input.getName()).collect(Collectors.toList()); assertThat(names, contains("a", "b", "c", "d", "e")); } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index fd528f48f33..660f1015c3d 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -26,6 +26,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.test.ESTestCase; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; public class PluginsServiceTests extends ESTestCase { @@ -81,7 +83,7 @@ public class PluginsServiceTests extends ESTestCase { } static PluginsService newPluginsService(Settings settings, Class... classpathPlugins) { - return new PluginsService(settings, new Environment(settings).pluginsFile(), Arrays.asList(classpathPlugins)); + return new PluginsService(settings, null, new Environment(settings).pluginsFile(), Arrays.asList(classpathPlugins)); } public void testAdditionalSettings() { @@ -123,4 +125,15 @@ public class PluginsServiceTests extends ESTestCase { assertEquals("boom", ex.getCause().getCause().getMessage()); } } + + public void testExistingPluginMissingDescriptor() throws Exception { + Path pluginsDir = createTempDir(); + Files.createDirectory(pluginsDir.resolve("plugin-missing-descriptor")); + try { + PluginsService.getPluginBundles(pluginsDir); + fail(); + } catch (IllegalStateException e) { + assertTrue(e.getMessage(), e.getMessage().contains("Could not load plugin descriptor for existing plugin")); + } + } } diff --git a/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java b/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java new file mode 100644 index 00000000000..05a65363ff5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/ClassPermissionTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.test.ESTestCase; + +import java.security.AllPermission; +import java.security.PermissionCollection; + +/** Very simple sanity checks for {@link ClassPermission} */ +public class ClassPermissionTests extends ESTestCase { + + public void testEquals() { + assertEquals(new ClassPermission("pkg.MyClass"), new ClassPermission("pkg.MyClass")); + assertFalse(new ClassPermission("pkg.MyClass").equals(new AllPermission())); + } + + public void testImplies() { + assertTrue(new ClassPermission("pkg.MyClass").implies(new ClassPermission("pkg.MyClass"))); + assertFalse(new ClassPermission("pkg.MyClass").implies(new ClassPermission("pkg.MyOtherClass"))); + assertFalse(new ClassPermission("pkg.MyClass").implies(null)); + assertFalse(new ClassPermission("pkg.MyClass").implies(new AllPermission())); + } + + public void testStandard() { + assertTrue(new ClassPermission("<>").implies(new ClassPermission("java.lang.Math"))); + assertFalse(new ClassPermission("<>").implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollection() { + ClassPermission math = new ClassPermission("java.lang.Math"); + PermissionCollection collection = math.newPermissionCollection(); + collection.add(math); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollectionStandard() { + ClassPermission standard = new ClassPermission("<>"); + PermissionCollection collection = standard.newPermissionCollection(); + collection.add(standard); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } + + /** not recommended but we test anyway */ + public void testWildcards() { + assertTrue(new ClassPermission("*").implies(new ClassPermission("pkg.MyClass"))); + assertTrue(new ClassPermission("pkg.*").implies(new ClassPermission("pkg.MyClass"))); + assertTrue(new ClassPermission("pkg.*").implies(new ClassPermission("pkg.sub.MyClass"))); + assertFalse(new ClassPermission("pkg.My*").implies(new ClassPermission("pkg.MyClass"))); + assertFalse(new ClassPermission("pkg*").implies(new ClassPermission("pkg.MyClass"))); + } + + public void testPermissionCollectionWildcards() { + ClassPermission lang = new ClassPermission("java.lang.*"); + PermissionCollection collection = lang.newPermissionCollection(); + collection.add(lang); + assertTrue(collection.implies(new ClassPermission("java.lang.Math"))); + assertFalse(collection.implies(new ClassPermission("pkg.MyClass"))); + } +} diff --git a/distribution/build.gradle b/distribution/build.gradle index deeba3bef5b..56a602c1aa2 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -39,20 +39,61 @@ buildscript { } } -allprojects { - project.ext { - // this is common configuration for distributions, but we also add it here for the license check to use - dependencyFiles = project(':core').configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') +// this is common configuration for distributions, but we also add it here for the license check to use +ext.dependencyFiles = project(':core').configurations.runtime.copyRecursive().exclude(module: 'slf4j-api') + + +/***************************************************************************** + * Modules * + *****************************************************************************/ + +task buildModules(type: Copy) { + into 'build/modules' +} + +ext.restTestExpansions = [ + 'expected.modules.count': 0, +] +// we create the buildModules task above so the distribution subprojects can +// depend on it, but we don't actually configure it until projects are evaluated +// so it can depend on the bundling of plugins (ie modules must have been configured) +project.gradle.projectsEvaluated { + project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each { Project module -> + buildModules { + dependsOn module.bundlePlugin + into(module.name) { + from { zipTree(module.bundlePlugin.outputs.files.singleFile) } + } + } + configure(subprojects.findAll { it.name != 'integ-test-zip' }) { Project distribution -> + distribution.integTest.mustRunAfter(module.integTest) + } + restTestExpansions['expected.modules.count'] += 1 } } +// make sure we have a clean task since we aren't a java project, but we have tasks that +// put stuff in the build dir +task clean(type: Delete) { + delete 'build' +} + subprojects { /***************************************************************************** * Rest test config * *****************************************************************************/ apply plugin: 'elasticsearch.rest-test' - integTest { - includePackaged true + project.integTest { + dependsOn(project.assemble) + includePackaged project.name == 'integ-test-zip' + cluster { + distribution = project.name + } + } + + processTestResources { + inputs.properties(project(':distribution').restTestExpansions) + MavenFilteringHack.filter(it, project(':distribution').restTestExpansions) } /***************************************************************************** @@ -81,7 +122,12 @@ subprojects { libFiles = copySpec { into 'lib' from project(':core').jar - from dependencyFiles + from project(':distribution').dependencyFiles + } + + modulesFiles = copySpec { + into 'modules' + from project(':distribution').buildModules } configFiles = copySpec { @@ -103,7 +149,7 @@ subprojects { /***************************************************************************** * Zip and tgz configuration * *****************************************************************************/ -configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { +configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) { project.ext.archivesFiles = copySpec { into("elasticsearch-${version}") { with libFiles @@ -121,6 +167,9 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { from('../src/main/resources') { include 'bin/*.exe' } + if (project.name != 'integ-test-zip') { + with modulesFiles + } } } } @@ -143,7 +192,7 @@ configure(subprojects.findAll { it.name == 'zip' || it.name == 'tar' }) { * directly from the filesystem. It doesn't want to process them through * MavenFilteringHack or any other copy-style action. */ -configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { +configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) { integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false File packagingFiles = new File(buildDir, 'packaging') project.ext.packagingFiles = packagingFiles @@ -233,6 +282,7 @@ configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) { user 'root' permissionGroup 'root' with libFiles + with modulesFiles with copySpec { with commonFiles if (project.name == 'deb') { @@ -305,7 +355,7 @@ task updateShas(type: UpdateShasTask) { parentTask = dependencyLicenses } -RunTask.configure(project) +task run(type: RunTask) {} /** * Build some variables that are replaced in the packages. This includes both diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle index 72f6216d7b9..d9bd8447ab9 100644 --- a/distribution/deb/build.gradle +++ b/distribution/deb/build.gradle @@ -18,7 +18,7 @@ */ task buildDeb(type: Deb) { - dependsOn dependencyFiles, preparePackagingFiles + dependsOn preparePackagingFiles baseName 'elasticsearch' // this is what pom generation uses for artifactId // Follow elasticsearch's deb file naming convention archiveName "${packageName}-${project.version}.deb" @@ -44,6 +44,4 @@ integTest { skip the test if they aren't around. */ enabled = new File('/usr/bin/dpkg-deb').exists() || // Standard location new File('/usr/local/bin/dpkg-deb').exists() // Homebrew location - dependsOn buildDeb - clusterConfig.distribution = 'deb' } diff --git a/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..a7f265dd3ad --- /dev/null +++ b/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.plugins: ${expected.modules.count} } diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/distribution/integ-test-zip/build.gradle similarity index 61% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java rename to distribution/integ-test-zip/build.gradle index c1e2ed47dc4..23191ff03a4 100644 --- a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java +++ b/distribution/integ-test-zip/build.gradle @@ -17,24 +17,15 @@ * under the License. */ -package org.elasticsearch.script.expression; - -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptModule; - -public class ExpressionPlugin extends Plugin { - - @Override - public String name() { - return "lang-expression"; - } - - @Override - public String description() { - return "Lucene expressions integration for Elasticsearch"; - } - - public void onModule(ScriptModule module) { - module.addScriptEngine(ExpressionScriptEngineService.class); - } +task buildZip(type: Zip) { + baseName = 'elasticsearch' + with archivesFiles } + +artifacts { + 'default' buildZip + archives buildZip +} + +integTest.dependsOn buildZip + diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest3IT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java similarity index 81% rename from test-framework/src/main/java/org/elasticsearch/test/rest/Rest3IT.java rename to distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java index 7cbc974de0d..fd12fd2e519 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest3IT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RestIT.java @@ -19,20 +19,20 @@ package org.elasticsearch.test.rest; -import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -/** Rest API tests subset 3 */ -public class Rest3IT extends ESRestTestCase { - public Rest3IT(@Name("yaml") RestTestCandidate testCandidate) { +/** Rest integration test. runs against external cluster in 'mvn verify' */ +public class RestIT extends ESRestTestCase { + public RestIT(RestTestCandidate testCandidate) { super(testCandidate); } + // we run them all sequentially: start simple! @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(3, 8); + return createParameters(0, 1); } } diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index 0af164bdb20..2ab78fe7e41 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -42,6 +42,4 @@ integTest { enabled = new File('/bin/rpm').exists() || // Standard location new File('/usr/bin/rpm').exists() || // Debian location new File('/usr/local/bin/rpm').exists() // Homebrew location - dependsOn buildRpm - clusterConfig.distribution = 'rpm' } diff --git a/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..a7f265dd3ad --- /dev/null +++ b/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.plugins: ${expected.modules.count} } diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle index 5cf24e72f82..7230ab50799 100644 --- a/distribution/tar/build.gradle +++ b/distribution/tar/build.gradle @@ -17,7 +17,7 @@ * under the License. */ -task buildTar(type: Tar, dependsOn: dependencyFiles) { +task buildTar(type: Tar) { baseName = 'elasticsearch' extension = 'tar.gz' with archivesFiles @@ -28,8 +28,3 @@ artifacts { 'default' buildTar archives buildTar } - -integTest { - dependsOn buildTar - clusterConfig.distribution = 'tar' -} diff --git a/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..a7f265dd3ad --- /dev/null +++ b/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.plugins: ${expected.modules.count} } diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle index d636e66f152..23191ff03a4 100644 --- a/distribution/zip/build.gradle +++ b/distribution/zip/build.gradle @@ -17,7 +17,7 @@ * under the License. */ -task buildZip(type: Zip, dependsOn: dependencyFiles) { +task buildZip(type: Zip) { baseName = 'elasticsearch' with archivesFiles } diff --git a/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml b/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml new file mode 100644 index 00000000000..a7f265dd3ad --- /dev/null +++ b/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yaml @@ -0,0 +1,13 @@ +# Integration tests for distributions with modules +# +"Correct Modules Count": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - length: { nodes.$master.plugins: ${expected.modules.count} } diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index 822d8864f71..e6c3e49b43d 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -90,6 +90,9 @@ The search exists api has been removed in favour of using the search api with The deprecated `/_optimize` endpoint has been removed. The `/_forcemerge` endpoint should be used in lieu of optimize. +The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the +`POST` HTTP verb. + ==== Deprecated queries removed The following deprecated queries have been removed: diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc index 2ac2963a239..b3c00d337f8 100644 --- a/docs/reference/setup/rolling_upgrade.asciidoc +++ b/docs/reference/setup/rolling_upgrade.asciidoc @@ -60,7 +60,7 @@ default. It is a good idea to place these directories in a different location so that there is no chance of deleting them when upgrading Elasticsearch. These -custom paths can be <> with the `path.config` and +custom paths can be <> with the `path.conf` and `path.data` settings. The Debian and RPM packages place these directories in the @@ -80,7 +80,7 @@ To upgrade using a zip or compressed tarball: overwrite the `config` or `data` directories. * Either copy the files in the `config` directory from your old installation - to your new installation, or use the `--path.config` option on the command + to your new installation, or use the `--path.conf` option on the command line to point to an external config directory. * Either copy the files in the `data` directory from your old installation diff --git a/docs/reference/setup/upgrade.asciidoc b/docs/reference/setup/upgrade.asciidoc index 15cd90f98dd..894f82a6db5 100644 --- a/docs/reference/setup/upgrade.asciidoc +++ b/docs/reference/setup/upgrade.asciidoc @@ -21,12 +21,10 @@ consult this table: [cols="1> -|< 0.90.7 |0.90.x |<> -|>= 0.90.7 |0.90.x |<> -|1.0.0 - 1.3.1 |1.x |<> (if <> set to `false`) -|>= 1.3.2 |1.x |<> +|0.90.x |2.x |<> |1.x |2.x |<> +|2.x |2.y |<> (where `y > x `) +|2.x |3.x |<> |======================================================================= TIP: Take plugins into consideration as well when upgrading. Most plugins will have to be upgraded alongside Elasticsearch, although some plugins accessed primarily through the browser (`_site` plugins) may continue to work given that API changes are compatible. diff --git a/modules/build.gradle b/modules/build.gradle new file mode 100644 index 00000000000..41f7a8873b4 --- /dev/null +++ b/modules/build.gradle @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +subprojects { + apply plugin: 'elasticsearch.esplugin' + + esplugin { + // for local ES plugins, the name of the plugin is the same as the directory + name project.name + } + + if (project.file('src/main/packaging').exists()) { + throw new InvalidModelException("Modules cannot contain packaging files") + } + if (project.file('src/main/bin').exists()) { + throw new InvalidModelException("Modules cannot contain bin files") + } + if (project.file('src/main/config').exists()) { + throw new InvalidModelException("Modules cannot contain config files") + } + + project.afterEvaluate { + if (esplugin.isolated == false) { + throw new InvalidModelException("Modules cannot disable isolation") + } + if (esplugin.jvm == false) { + throw new InvalidModelException("Modules must be jvm plugins") + } + } +} diff --git a/plugins/lang-expression/build.gradle b/modules/lang-expression/build.gradle similarity index 100% rename from plugins/lang-expression/build.gradle rename to modules/lang-expression/build.gradle diff --git a/plugins/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 b/modules/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 rename to modules/lang-expression/licenses/antlr4-runtime-4.5.1-1.jar.sha1 diff --git a/plugins/lang-expression/licenses/antlr4-runtime-LICENSE.txt b/modules/lang-expression/licenses/antlr4-runtime-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/antlr4-runtime-LICENSE.txt rename to modules/lang-expression/licenses/antlr4-runtime-LICENSE.txt diff --git a/plugins/lang-expression/licenses/antlr4-runtime-NOTICE.txt b/modules/lang-expression/licenses/antlr4-runtime-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/antlr4-runtime-NOTICE.txt rename to modules/lang-expression/licenses/antlr4-runtime-NOTICE.txt diff --git a/plugins/lang-expression/licenses/asm-5.0.4.jar.sha1 b/modules/lang-expression/licenses/asm-5.0.4.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/asm-5.0.4.jar.sha1 rename to modules/lang-expression/licenses/asm-5.0.4.jar.sha1 diff --git a/plugins/lang-expression/licenses/asm-LICENSE.txt b/modules/lang-expression/licenses/asm-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-LICENSE.txt rename to modules/lang-expression/licenses/asm-LICENSE.txt diff --git a/plugins/lang-expression/licenses/asm-NOTICE.txt b/modules/lang-expression/licenses/asm-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-NOTICE.txt rename to modules/lang-expression/licenses/asm-NOTICE.txt diff --git a/plugins/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 b/modules/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 rename to modules/lang-expression/licenses/asm-commons-5.0.4.jar.sha1 diff --git a/plugins/lang-expression/licenses/asm-commons-LICENSE.txt b/modules/lang-expression/licenses/asm-commons-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-LICENSE.txt rename to modules/lang-expression/licenses/asm-commons-LICENSE.txt diff --git a/plugins/lang-expression/licenses/asm-commons-NOTICE.txt b/modules/lang-expression/licenses/asm-commons-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/asm-commons-NOTICE.txt rename to modules/lang-expression/licenses/asm-commons-NOTICE.txt diff --git a/plugins/lang-expression/licenses/lucene-LICENSE.txt b/modules/lang-expression/licenses/lucene-LICENSE.txt similarity index 100% rename from plugins/lang-expression/licenses/lucene-LICENSE.txt rename to modules/lang-expression/licenses/lucene-LICENSE.txt diff --git a/plugins/lang-expression/licenses/lucene-NOTICE.txt b/modules/lang-expression/licenses/lucene-NOTICE.txt similarity index 100% rename from plugins/lang-expression/licenses/lucene-NOTICE.txt rename to modules/lang-expression/licenses/lucene-NOTICE.txt diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 similarity index 100% rename from plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 rename to modules/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java new file mode 100644 index 00000000000..48c7b4cb34d --- /dev/null +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.js.JavascriptCompiler; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptModule; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.text.ParseException; + +public class ExpressionPlugin extends Plugin { + + // lucene expressions has crazy checks in its clinit for the functions map + // it violates rules of classloaders to detect accessibility + // TODO: clean that up + static { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + JavascriptCompiler.compile("0"); + } catch (ParseException e) { + throw new RuntimeException(e); + } + return null; + } + }); + } + + @Override + public String name() { + return "lang-expression"; + } + + @Override + public String description() { + return "Lucene expressions integration for Elasticsearch"; + } + + public void onModule(ScriptModule module) { + module.addScriptEngine(ExpressionScriptEngineService.class); + } +} diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java similarity index 90% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 72a1dd7d5c6..a7f93925119 100644 --- a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.script.ClassPermission; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptEngineService; @@ -44,6 +45,7 @@ import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.lookup.SearchLookup; +import java.security.AccessControlContext; import java.security.AccessController; import java.security.PrivilegedAction; import java.text.ParseException; @@ -95,7 +97,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements @Override public Object compile(String script) { // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } @@ -103,8 +105,24 @@ public class ExpressionScriptEngineService extends AbstractComponent implements @Override public Expression run() { try { + // snapshot our context here, we check on behalf of the expression + AccessControlContext engineContext = AccessController.getContext(); + ClassLoader loader = getClass().getClassLoader(); + if (sm != null) { + loader = new ClassLoader(loader) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }; + } // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here - return JavascriptCompiler.compile(script); + return JavascriptCompiler.compile(script, JavascriptCompiler.DEFAULT_FUNCTIONS, loader); } catch (ParseException e) { throw new ScriptException("Failed to parse expression: " + script, e); } diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java diff --git a/plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java similarity index 100% rename from plugins/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..9f50be3dd05 --- /dev/null +++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // needed to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; + // needed because of security problems in JavascriptCompiler + permission java.lang.RuntimePermission "getClassLoader"; + + // expression runtime + permission org.elasticsearch.script.ClassPermission "java.lang.String"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.queries.function.FunctionValues"; + // available functions + permission org.elasticsearch.script.ClassPermission "java.lang.Math"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; + permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; +}; diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java similarity index 100% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionRestIT.java diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java similarity index 100% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java similarity index 100% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java diff --git a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java similarity index 98% rename from plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index 4ac4b402c17..89a5be7ff1c 100644 --- a/plugins/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -97,6 +97,16 @@ public class MoreExpressionTests extends ESIntegTestCase { assertEquals(1, rsp.getHits().getTotalHits()); assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); } + + public void testFunction() throws Exception { + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); + SearchResponse rsp = buildRequest("doc['foo'] + abs(1)").get(); + assertSearchResponse(rsp); + assertEquals(1, rsp.getHits().getTotalHits()); + assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D); + } public void testBasicUsingDotValue() throws Exception { createIndex("test"); diff --git a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml similarity index 62% rename from plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml rename to modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml index 9c7819d925d..1550f2a7f81 100644 --- a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml +++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yaml @@ -10,5 +10,5 @@ - do: nodes.info: {} - - match: { nodes.$master.plugins.0.name: lang-expression } - - match: { nodes.$master.plugins.0.jvm: true } + - match: { nodes.$master.modules.0.name: lang-expression } + - match: { nodes.$master.modules.0.jvm: true } diff --git a/plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml similarity index 100% rename from plugins/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml rename to modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yaml diff --git a/plugins/lang-groovy/build.gradle b/modules/lang-groovy/build.gradle similarity index 100% rename from plugins/lang-groovy/build.gradle rename to modules/lang-groovy/build.gradle diff --git a/plugins/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 b/modules/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 rename to modules/lang-groovy/licenses/groovy-all-2.4.4-indy.jar.sha1 diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-ANTLR.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-ASM.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-CLI.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE-JSR223.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-LICENSE.txt b/modules/lang-groovy/licenses/groovy-all-LICENSE.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-LICENSE.txt rename to modules/lang-groovy/licenses/groovy-all-LICENSE.txt diff --git a/plugins/lang-groovy/licenses/groovy-all-NOTICE.txt b/modules/lang-groovy/licenses/groovy-all-NOTICE.txt similarity index 100% rename from plugins/lang-groovy/licenses/groovy-all-NOTICE.txt rename to modules/lang-groovy/licenses/groovy-all-NOTICE.txt diff --git a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java similarity index 100% rename from plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java rename to modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java diff --git a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java similarity index 85% rename from plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java rename to modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index d1e7160282b..85f57694ce6 100644 --- a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -51,6 +51,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.security.AccessControlContext; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.HashMap; @@ -65,16 +66,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri * The name of the scripting engine/language. */ public static final String NAME = "groovy"; - /** - * The setting to enable or disable invokedynamic instruction support in Java 7+. - *

- * Note: If this is disabled because invokedynamic is causing issues, then the Groovy - * indy jar needs to be replaced by the non-indy variant of it on the classpath (e.g., - * groovy-all-2.4.4-indy.jar should be replaced by groovy-all-2.4.4.jar). - *

- * Defaults to {@code true}. - */ - public static final String GROOVY_INDY_ENABLED = "script.groovy.indy"; /** * The name of the Groovy compiler setting to use associated with activating invokedynamic support. */ @@ -96,22 +87,33 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri // Add BigDecimal -> Double transformer config.addCompilationCustomizers(new GroovyBigDecimalTransformer(CompilePhase.CONVERSION)); - // Implicitly requires Java 7u60 or later to get valid support - if (settings.getAsBoolean(GROOVY_INDY_ENABLED, true)) { - // maintain any default optimizations - config.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true); - } + // always enable invokeDynamic, not the crazy softreference-based stuff + config.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true); // Groovy class loader to isolate Groovy-land code // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } this.loader = AccessController.doPrivileged(new PrivilegedAction() { @Override public GroovyClassLoader run() { - return new GroovyClassLoader(getClass().getClassLoader(), config); + // snapshot our context (which has permissions for classes), since the script has none + final AccessControlContext engineContext = AccessController.getContext(); + return new GroovyClassLoader(new ClassLoader(getClass().getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (sm != null) { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + } + return super.loadClass(name, resolve); + } + }, config); } }); } @@ -172,13 +174,15 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri } String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8))); // same logic as GroovyClassLoader.parseClass() but with a different codesource string: - GroovyCodeSource gcs = AccessController.doPrivileged(new PrivilegedAction() { - public GroovyCodeSource run() { - return new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE); + return AccessController.doPrivileged(new PrivilegedAction() { + public Class run() { + GroovyCodeSource gcs = new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE); + gcs.setCachable(false); + // TODO: we could be more complicated and paranoid, and move this to separate block, to + // sandbox the compilation process itself better. + return loader.parseClass(gcs); } }); - gcs.setCachable(false); - return loader.parseClass(gcs); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("exception compiling Groovy script:", e); @@ -293,7 +297,14 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri @Override public Object run() { try { - return script.run(); + // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader) + // we don't do a security check just as a tradeoff, it cannot really escalate to anything. + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Object run() { + return script.run(); + } + }); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("failed to run " + compiledScript, e); diff --git a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..7de3e1a62aa --- /dev/null +++ b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grant { + // needed to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; + // needed by IndyInterface + permission java.lang.RuntimePermission "getClassLoader"; + // needed by groovy engine + permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; + // needed by GroovyScriptEngineService to close its classloader (why?) + permission java.lang.RuntimePermission "closeClassLoader"; + // Allow executing groovy scripts with codesource of /untrusted + permission groovy.security.GroovyCodeSourcePermission "/untrusted"; + + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; + // groovy runtime (TODO: clean these up if possible) + permission org.elasticsearch.script.ClassPermission "groovy.grape.GrabAnnotationTransformation"; + permission org.elasticsearch.script.ClassPermission "groovy.json.JsonOutput"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.Binding"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.GroovyObject"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.GString"; + permission org.elasticsearch.script.ClassPermission "groovy.lang.Script"; + permission org.elasticsearch.script.ClassPermission "groovy.util.GroovyCollections"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.ast.builder.AstBuilderTransformation"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.reflection.ClassInfo"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.GStringImpl"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.powerassert.ValueRecorder"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.powerassert.AssertionRenderer"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.ScriptBytecodeAdapter"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation"; + permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.vmplugin.v7.IndyInterface"; + permission org.elasticsearch.script.ClassPermission "sun.reflect.ConstructorAccessorImpl"; +}; diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RangeTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptIndexSettingsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptedMetricTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java similarity index 100% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java diff --git a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java similarity index 60% rename from plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java rename to modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java index 258d957d77e..5f91631c021 100644 --- a/plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java @@ -20,16 +20,22 @@ package org.elasticsearch.script.groovy; import org.apache.lucene.util.Constants; +import org.codehaus.groovy.control.MultipleCompilationErrorsException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; +import groovy.lang.MissingPropertyException; + import java.nio.file.Path; +import java.security.PrivilegedActionException; import java.util.AbstractMap; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -48,7 +54,7 @@ public class GroovySecurityTests extends ESTestCase { @Override public void setUp() throws Exception { super.setUp(); - se = new GroovyScriptEngineService(Settings.Builder.EMPTY_SETTINGS); + se = new GroovyScriptEngineService(Settings.EMPTY); // otherwise will exit your VM and other bad stuff assumeTrue("test requires security manager to be enabled", System.getSecurityManager() != null); } @@ -62,8 +68,16 @@ public class GroovySecurityTests extends ESTestCase { public void testEvilGroovyScripts() throws Exception { // Plain test assertSuccess(""); - // field access + // field access (via map) assertSuccess("def foo = doc['foo'].value; if (foo == null) { return 5; }"); + // field access (via list) + assertSuccess("def foo = mylist[0]; if (foo == null) { return 5; }"); + // field access (via array) + assertSuccess("def foo = myarray[0]; if (foo == null) { return 5; }"); + // field access (via object) + assertSuccess("def foo = myobject.primitive.toString(); if (foo == null) { return 5; }"); + assertSuccess("def foo = myobject.object.toString(); if (foo == null) { return 5; }"); + assertSuccess("def foo = myobject.list[0].primitive.toString(); if (foo == null) { return 5; }"); // List assertSuccess("def list = [doc['foo'].value, 3, 4]; def v = list.get(1); list.add(10)"); // Ranges @@ -78,35 +92,35 @@ public class GroovySecurityTests extends ESTestCase { assertSuccess("def n = [1,2,3]; GroovyCollections.max(n)"); // Fail cases: - // AccessControlException[access denied ("java.io.FilePermission" "<>" "execute")] - assertFailure("pr = Runtime.getRuntime().exec(\"touch /tmp/gotcha\"); pr.waitFor()"); + assertFailure("pr = Runtime.getRuntime().exec(\"touch /tmp/gotcha\"); pr.waitFor()", MissingPropertyException.class); - // AccessControlException[access denied ("java.lang.RuntimePermission" "accessClassInPackage.sun.reflect")] - assertFailure("d = new DateTime(); d.getClass().getDeclaredMethod(\"year\").setAccessible(true)"); + // infamous: + assertFailure("java.lang.Math.class.forName(\"java.lang.Runtime\")", PrivilegedActionException.class); + // filtered directly by our classloader + assertFailure("getClass().getClassLoader().loadClass(\"java.lang.Runtime\").availableProcessors()", PrivilegedActionException.class); + // unfortunately, we have access to other classloaders (due to indy mechanism needing getClassLoader permission) + // but we can't do much with them directly at least. + assertFailure("myobject.getClass().getClassLoader().loadClass(\"java.lang.Runtime\").availableProcessors()", SecurityException.class); + assertFailure("d = new DateTime(); d.getClass().getDeclaredMethod(\"year\").setAccessible(true)", SecurityException.class); assertFailure("d = new DateTime(); d.\"${'get' + 'Class'}\"()." + - "\"${'getDeclared' + 'Method'}\"(\"year\").\"${'set' + 'Accessible'}\"(false)"); - assertFailure("Class.forName(\"org.joda.time.DateTime\").getDeclaredMethod(\"year\").setAccessible(true)"); + "\"${'getDeclared' + 'Method'}\"(\"year\").\"${'set' + 'Accessible'}\"(false)", SecurityException.class); + assertFailure("Class.forName(\"org.joda.time.DateTime\").getDeclaredMethod(\"year\").setAccessible(true)", MissingPropertyException.class); - // AccessControlException[access denied ("groovy.security.GroovyCodeSourcePermission" "/groovy/shell")] - assertFailure("Eval.me('2 + 2')"); - assertFailure("Eval.x(5, 'x + 2')"); + assertFailure("Eval.me('2 + 2')", MissingPropertyException.class); + assertFailure("Eval.x(5, 'x + 2')", MissingPropertyException.class); - // AccessControlException[access denied ("java.lang.RuntimePermission" "accessDeclaredMembers")] assertFailure("d = new Date(); java.lang.reflect.Field f = Date.class.getDeclaredField(\"fastTime\");" + - " f.setAccessible(true); f.get(\"fastTime\")"); + " f.setAccessible(true); f.get(\"fastTime\")", MultipleCompilationErrorsException.class); - // AccessControlException[access denied ("java.io.FilePermission" "<>" "execute")] - assertFailure("def methodName = 'ex'; Runtime.\"${'get' + 'Runtime'}\"().\"${methodName}ec\"(\"touch /tmp/gotcha2\")"); + assertFailure("def methodName = 'ex'; Runtime.\"${'get' + 'Runtime'}\"().\"${methodName}ec\"(\"touch /tmp/gotcha2\")", MissingPropertyException.class); - // AccessControlException[access denied ("java.lang.RuntimePermission" "modifyThreadGroup")] - assertFailure("t = new Thread({ println 3 });"); + assertFailure("t = new Thread({ println 3 });", MultipleCompilationErrorsException.class); // test a directory we normally have access to, but the groovy script does not. Path dir = createTempDir(); // TODO: figure out the necessary escaping for windows paths here :) if (!Constants.WINDOWS) { - // access denied ("java.io.FilePermission" ".../tempDir-00N" "read") - assertFailure("new File(\"" + dir + "\").exists()"); + assertFailure("new File(\"" + dir + "\").exists()", MultipleCompilationErrorsException.class); } } @@ -115,8 +129,18 @@ public class GroovySecurityTests extends ESTestCase { Map vars = new HashMap(); // we add a "mock document" containing a single field "foo" that returns 4 (abusing a jdk class with a getValue() method) vars.put("doc", Collections.singletonMap("foo", new AbstractMap.SimpleEntry(null, 4))); + vars.put("mylist", Arrays.asList("foo")); + vars.put("myarray", Arrays.asList("foo")); + vars.put("myobject", new MyObject()); + se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(script)), vars).run(); } + + public static class MyObject { + public int getPrimitive() { return 0; } + public Object getObject() { return "value"; } + public List getList() { return Arrays.asList(new MyObject()); } + } /** asserts that a script runs without exception */ private void assertSuccess(String script) { @@ -124,14 +148,16 @@ public class GroovySecurityTests extends ESTestCase { } /** asserts that a script triggers securityexception */ - private void assertFailure(String script) { + private void assertFailure(String script, Class exceptionClass) { try { doTest(script); fail("did not get expected exception"); } catch (ScriptException expected) { Throwable cause = expected.getCause(); assertNotNull(cause); - assertTrue("unexpected exception: " + cause, cause instanceof SecurityException); + if (exceptionClass.isAssignableFrom(cause.getClass()) == false) { + throw new AssertionError("unexpected exception: " + cause, expected); + } } } } diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/combine_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/init_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/map_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/reduce_script.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_no_params.groovy diff --git a/plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy b/modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy similarity index 100% rename from plugins/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy rename to modules/lang-groovy/src/test/resources/org/elasticsearch/messy/tests/conf/scripts/significance_script_with_params.groovy diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml similarity index 62% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml index 123b02fc7fa..c276bab6495 100644 --- a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml +++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/10_basic.yaml @@ -10,5 +10,5 @@ - do: nodes.info: {} - - match: { nodes.$master.plugins.0.name: lang-groovy } - - match: { nodes.$master.plugins.0.jvm: true } + - match: { nodes.$master.modules.0.name: lang-groovy } + - match: { nodes.$master.modules.0.jvm: true } diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/15_update.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/20_versions.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/25_script_upsert.yaml diff --git a/plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml similarity index 100% rename from plugins/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml rename to modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/90_missing.yaml diff --git a/plugins/build.gradle b/plugins/build.gradle index 90429cc83d1..bdcc604a296 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -17,8 +17,6 @@ * under the License. */ -import org.elasticsearch.gradle.precommit.DependencyLicensesTask - subprojects { group = 'org.elasticsearch.plugin' diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle index d72c203d089..5042824eb07 100644 --- a/plugins/discovery-azure/build.gradle +++ b/plugins/discovery-azure/build.gradle @@ -23,18 +23,19 @@ esplugin { } versions << [ - 'azure': '0.7.0', + 'azure': '0.9.0', 'jersey': '1.13' ] dependencies { - compile "com.microsoft.azure:azure-management-compute:${versions.azure}" - compile "com.microsoft.azure:azure-management:${versions.azure}" + compile "com.microsoft.azure:azure-svc-mgmt-compute:${versions.azure}" compile "com.microsoft.azure:azure-core:${versions.azure}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "commons-lang:commons-lang:2.6" + compile "commons-io:commons-io:2.4" compile 'javax.mail:mail:1.4.5' compile 'javax.activation:activation:1.1' compile 'javax.inject:javax.inject:1' diff --git a/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 deleted file mode 100644 index f7d0b7caabc..00000000000 --- a/plugins/discovery-azure/licenses/azure-core-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -feed802efe8a7a83d15962d11c6780c63997c528 diff --git a/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 new file mode 100644 index 00000000000..f9696307afe --- /dev/null +++ b/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 @@ -0,0 +1 @@ +050719f91deceed1be1aaf87e85099a861295fa2 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 deleted file mode 100644 index f69856a386e..00000000000 --- a/plugins/discovery-azure/licenses/azure-management-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0dfdd1c3a9bd783b087050e979f6ba34f06a68f3 diff --git a/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 deleted file mode 100644 index bcab189bc14..00000000000 --- a/plugins/discovery-azure/licenses/azure-management-compute-0.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b945fc3968a4e5a64bbde419c14d92a4a53fa7a1 diff --git a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 new file mode 100644 index 00000000000..c971d7c5724 --- /dev/null +++ b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 @@ -0,0 +1 @@ +887ca8ee5564e8ba2351e6b5db2a1293a8d04674 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 b/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 new file mode 100644 index 00000000000..2f5b30d0edb --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 @@ -0,0 +1 @@ +b1b6ea3b7e4aa4f492509a4952029cd8e48019ad \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-io-LICENSE.txt b/plugins/discovery-azure/licenses/commons-io-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/discovery-azure/licenses/commons-io-NOTICE.txt b/plugins/discovery-azure/licenses/commons-io-NOTICE.txt new file mode 100644 index 00000000000..a6b77d1eb60 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-io-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons IO +Copyright 2002-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 b/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 new file mode 100644 index 00000000000..4ee9249d2b7 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 @@ -0,0 +1 @@ +0ce1edb914c94ebc388f086c6827e8bdeec71ac2 \ No newline at end of file diff --git a/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt b/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt b/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt new file mode 100644 index 00000000000..592023af76b --- /dev/null +++ b/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt @@ -0,0 +1,8 @@ +Apache Commons Lang +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java index a95d1a73a75..ffa76c6b9b3 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.plugin.discovery.ec2; +import org.elasticsearch.SpecialPermission; import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl; import org.elasticsearch.cloud.aws.Ec2Module; import org.elasticsearch.common.component.LifecycleComponent; @@ -31,6 +32,8 @@ import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider; import org.elasticsearch.discovery.ec2.Ec2Discovery; import org.elasticsearch.plugins.Plugin; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; @@ -38,6 +41,26 @@ import java.util.Collection; * */ public class Ec2DiscoveryPlugin extends Plugin { + + // ClientConfiguration clinit has some classloader problems + // TODO: fix that + static { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + Class.forName("com.amazonaws.ClientConfiguration"); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + return null; + } + }); + } private final Settings settings; protected final ESLogger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class); diff --git a/plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy similarity index 62% rename from plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy rename to plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy index 55c2fab13f7..42bd707b7b9 100644 --- a/plugins/lang-groovy/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy @@ -18,12 +18,9 @@ */ grant { - // needed to generate runtime classes - permission java.lang.RuntimePermission "createClassLoader"; - // needed by groovy engine - permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; - // needed by GroovyScriptEngineService to close its classloader (why?) - permission java.lang.RuntimePermission "closeClassLoader"; - // Allow executing groovy scripts with codesource of /untrusted - permission groovy.security.GroovyCodeSourcePermission "/untrusted"; + // needed because of problems in ClientConfiguration + // TODO: get this fixed in aws sdk + // NOTE: no tests fail without this, but we know the problem + // exists in AWS sdk, and tests here are not thorough + permission java.lang.RuntimePermission "getClassLoader"; }; diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java index 621d338128f..33a4e55801b 100644 --- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java +++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java @@ -39,7 +39,10 @@ import org.mozilla.javascript.Script; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.security.AccessControlContext; +import java.security.AccessController; import java.security.CodeSource; +import java.security.PrivilegedAction; import java.security.cert.Certificate; import java.util.List; import java.util.Map; @@ -54,18 +57,47 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements private static WrapFactory wrapFactory = new CustomWrapFactory(); - private final int optimizationLevel; - private Scriptable globalScope; // one time initialization of rhino security manager integration private static final CodeSource DOMAIN; + private static final int OPTIMIZATION_LEVEL = 1; + static { try { DOMAIN = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null); } catch (MalformedURLException e) { throw new RuntimeException(e); } + ContextFactory factory = new ContextFactory() { + @Override + protected void onContextCreated(Context cx) { + cx.setWrapFactory(wrapFactory); + cx.setOptimizationLevel(OPTIMIZATION_LEVEL); + } + }; + if (System.getSecurityManager() != null) { + factory.initApplicationClassLoader(AccessController.doPrivileged(new PrivilegedAction() { + @Override + public ClassLoader run() { + // snapshot our context (which has permissions for classes), since the script has none + final AccessControlContext engineContext = AccessController.getContext(); + return new ClassLoader(JavaScriptScriptEngineService.class.getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }; + } + })); + } + factory.seal(); + ContextFactory.initGlobal(factory); SecurityController.initGlobal(new PolicySecurityController() { @Override public GeneratedClassLoader createClassLoader(ClassLoader parent, Object securityDomain) { @@ -78,6 +110,7 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements if (securityDomain != DOMAIN) { throw new SecurityException("illegal securityDomain: " + securityDomain); } + return super.createClassLoader(parent, securityDomain); } }); @@ -90,11 +123,8 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public JavaScriptScriptEngineService(Settings settings) { super(settings); - this.optimizationLevel = settings.getAsInt("script.javascript.optimization_level", 1); - Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); globalScope = ctx.initStandardObjects(null, true); } finally { Context.exit(); @@ -130,8 +160,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public Object compile(String script) { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - ctx.setOptimizationLevel(optimizationLevel); return ctx.compileString(script, generateScriptName(), 1, DOMAIN); } finally { Context.exit(); @@ -142,8 +170,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public ExecutableScript executable(CompiledScript compiledScript, Map vars) { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - Scriptable scope = ctx.newObject(globalScope); scope.setPrototype(globalScope); scope.setParentScope(null); @@ -161,8 +187,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public SearchScript search(final CompiledScript compiledScript, final SearchLookup lookup, @Nullable final Map vars) { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); - final Scriptable scope = ctx.newObject(globalScope); scope.setPrototype(globalScope); scope.setParentScope(null); @@ -215,7 +239,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public Object run() { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); } finally { Context.exit(); @@ -276,7 +299,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements public Object run() { Context ctx = Context.enter(); try { - ctx.setWrapFactory(wrapFactory); return ScriptValueConverter.unwrapValue(script.exec(ctx, scope)); } finally { Context.exit(); diff --git a/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy b/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy index e45c1b86ceb..739a2531d2f 100644 --- a/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/lang-javascript/src/main/plugin-metadata/plugin-security.policy @@ -20,4 +20,15 @@ grant { // needed to generate runtime classes permission java.lang.RuntimePermission "createClassLoader"; + + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; + // rhino runtime (TODO: clean these up if possible) + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.ContextFactory"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Callable"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.NativeFunction"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Script"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.ScriptRuntime"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.Undefined"; + permission org.elasticsearch.script.ClassPermission "org.mozilla.javascript.optimizer.OptRuntime"; }; diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java index 410099de0a7..c6f9805f818 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; +import org.mozilla.javascript.EcmaError; import org.mozilla.javascript.WrappedException; import java.util.HashMap; @@ -61,14 +62,20 @@ public class JavaScriptSecurityTests extends ESTestCase { } /** assert that a security exception is hit */ - private void assertFailure(String script) { + private void assertFailure(String script, Class exceptionClass) { try { doTest(script); fail("did not get expected exception"); } catch (WrappedException expected) { Throwable cause = expected.getCause(); assertNotNull(cause); - assertTrue("unexpected exception: " + cause, cause instanceof SecurityException); + if (exceptionClass.isAssignableFrom(cause.getClass()) == false) { + throw new AssertionError("unexpected exception: " + expected, expected); + } + } catch (EcmaError expected) { + if (exceptionClass.isAssignableFrom(expected.getClass()) == false) { + throw new AssertionError("unexpected exception: " + expected, expected); + } } } @@ -79,22 +86,22 @@ public class JavaScriptSecurityTests extends ESTestCase { } /** Test some javascripts that should hit security exception */ - public void testNotOK() { + public void testNotOK() throws Exception { // sanity check :) - assertFailure("java.lang.Runtime.getRuntime().halt(0)"); + assertFailure("java.lang.Runtime.getRuntime().halt(0)", EcmaError.class); // check a few things more restrictive than the ordinary policy // no network - assertFailure("new java.net.Socket(\"localhost\", 1024)"); + assertFailure("new java.net.Socket(\"localhost\", 1024)", EcmaError.class); // no files - assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")"); + assertFailure("java.io.File.createTempFile(\"test\", \"tmp\")", EcmaError.class); } public void testDefinitelyNotOK() { // no mucking with security controller assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " + - "ctx.setSecurityController(new org.mozilla.javascript.PolicySecurityController());"); + "ctx.setSecurityController(new org.mozilla.javascript.PolicySecurityController());", EcmaError.class); // no compiling scripts from scripts assertFailure("var ctx = org.mozilla.javascript.Context.getCurrentContext(); " + - "ctx.compileString(\"1 + 1\", \"foobar\", 1, null); "); + "ctx.compileString(\"1 + 1\", \"foobar\", 1, null); ", EcmaError.class); } } diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index 3dfa4bcd0f9..1930f530671 100644 --- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -25,7 +25,11 @@ import java.security.AccessController; import java.security.Permissions; import java.security.PrivilegedAction; import java.security.ProtectionDomain; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; @@ -34,6 +38,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ClassPermission; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; @@ -55,20 +60,36 @@ import org.python.util.PythonInterpreter; public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService { private final PythonInterpreter interp; - + @Inject public PythonScriptEngineService(Settings settings) { super(settings); // classloader created here - SecurityManager sm = System.getSecurityManager(); + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { sm.checkPermission(new SpecialPermission()); } this.interp = AccessController.doPrivileged(new PrivilegedAction () { @Override public PythonInterpreter run() { - return PythonInterpreter.threadLocalStateInterpreter(null); + // snapshot our context here for checks, as the script has no permissions + final AccessControlContext engineContext = AccessController.getContext(); + PythonInterpreter interp = PythonInterpreter.threadLocalStateInterpreter(null); + if (sm != null) { + interp.getSystemState().setClassLoader(new ClassLoader(getClass().getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + engineContext.checkPermission(new ClassPermission(name)); + } catch (SecurityException e) { + throw new ClassNotFoundException(name, e); + } + return super.loadClass(name, resolve); + } + }); + } + return interp; } }); } diff --git a/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy b/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy index e45c1b86ceb..86f4df64db4 100644 --- a/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/lang-python/src/main/plugin-metadata/plugin-security.policy @@ -20,4 +20,8 @@ grant { // needed to generate runtime classes permission java.lang.RuntimePermission "createClassLoader"; + // needed by PySystemState init (TODO: see if we can avoid this) + permission java.lang.RuntimePermission "getClassLoader"; + // Standard set of classes + permission org.elasticsearch.script.ClassPermission "<>"; }; diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java index dd25db815ac..e90ac503f13 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java @@ -25,7 +25,9 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.python.core.PyException; +import java.text.DecimalFormatSymbols; import java.util.HashMap; +import java.util.Locale; import java.util.Map; /** @@ -66,12 +68,12 @@ public class PythonSecurityTests extends ESTestCase { doTest(script); fail("did not get expected exception"); } catch (PyException expected) { - Throwable cause = expected.getCause(); // TODO: fix jython localization bugs: https://github.com/elastic/elasticsearch/issues/13967 - // this is the correct assert: - // assertNotNull("null cause for exception: " + expected, cause); - assertNotNull("null cause for exception", cause); - assertTrue("unexpected exception: " + cause, cause instanceof SecurityException); + // we do a gross hack for now + DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(Locale.getDefault()); + if (symbols.getZeroDigit() == '0') { + assertTrue(expected.toString().contains("cannot import")); + } } } @@ -91,4 +93,16 @@ public class PythonSecurityTests extends ESTestCase { // no files assertFailure("from java.io import File\nFile.createTempFile(\"test\", \"tmp\")"); } + + /** Test again from a new thread, python has complex threadlocal configuration */ + public void testNotOKFromSeparateThread() throws Exception { + Thread t = new Thread() { + @Override + public void run() { + assertFailure("from java.lang import Runtime\nRuntime.availableProcessors()"); + } + }; + t.start(); + t.join(); + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java index 2911e278c38..a38a8ed3c51 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.plugin.repository.s3; +import org.elasticsearch.SpecialPermission; import org.elasticsearch.cloud.aws.S3Module; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; @@ -27,6 +28,9 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.s3.S3Repository; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.text.ParseException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -36,6 +40,26 @@ import java.util.Collections; */ public class S3RepositoryPlugin extends Plugin { + // ClientConfiguration clinit has some classloader problems + // TODO: fix that + static { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + Class.forName("com.amazonaws.ClientConfiguration"); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + return null; + } + }); + } + @Override public String name() { return "repository-s3"; diff --git a/plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy similarity index 84% rename from plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy rename to plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy index e45c1b86ceb..62b29a2b78f 100644 --- a/plugins/lang-expression/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -18,6 +18,7 @@ */ grant { - // needed to generate runtime classes - permission java.lang.RuntimePermission "createClassLoader"; + // needed because of problems in ClientConfiguration + // TODO: get this fixed in aws sdk + permission java.lang.RuntimePermission "getClassLoader"; }; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java index 20ef464e83b..695d2a42321 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java @@ -111,6 +111,8 @@ public class EvilSecurityTests extends ESTestCase { assertExactPermissions(new FilePermission(environment.binFile().toString(), "read,readlink"), permissions); // lib file: ro assertExactPermissions(new FilePermission(environment.libFile().toString(), "read,readlink"), permissions); + // modules file: ro + assertExactPermissions(new FilePermission(environment.modulesFile().toString(), "read,readlink"), permissions); // config file: ro assertExactPermissions(new FilePermission(environment.configFile().toString(), "read,readlink"), permissions); // scripts file: ro diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java index 0e7597e61e8..34a57d7fbcb 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java @@ -630,8 +630,6 @@ public class PluginManagerTests extends ESIntegTestCase { PluginManager.checkForOfficialPlugins("analysis-smartcn"); PluginManager.checkForOfficialPlugins("analysis-stempel"); PluginManager.checkForOfficialPlugins("delete-by-query"); - PluginManager.checkForOfficialPlugins("lang-expression"); - PluginManager.checkForOfficialPlugins("lang-groovy"); PluginManager.checkForOfficialPlugins("lang-javascript"); PluginManager.checkForOfficialPlugins("lang-python"); PluginManager.checkForOfficialPlugins("mapper-attachments"); diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index 864a58baf25..70611aed371 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -21,21 +21,19 @@ import org.elasticsearch.gradle.MavenFilteringHack apply plugin: 'elasticsearch.rest-test' -ext.pluginCount = 0 -for (Project subproj : project.rootProject.subprojects) { - if (subproj.path.startsWith(':plugins:')) { - integTest { - cluster { - // need to get a non-decorated project object, so must re-lookup the project by path - plugin subproj.name, project(subproj.path) - } +ext.pluginsCount = 0 +project.rootProject.subprojects.findAll { it.path.startsWith(':projects:') }.each { subproj -> + integTest { + cluster { + // need to get a non-decorated project object, so must re-lookup the project by path + plugin subproj.name, project(subproj.path) } - pluginCount += 1 } -} + pluginCount += 1 +} ext.expansions = [ - 'expected.plugin.count': pluginCount + 'expected.plugins.count': pluginsCount ] processTestResources { diff --git a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml index dbb09225fce..6a92845a062 100644 --- a/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml +++ b/qa/smoke-test-plugins/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yaml @@ -10,4 +10,4 @@ - do: nodes.info: {} - - length: { nodes.$master.plugins: ${expected.plugin.count} } + - length: { nodes.$master.plugins: ${expected.plugins.count} } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats index 383375f9531..f6a9b22ec3b 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats @@ -92,7 +92,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=elasticsearch install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests stop_elasticsearch_service diff --git a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats index 553f867f60d..048b208105e 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats @@ -86,7 +86,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats index 7f447e51cb3..d27622ffdda 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats @@ -81,7 +81,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy start_elasticsearch_service run_elasticsearch_tests } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats index 6558a3831b3..da7b6a180f1 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats @@ -68,7 +68,6 @@ setup() { # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. install_elasticsearch_test_scripts - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy systemctl start elasticsearch.service wait_for_elasticsearch_status assert_file_exist "/var/run/elasticsearch/elasticsearch.pid" diff --git a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats index 1c5cce59174..fad764eb711 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats @@ -70,7 +70,6 @@ setup() { # Install scripts used to test script filters and search templates before # starting Elasticsearch so we don't have to wait for elasticsearch to scan for # them. - ESPLUGIN_COMMAND_USER=root install_and_check_plugin lang groovy install_elasticsearch_test_scripts service elasticsearch start wait_for_elasticsearch_status diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash index 79e4426095a..48c34fa03af 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash @@ -223,14 +223,6 @@ fi install_and_check_plugin discovery multicast } -@test "[$GROUP] install lang-expression plugin" { - install_and_check_plugin lang expression -} - -@test "[$GROUP] install lang-groovy plugin" { - install_and_check_plugin lang groovy -} - @test "[$GROUP] install javascript plugin" { install_and_check_plugin lang javascript rhino-*.jar } @@ -331,14 +323,6 @@ fi remove_plugin discovery-multicast } -@test "[$GROUP] remove lang-expression plugin" { - remove_plugin lang-expression -} - -@test "[$GROUP] remove lang-groovy plugin" { - remove_plugin lang-groovy -} - @test "[$GROUP] remove javascript plugin" { remove_plugin lang-javascript } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json index 0e6c6ab23f9..c4170c1962a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json @@ -1,7 +1,7 @@ { "indices.forcemerge": { "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-forcemerge.html", - "methods": ["POST", "GET"], + "methods": ["POST"], "url": { "path": "/_forcemerge", "paths": ["/_forcemerge", "/{index}/_forcemerge"], diff --git a/settings.gradle b/settings.gradle index e91f732c7c4..adeabd78171 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,11 +3,14 @@ rootProject.name = 'elasticsearch' List projects = [ 'rest-api-spec', 'core', + 'distribution:integ-test-zip', 'distribution:zip', 'distribution:tar', 'distribution:deb', 'distribution:rpm', 'test-framework', + 'modules:lang-expression', + 'modules:lang-groovy', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', @@ -19,8 +22,6 @@ List projects = [ 'plugins:discovery-gce', 'plugins:discovery-multicast', 'plugins:ingest', - 'plugins:lang-expression', - 'plugins:lang-groovy', 'plugins:lang-javascript', 'plugins:lang-python', 'plugins:mapper-attachments', diff --git a/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 457a283c839..7176916cdfe 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test-framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -30,7 +30,7 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; @@ -741,7 +741,7 @@ public class ElasticsearchAssertions { Assert.assertThat(response.getNodesMap().get(nodeId), notNullValue()); - PluginsInfo plugins = response.getNodesMap().get(nodeId).getPlugins(); + PluginsAndModules plugins = response.getNodesMap().get(nodeId).getPlugins(); Assert.assertThat(plugins, notNullValue()); List pluginNames = filterAndMap(plugins, jvmPluginPredicate, nameFunction); @@ -761,7 +761,7 @@ public class ElasticsearchAssertions { boolean anyHaveUrls = plugins - .getInfos() + .getPluginInfos() .stream() .filter(jvmPluginPredicate.and(sitePluginPredicate.negate())) .map(urlFunction) @@ -791,8 +791,8 @@ public class ElasticsearchAssertions { } } - private static List filterAndMap(PluginsInfo pluginsInfo, Predicate predicate, Function function) { - return pluginsInfo.getInfos().stream().filter(predicate).map(function).collect(Collectors.toList()); + private static List filterAndMap(PluginsAndModules pluginsInfo, Predicate predicate, Function function) { + return pluginsInfo.getPluginInfos().stream().filter(predicate).map(function).collect(Collectors.toList()); } private static Predicate jvmPluginPredicate = p -> p.isJvm(); diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest0IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest0IT.java deleted file mode 100644 index e73bf347093..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest0IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 0 */ -public class Rest0IT extends ESRestTestCase { - public Rest0IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(0, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest1IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest1IT.java deleted file mode 100644 index bc80123debc..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest1IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 1 */ -public class Rest1IT extends ESRestTestCase { - public Rest1IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(1, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest2IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest2IT.java deleted file mode 100644 index a2fb5ad9226..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest2IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 2 */ -public class Rest2IT extends ESRestTestCase { - public Rest2IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(2, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest4IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest4IT.java deleted file mode 100644 index 3910167702a..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest4IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 4 */ -public class Rest4IT extends ESRestTestCase { - public Rest4IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(4, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest5IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest5IT.java deleted file mode 100644 index 748b06c2c2a..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest5IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 5 */ -public class Rest5IT extends ESRestTestCase { - public Rest5IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(5, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest6IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest6IT.java deleted file mode 100644 index e8fbcd4826c..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest6IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 6 */ -public class Rest6IT extends ESRestTestCase { - public Rest6IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(6, 8); - } -} diff --git a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest7IT.java b/test-framework/src/main/java/org/elasticsearch/test/rest/Rest7IT.java deleted file mode 100644 index cf68bdb5606..00000000000 --- a/test-framework/src/main/java/org/elasticsearch/test/rest/Rest7IT.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.parser.RestTestParseException; - -import java.io.IOException; - -/** Rest API tests subset 7 */ -public class Rest7IT extends ESRestTestCase { - public Rest7IT(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); - } - @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { - return createParameters(7, 8); - } -}