Add snapshot only test modules (#61954)

This commit adds external test modules. These are modules meant for
external systems to test edge cases in elasticsearch, but only within
snapshots. They are not meant to be used in production, so protections
are also added from their accidental inclusion in release builds.

Note that this commit does not actually add any new modules, it only
adds the infrastructure for the new modules, under
`test/external-modules`.

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -71,7 +71,7 @@ class BuildPlugin implements Plugin<Project> {
         configureLicenseAndNotice(project)
         project.pluginManager.apply('elasticsearch.publish')
         project.pluginManager.apply(DependenciesInfoPlugin)
-        
+
         PrecommitTasks.create(project, true)
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java

@@ -21,7 +21,6 @@ package org.elasticsearch.gradle.test.rest;
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
-import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.test.RestIntegTestTask;
 import org.gradle.api.Project;
 import org.gradle.api.plugins.JavaBasePlugin;
@@ -60,27 +59,12 @@ public class RestTestUtil {
         project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
             Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
             testTask.dependsOn(bundle);
-            if (project.getPath().startsWith(":modules:")) {
+            if (project.getPath().contains("modules:")) {
                 testTask.getClusters().forEach(c -> c.module(bundle.getArchiveFile()));
             } else {
                 testTask.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())));
             }
         });
-
-        // es-plugins may declare dependencies on additional modules, add those to the test cluster too.
-        project.afterEvaluate(p -> {
-            PluginPropertiesExtension pluginPropertiesExtension = project.getExtensions().findByType(PluginPropertiesExtension.class);
-            if (pluginPropertiesExtension != null) { // not all projects are defined as plugins
-                pluginPropertiesExtension.getExtendedPlugins().forEach(pluginName -> {
-                    Project extensionProject = project.getProject().findProject(":modules:" + pluginName);
-                    if (extensionProject != null) { // extension plugin may be defined, but not required to be a module
-                        Zip extensionBundle = (Zip) extensionProject.getTasks().getByName("bundlePlugin");
-                        testTask.dependsOn(extensionBundle);
-                        testTask.getClusters().forEach(c -> c.module(extensionBundle.getArchiveFile()));
-                    }
-                });
-            }
-        });
     }
 
     /**

distribution/build.gradle

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.DependenciesInfoTask
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.info.BuildParams
 
 import java.nio.file.Files
 import java.nio.file.Path
@@ -86,6 +87,7 @@ String ossOutputs = 'build/outputs/oss'
 String defaultOutputs = 'build/outputs/default'
 String systemdOutputs = 'build/outputs/systemd'
 String transportOutputs = 'build/outputs/transport-only'
+String externalTestOutputs = 'build/outputs/external-test'
 
 tasks.register("processOssOutputs", Sync) {
   into ossOutputs
@@ -100,6 +102,10 @@ tasks.register("processSystemdOutputs", Sync) {
   into systemdOutputs
 }
 
+tasks.register("processExternalTestOutputs", Sync) {
+  into externalTestOutputs
+}
+
 // Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
 // All transport modules are included so that they may be randomized for testing
 tasks.register("processTransportOutputs", Sync) {
@@ -139,11 +145,22 @@ tasks.register("buildTransportModules") {
   dependsOn "processTransportOutputs"
   outputs.dir "${transportOutputs}/modules"
 }
+tasks.register("buildExternalTestModules") {
+  dependsOn "processExternalTestOutputs"
+  outputs.dir "${externalTestOutputs}/modules"
+}
+
+Configuration moduleZip(Project module) {
+  Dependency dep = project.dependencies.project(path: module.path, configuration: 'zip')
+  Configuration config = project.configurations.detachedConfiguration(dep)
+  return config
+}
 
 void copyModule(Sync copyTask, Project module) {
+  Configuration moduleConfig = moduleZip(module)
   copyTask.configure {
-    dependsOn "${module.path}:bundlePlugin"
-    from({ zipTree(module.bundlePlugin.outputs.files.singleFile) }) {
+    dependsOn moduleConfig
+    from({ zipTree(moduleConfig.singleFile) }) {
       includeEmptyDirs false
 
       // these are handled separately in the log4j config tasks below
@@ -196,9 +213,10 @@ tasks.named("buildDefaultLog4jConfig").configure {
 
 // copy log4j2.properties from modules that have it
 void copyLog4jProperties(Task buildTask, Project module) {
-  buildTask.dependsOn "${module.path}:bundlePlugin"
+  Configuration moduleConfig = moduleZip(module)
+  buildTask.dependsOn moduleConfig
   buildTask.doFirst {
-    FileTree tree = zipTree(module.bundlePlugin.outputs.files.singleFile)
+    FileTree tree = zipTree(moduleConfig.singleFile)
     FileTree filtered = tree.matching {
       include 'config/log4j2.properties'
       include '*/config/log4j2.properties' // could be in a bundled plugin
@@ -256,6 +274,10 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
 
 copyModule(processSystemdOutputs, project(':modules:systemd'))
 
+project(':test:external-modules').subprojects.each { Project testModule ->
+  copyModule(processExternalTestOutputs, testModule)
+}
+
 configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   apply plugin: 'elasticsearch.jdk-download'
@@ -361,6 +383,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             exclude "**/platform/${excludePlatform}/**"
           }
         }
+        if (BuildParams.isSnapshotBuild()) {
+          from(project(':distribution').buildExternalTestModules)
+        }
         if (project.path.startsWith(':distribution:packages')) {
           from(project(':distribution').buildSystemdModule)
         }

server/src/main/java/org/elasticsearch/plugins/PluginsService.java

@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.util.SPIClassIterator;
+import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
@@ -407,6 +408,9 @@ public class PluginsService implements ReportingService<PluginsAndModules> {
         if (bundles.add(bundle) == false) {
             throw new IllegalStateException("duplicate " + type + ": " + info);
         }
+        if (type.equals("module") && info.getName().startsWith("test-") && Build.CURRENT.isSnapshot() == false) {
+            throw new IllegalStateException("external test module [" + plugin.getFileName() + "] found in non-snapshot build");
+        }
         return bundle;
     }
 

settings.gradle

@@ -114,6 +114,7 @@ addSubProjects('', new File(rootProject.projectDir, 'libs'))
 addSubProjects('', new File(rootProject.projectDir, 'modules'))
 addSubProjects('', new File(rootProject.projectDir, 'plugins'))
 addSubProjects('', new File(rootProject.projectDir, 'qa'))
+addSubProjects('test', new File(rootProject.projectDir, 'test/external-modules'))
 addSubProjects('', new File(rootProject.projectDir, 'x-pack'))
 
 List startTasks = gradle.startParameter.taskNames
@@ -127,6 +128,10 @@ project(":libs").children.each { libsProject ->
   libsProject.name = "elasticsearch-${libsProject.name}"
 }
 
+project(":test:external-modules").children.each { testProject ->
+  testProject.name = "test-${testProject.name}"
+}
+
 // look for extra plugins for elasticsearch
 File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
 if (extraProjects.exists()) {

test/build.gradle

@@ -18,17 +18,5 @@
  */
 
 subprojects {
-  // fixtures is just intermediate parent project
-  if (name == 'fixtures') return
-
   group = 'org.elasticsearch.test'
-  apply plugin: 'elasticsearch.build'
-  apply plugin: 'elasticsearch.publish'
-
-  // TODO: should we have licenses for our test deps?
-  tasks.named("dependencyLicenses").configure { it.enabled = false }
-  dependenciesInfo.enabled = false
-
-  // TODO: why is the test framework pulled in...
-  jarHell.enabled = false
 }

test/external-modules/build.gradle

@@ -0,0 +1,10 @@
+
+subprojects {
+  apply plugin: 'elasticsearch.esplugin'
+
+  esplugin {
+    name it.name
+    licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
+    noticeFile rootProject.file('NOTICE.txt')
+  }
+}

test/fixtures/azure-fixture/build.gradle

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 apply plugin: 'elasticsearch.test.fixtures'
 
 description = 'Fixture for Azure external service'

test/fixtures/build.gradle

@@ -1,4 +1,9 @@
+
 subprojects {
-  // fixtures are mostly external and by default we don't want to check forbidden apis
-  disableTasks('forbiddenApisMain')
+  // fixtures don't have tests, these are external projects used by the build
+  pluginManager.withPlugin('java') {
+    tasks.named('test').configure {
+      enabled = false
+    }
+  }
 }

test/fixtures/gcs-fixture/build.gradle

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 apply plugin: 'elasticsearch.test.fixtures'
 
 description = 'Fixture for Google Cloud Storage service'

test/fixtures/hdfs-fixture/build.gradle

@@ -17,11 +17,8 @@
  * under the License.
  */
 
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 
 dependencies {
   api "org.apache.hadoop:hadoop-minicluster:2.8.5"
 }
-
-test.enabled = false
-thirdPartyAudit.enabled = false

test/fixtures/krb5kdc-fixture/build.gradle

@@ -18,12 +18,6 @@
  */
 apply plugin: 'elasticsearch.test.fixtures'
 
-// installKDC uses tabs in it for the Kerberos ACL file.
-// Ignore it for pattern checking.
-forbiddenPatterns {
-  exclude "**/installkdc.sh"
-}
-
 List<String> services = ["peppa", "hdfs"]
 
 preProcessFixture.doLast {
@@ -49,5 +43,3 @@ postProcessFixture {
 
 project.ext.krb5Conf = { service -> file("$testFixturesDir/shared/${service}/krb5.conf") }
 project.ext.krb5Keytabs = { service, fileName -> file("$testFixturesDir/shared/${service}/keytabs/${fileName}") }
-
-test.enabled = false

test/fixtures/minio-fixture/build.gradle

@@ -16,9 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.test.fixtures'
 
 description = 'Fixture for MinIO Storage service'
-test.enabled = false
 

test/fixtures/old-elasticsearch/build.gradle

@@ -23,7 +23,7 @@ These need special handling because they do not support writing
 a "ports" file with the port on which Elasticsearch is running.
 """
 
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 test.enabled = false
 
 dependencies {

test/fixtures/s3-fixture/build.gradle

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.java'
 apply plugin: 'elasticsearch.test.fixtures'
 
 description = 'Fixture for S3 Storage service'

test/framework/build.gradle

@@ -18,6 +18,9 @@
  */
 import org.elasticsearch.gradle.info.BuildParams;
 
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.publish'
+
 dependencies {
   api project(":client:rest")
   api project(":client:sniffer")

test/logger-usage/build.gradle

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.java'
+
 dependencies {
   api 'org.ow2.asm:asm:7.1'
   api 'org.ow2.asm:asm-tree:7.1'
@@ -24,28 +26,3 @@ dependencies {
   api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   testImplementation project(":test:framework")
 }
-
-tasks.named("loggerUsageCheck").configure {
-  enabled = false
-}
-
-tasks.named('forbiddenApisMain').configure {
-  replaceSignatureFiles 'jdk-signatures' // does not depend on core, only jdk signatures
-}
-
-jarHell.enabled = true // disabled by parent project
-
-tasks.named("thirdPartyAudit").configure {
-  ignoreMissingClasses(
-          // log4j
-          'org.osgi.framework.AdaptPermission',
-          'org.osgi.framework.AdminPermission',
-          'org.osgi.framework.Bundle',
-          'org.osgi.framework.BundleActivator',
-          'org.osgi.framework.BundleContext',
-          'org.osgi.framework.BundleEvent',
-          'org.osgi.framework.SynchronousBundleListener',
-          'org.osgi.framework.wiring.BundleWire',
-          'org.osgi.framework.wiring.BundleWiring'
-  )
-}