Build: Change project attachment into special extra-plugins dir

Currently we use the "gradle project attachment plugin" to support
building elasticsearch as part of another project. However, this plugin
has a number of issues, a large part of which is requiring consistent
use of the projectsPrefix.

This change removes projectsPrefix, and adds support for a special
extra-plugins directory in the root of elasticsearch. Any projects
checked out within this directory will be automatically added to
elasticsearch.

TESTING.asciidoc

@@ -461,3 +461,15 @@ supports a remote debugging option:
 ---------------------------------------------------------------------------
 gradle run --debug-jvm
 ---------------------------------------------------------------------------
+
+== Building with extra plugins
+Additional plugins may be built alongside elasticsearch, where their
+dependency on elasticsearch will be substituted with the local elasticsearch
+build. To add your plugin, check it out into the extra-plugins directory.
+The build will automatically pick it up. You can verify the plugin is
+included as part of the build by checking the projects of the build.
+
+---------------------------------------------------------------------------
+gradle projects
+---------------------------------------------------------------------------
+

build.gradle

@@ -72,12 +72,6 @@ subprojects {
   }
 }
 
-if (hasProperty('projectsPrefix') == false) {
-  allprojects {
-    project.ext['projectsPrefix'] = ''
-  }  
-}
-
 allprojects {
   // injecting groovy property variables into all projects
   project.ext {
@@ -108,11 +102,11 @@ subprojects {
   }
 
   ext.projectSubstitutions = [
-    "org.elasticsearch:rest-api-spec:${version}": "${projectsPrefix}:rest-api-spec",
+    "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
-    "org.elasticsearch:elasticsearch:${version}": "${projectsPrefix}:core",
+    "org.elasticsearch:elasticsearch:${version}": ':core',
-    "org.elasticsearch:test-framework:${version}": "${projectsPrefix}:test-framework",
+    "org.elasticsearch:test-framework:${version}": ':test-framework',
-    "org.elasticsearch.distribution.zip:elasticsearch:${version}": "${projectsPrefix}:distribution:zip",
+    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": "${projectsPrefix}:distribution:tar"
+    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar'
   ]
   configurations.all {
     resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
@@ -169,12 +163,19 @@ allprojects {
   apply plugin: 'idea'
 }
 
+<<<<<<< HEAD
 if (projectsPrefix.isEmpty()) {
   idea {
     project {
       languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava.toString()
       vcs = 'Git'
     }
+=======
+idea {
+  project {
+    languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava
+    vcs = 'Git'
+>>>>>>> Build: Change project attachment into special extra-plugins dir
   }
 }
 

core/build.gradle

@@ -87,7 +87,7 @@ dependencies {
 
   compile "net.java.dev.jna:jna:${versions.jna}", optional
 
-  if (isEclipse == false || project.path == "${projectsPrefix}:core-tests") {
+  if (isEclipse == false || project.path == ":core-tests") {
     testCompile("org.elasticsearch:test-framework:${version}") {
       // tests use the locally compiled version of core
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
@@ -98,7 +98,7 @@ dependencies {
 if (isEclipse) {
   // in eclipse the project is under a fake root, we need to change around the source sets
   sourceSets {
-    if (project.path == "${projectsPrefix}:core") {
+    if (project.path == ":core") {
       main.java.srcDirs = ['java']
       main.resources.srcDirs = ['resources']
     } else {
@@ -117,7 +117,7 @@ forbiddenPatterns {
   exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
 }
 
-if (isEclipse == false || project.path == "${projectsPrefix}:core-tests") {
+if (isEclipse == false || project.path == ":core-tests") {
   task integTest(type: RandomizedTestingTask,
                  group: JavaBasePlugin.VERIFICATION_GROUP,
                  description: 'Multi-node tests',

core/src/test/eclipse-build.gradle

@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project("${projectsPrefix}:core")
+  testCompile project(':core')
 }

distribution/build.gradle

@@ -41,7 +41,7 @@ buildscript {
 allprojects {
   project.ext {
     // this is common configuration for distributions, but we also add it here for the license check to use
-    dependencyFiles = project("${projectsPrefix}:core").configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
+    dependencyFiles = project(':core').configurations.runtime.copyRecursive().exclude(module: 'slf4j-api')
   }
 }
 
@@ -79,7 +79,7 @@ subprojects {
      *****************************************************************************/
     libFiles = copySpec {
       into 'lib'
-      from project("${projectsPrefix}:core").jar
+      from project(':core').jar
       from dependencyFiles
     }
 

extra-plugins/.gitignore

@@ -0,0 +1 @@
+*/

extra-plugins/build.gradle

@@ -0,0 +1,10 @@
+
+// This file exists solely for the purpose of allowing a nice error message
+// if someone tries running a gradle command from within the extra-plugins dir.
+
+println '''
+  Gradle commands are not supported from within the extra-plugins dir.
+  Please run your command either at the root of the elasticsearch checkout
+  or within a specific extra-plugins project.
+'''
+throw new GradleException('Cannot run commands in extra-plugins dir')

qa/vagrant/build.gradle

@@ -42,14 +42,14 @@ repositories {
 }
 
 dependencies {
-  test project(path: "${projectsPrefix}:distribution:tar", configuration: 'archives')
+  test project(path: ':distribution:tar', configuration: 'archives')
-  test project(path: "${projectsPrefix}:distribution:rpm", configuration: 'archives')
+  test project(path: ':distribution:rpm', configuration: 'archives')
-  test project(path: "${projectsPrefix}:distribution:deb", configuration: 'archives')
+  test project(path: ':distribution:deb', configuration: 'archives')
 
   // Collect all the plugins
   for (Project subproj : project.rootProject.subprojects) {
     if (subproj.path.startsWith(':plugins:')) {
-      test project("${projectsPrefix}${subproj.path}")
+      test project("${subproj.path}")
     }
   }
 

settings.gradle

@@ -52,3 +52,27 @@ if (isEclipse) {
   project(":core-tests").buildFileName = 'eclipse-build.gradle'
 }
 
+/**
+  * Iterates over sub directories, looking for build.gradle, and adds a project if found
+  * for that dir with the given path prefix. Note that this requires each level
+  * of the dir hiearchy to have a build.gradle. Otherwise we would have to iterate
+  * all files/directories in the source tree to find all projects.
+  */
+void addSubProjects(String path, File dir) {
+  if (dir.isDirectory() == false) return;
+  if (new File(dir, 'build.gradle').exists() == false) return;
+
+  String projectName = "${path}:${dir.name}"
+  include projectName
+  project(projectName).projectDir = dir
+
+  for (File subdir : dir.listFiles()) {
+    addSubProjects(projectName, subdir)
+  }
+}
+
+File extraPlugins = new File(rootProject.projectDir, 'extra-plugins')
+for (File extraPluginDir : extraPlugins.listFiles()) {
+  addSubProjects('', extraPluginDir)
+}
+