Build: Switch to maven-publish plugin

The build currently uses the old maven support in gradle. This commit
switches to use the newer maven-publish plugin. This will allow future
changes, for example, easily publishing to artifactory.

An additional part of this change makes publishing of build-tools part
of the normal publishing, instead of requiring a separate upload step
from within buildSrc. That also sets us up for a follow up to enable
precomit checks on the buildSrc code itself.

build.gradle

@@ -17,54 +17,61 @@
  * under the License.
  */
 
-import com.bmuschko.gradle.nexus.NexusPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
 // common maven publishing configuration
 subprojects {
+  if (path.startsWith(':x-plugins')) {
+    // don't try to configure publshing for extra plugins attached to this build
+    return
+  }
   group = 'org.elasticsearch'
   version = org.elasticsearch.gradle.VersionProperties.elasticsearch
 
-  plugins.withType(NexusPlugin).whenPluginAdded {
+  plugins.withType(MavenPublishPlugin).whenPluginAdded {
-    modifyPom {
+    publishing {
-      project {
+      publications {
-        url 'https://github.com/elastic/elasticsearch'
+        // add license information to generated poms
-        inceptionYear '2009'
+        all {
+          pom.withXml { XmlProvider xml ->
+            Node node = xml.asNode()
+            node.appendNode('inceptionYear', '2009')
 
-        scm {
+            Node license = node.appendNode('licenses').appendNode('license')
-          url 'https://github.com/elastic/elasticsearch'
+            license.appendNode('name', 'The Apache Software License, Version 2.0')
-          connection 'scm:https://elastic@github.com/elastic/elasticsearch'
+            license.appendNode('url', 'http://www.apache.org/licenses/LICENSE-2.0.txt')
-          developerConnection 'scm:git://github.com/elastic/elasticsearch.git'
+            license.appendNode('distribution', 'repo')
-        }
-
-        licenses {
-          license {
-            name 'The Apache Software License, Version 2.0'
-            url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
-            distribution 'repo'
           }
         }
       }
-    }
+      repositories.maven {
-    extraArchive {
+        name 'sonatype'
-      javadoc = true
+        if (version.endsWith('-SNAPSHOT')) {
-      tests = false
+          url 'https://oss.sonatype.org/content/repositories/snapshots/'
-    }
+        } else {
-    // we have our own username/password prompts so that they only happen once
+          url 'https://oss.sonatype.org/service/local/staging/deploy/maven2/'
-    // TODO: add gpg signing prompts
-    project.gradle.taskGraph.whenReady { taskGraph ->
-      if (taskGraph.allTasks.any { it.name == 'uploadArchives' }) {
-        Console console = System.console()
-        if (project.hasProperty('nexusUsername') == false) {
-          String nexusUsername = console.readLine('\nNexus username: ')
-          project.rootProject.allprojects.each {
-            it.ext.nexusUsername = nexusUsername
-          }
         }
-        if (project.hasProperty('nexusPassword') == false) {
+
-          String nexusPassword = new String(console.readPassword('\nNexus password: '))
+        // It would be nice to pass a custom impl of PasswordCredentials
-          project.rootProject.allprojects.each {
+        // that could lazily read username/password from the console if not
-            it.ext.nexusPassword = nexusPassword
+        // present as properties. However, gradle's credential handling is
+        // completely broken for custom impls. It checks that the class
+        // passed in is exactly PasswordCredentials or AwsCredentials.
+        // So instead, we must rely on heuristics of "are we publishing"
+        // by inspecting the command line, stash the credentials
+        // once read in the root project, and set them on each project
+        if (gradle.startParameter.taskNames.contains('publish')) {
+          Console console = System.console()
+          if (project.rootProject.hasProperty('nexusUsername') == false) {
+            project.rootProject.ext.nexusUsername = console.readLine('\nNexus username: ')
+          }
+          if (project.rootProject.hasProperty('nexusPassword') == false) {
+            project.rootProject.ext.nexusPassword = new String(console.readPassword("\nNexus password: "))
+          }
+
+          credentials {
+            username = project.rootProject.nexusUsername 
+            password = project.rootProject.nexusPassword
           }
         }
       }
@@ -72,6 +79,7 @@ subprojects {
   }
 }
 
+
 allprojects {
   // injecting groovy property variables into all projects
   project.ext {

buildSrc/.gitignore

@@ -0,0 +1 @@
+build-bootstrap/

buildSrc/build.gradle

@@ -1,5 +1,3 @@
-import java.nio.file.Files
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -19,25 +17,21 @@ import java.nio.file.Files
  * under the License.
  */
 
-// we must use buildscript + apply so that an external plugin
+import java.nio.file.Files
-// can apply this file, since the plugins directive is not
+
-// supported through file includes
-buildscript {
-  repositories {
-    jcenter()
-  }
-  dependencies {
-    classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1'
-  }
-}
 apply plugin: 'groovy'
-apply plugin: 'com.bmuschko.nexus'
-// TODO: move common IDE configuration to a common file to include
-apply plugin: 'idea'
-apply plugin: 'eclipse'
 
 group = 'org.elasticsearch.gradle'
-archivesBaseName = 'build-tools'
+
+if (project == rootProject) {
+  // change the build dir used during build init, so that doing a clean
+  // won't wipe out the buildscript jar
+  buildDir = 'build-bootstrap'
+}
+
+/*****************************************************************************
+ *         Propagating version.properties to the rest of the build           *
+ *****************************************************************************/
 
 Properties props = new Properties()
 props.load(project.file('version.properties').newDataInputStream())
@@ -51,32 +45,6 @@ if (snapshot) {
   props.put("elasticsearch", version);
 }
 
-
-repositories {
-  mavenCentral()
-  maven {
-    name 'sonatype-snapshots'
-    url "https://oss.sonatype.org/content/repositories/snapshots/"
-  }
-  jcenter()
-}
-
-dependencies {
-  compile gradleApi()
-  compile localGroovy()
-  compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
-  compile("junit:junit:${props.getProperty('junit')}") {
-    transitive = false
-  }
-  compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
-  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
-  compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
-  compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.0'
-  compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
-  compile 'org.apache.rat:apache-rat:0.11'
-}
-
 File tempPropertiesFile = new File(project.buildDir, "version.properties")
 task writeVersionProperties {
   inputs.properties(props)
@@ -95,31 +63,92 @@ processResources {
   from tempPropertiesFile
 }
 
-extraArchive {
+/*****************************************************************************
-  javadoc = false
+ *                    Dependencies used by the entire build                  *
-  tests = false
+ *****************************************************************************/
+
+repositories {
+  jcenter()
 }
 
-idea {
+dependencies {
-  module {
+  compile gradleApi()
-    inheritOutputDirs = false
+  compile localGroovy()
-    outputDir = file('build-idea/classes/main')
+  compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
-    testOutputDir = file('build-idea/classes/test')
+  compile("junit:junit:${props.getProperty('junit')}") {
+    transitive = false
+  }
+  compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
+  compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
+  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
+  compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
+  compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
+  compile 'de.thetaphi:forbiddenapis:2.0'
+  compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
+  compile 'org.apache.rat:apache-rat:0.11'
+}
+
+
+/*****************************************************************************
+ *                    Bootstrap repositories and IDE setup                   *
+ *****************************************************************************/
+// this will only happen when buildSrc is built on its own during build init
+if (project == rootProject) {
+
+  repositories {
+    mavenCentral()
+    maven {
+      name 'sonatype-snapshots'
+      url "https://oss.sonatype.org/content/repositories/snapshots/"
+    }
+  }
+
+  apply plugin: 'idea'
+  apply plugin: 'eclipse'
+
+  idea {
+    module {
+      inheritOutputDirs = false
+      outputDir = file('build-idea/classes/main')
+      testOutputDir = file('build-idea/classes/test')
+    }
+  }
+
+  eclipse {
+    classpath {
+      defaultOutputDir = file('build-eclipse')
+    }
+  }
+
+  task copyEclipseSettings(type: Copy) {
+    from project.file('src/main/resources/eclipse.settings')
+    into '.settings'
+  }
+  // otherwise .settings is not nuked entirely
+  tasks.cleanEclipse {
+    delete '.settings'
+  }
+  tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
+}
+
+/*****************************************************************************
+ *                           Normal project checks                           *
+ *****************************************************************************/
+
+// this happens when included as a normal project in the build, which we do
+// to enforce precommit checks like forbidden apis, as well as setup publishing
+if (project != rootProject) {
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+  apply plugin: 'nebula.source-jar'
+  apply plugin: 'nebula.javadoc-jar'
+
+  publishing {
+    publications {
+      nebula {
+        artifactId 'build-tools'
+      }
+    }
   }
 }
 
-eclipse {
-  classpath {
-    defaultOutputDir = file('build-eclipse')
-  }
-}
-
-task copyEclipseSettings(type: Copy) {
-  from project.file('src/main/resources/eclipse.settings')
-  into '.settings'
-}
-// otherwise .settings is not nuked entirely
-tasks.cleanEclipse {
-  delete '.settings'
-}
-tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -32,7 +32,8 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.maven.MavenPom
+import org.gradle.api.publish.maven.MavenPublication
+import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.internal.jvm.Jvm
@@ -60,7 +61,6 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('nebula.info-java')
         project.pluginManager.apply('nebula.info-scm')
         project.pluginManager.apply('nebula.info-jar')
-        project.pluginManager.apply('com.bmuschko.nexus')
         project.pluginManager.apply(ProvidedBasePlugin)
 
         globalBuildInfo(project)
@@ -68,6 +68,7 @@ class BuildPlugin implements Plugin<Project> {
         configureConfigurations(project)
         project.ext.versions = VersionProperties.versions
         configureCompile(project)
+        configurePublishing(project)
 
         configureTest(project)
         configurePrecommit(project)
@@ -260,48 +261,6 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
         project.configurations.provided.dependencies.all(disableTransitiveDeps)
-
-        // add exclusions to the pom directly, for each of the transitive deps of this project's deps
-        project.modifyPom { MavenPom pom ->
-            pom.withXml { XmlProvider xml ->
-                // first find if we have dependencies at all, and grab the node
-                NodeList depsNodes = xml.asNode().get('dependencies')
-                if (depsNodes.isEmpty()) {
-                    return
-                }
-
-                // check each dependency for any transitive deps
-                for (Node depNode : depsNodes.get(0).children()) {
-                    String groupId = depNode.get('groupId').get(0).text()
-                    String artifactId = depNode.get('artifactId').get(0).text()
-                    String version = depNode.get('version').get(0).text()
-
-                    // collect the transitive deps now that we know what this dependency is
-                    String depConfig = transitiveDepConfigName(groupId, artifactId, version)
-                    Configuration configuration = project.configurations.findByName(depConfig)
-                    if (configuration == null) {
-                        continue // we did not make this dep non-transitive
-                    }
-                    Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
-                    if (artifacts.size() <= 1) {
-                        // this dep has no transitive deps (or the only artifact is itself)
-                        continue
-                    }
-
-                    // we now know we have something to exclude, so add the exclusion elements
-                    Node exclusions = depNode.appendNode('exclusions')
-                    for (ResolvedArtifact transitiveArtifact : artifacts) {
-                        ModuleVersionIdentifier transitiveDep = transitiveArtifact.moduleVersion.id
-                        if (transitiveDep.group == groupId && transitiveDep.name == artifactId) {
-                            continue; // don't exclude the dependency itself!
-                        }
-                        Node exclusion = exclusions.appendNode('exclusion')
-                        exclusion.appendNode('groupId', transitiveDep.group)
-                        exclusion.appendNode('artifactId', transitiveDep.name)
-                    }
-                }
-            }
-        }
     }
 
     /** Adds repositores used by ES dependencies */
@@ -375,6 +334,59 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    /**
+     * Adds a hook to all publications that will effectively make the maven pom transitive dependency free.
+     */
+    private static void configurePublishing(Project project) {
+        project.plugins.withType(MavenPublishPlugin.class).whenPluginAdded {
+            project.publishing {
+                publications {
+                    all { MavenPublication publication -> // we only deal with maven
+                        // add exclusions to the pom directly, for each of the transitive deps of this project's deps
+                        publication.pom.withXml { XmlProvider xml ->
+                            // first find if we have dependencies at all, and grab the node
+                            NodeList depsNodes = xml.asNode().get('dependencies')
+                            if (depsNodes.isEmpty()) {
+                                return
+                            }
+
+                            // check each dependency for any transitive deps
+                            for (Node depNode : depsNodes.get(0).children()) {
+                                String groupId = depNode.get('groupId').get(0).text()
+                                String artifactId = depNode.get('artifactId').get(0).text()
+                                String version = depNode.get('version').get(0).text()
+
+                                // collect the transitive deps now that we know what this dependency is
+                                String depConfig = transitiveDepConfigName(groupId, artifactId, version)
+                                Configuration configuration = project.configurations.findByName(depConfig)
+                                if (configuration == null) {
+                                    continue // we did not make this dep non-transitive
+                                }
+                                Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
+                                if (artifacts.size() <= 1) {
+                                    // this dep has no transitive deps (or the only artifact is itself)
+                                    continue
+                                }
+
+                                // we now know we have something to exclude, so add the exclusion elements
+                                Node exclusions = depNode.appendNode('exclusions')
+                                for (ResolvedArtifact transitiveArtifact : artifacts) {
+                                    ModuleVersionIdentifier transitiveDep = transitiveArtifact.moduleVersion.id
+                                    if (transitiveDep.group == groupId && transitiveDep.name == artifactId) {
+                                        continue; // don't exclude the dependency itself!
+                                    }
+                                    Node exclusion = exclusions.appendNode('exclusion')
+                                    exclusion.appendNode('groupId', transitiveDep.group)
+                                    exclusion.appendNode('artifactId', transitiveDep.name)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     /** Returns a closure of common configuration shared by unit and integration tests. */
     static Closure commonTestConfig(Project project) {
         return {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -18,11 +18,14 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import nebula.plugin.publishing.maven.MavenManifestPlugin
+import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Dependency
+import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
 
@@ -34,6 +37,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
+
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
@@ -50,6 +54,10 @@ public class PluginBuildPlugin extends BuildPlugin {
             } else {
                 project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
                 project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+
+                if (project.pluginProperties.extension.publish) {
+                    configurePublishing(project)
+                }
             }
 
             project.namingConventions {
@@ -59,6 +67,7 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
         createIntegTestTask(project)
         createBundleTask(project)
+        configurePublishing(project)
         project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
@@ -125,4 +134,32 @@ public class PluginBuildPlugin extends BuildPlugin {
         project.configurations.getByName('default').extendsFrom = []
         project.artifacts.add('default', bundle)
     }
+
+    /**
+     * Adds the plugin jar and zip as publications.
+     */
+    private static void configurePublishing(Project project) {
+        project.plugins.apply(MavenScmPlugin.class)
+        project.plugins.apply(MavenManifestPlugin.class)
+
+        project.publishing {
+            publications {
+                nebula {
+                    artifact project.bundlePlugin
+                    pom.withXml {
+                        // overwrite the name/description in the pom nebula set up
+                        Node root = asNode()
+                        for (Node node : root.children()) {
+                            if (node.name() == 'name') {
+                                node.setValue(name)
+                            } else if (node.name() == 'description') {
+                                node.setValue(project.pluginProperties.extension.description)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -42,6 +42,10 @@ class PluginPropertiesExtension {
     @Input
     boolean isolated = true
 
+    /** Whether the plugin should be published to maven. */
+    @Input
+    boolean publish = false
+
     PluginPropertiesExtension(Project project) {
         name = project.name
         version = project.version

core/build.gradle

@@ -22,10 +22,19 @@ import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
 
 apply plugin: 'elasticsearch.build'
-apply plugin: 'com.bmuschko.nexus'
 apply plugin: 'nebula.optional-base'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+apply plugin: 'nebula.source-jar'
+apply plugin: 'nebula.javadoc-jar'
 
-archivesBaseName = 'elasticsearch'
+publishing {
+  publications {
+    nebula {
+      artifactId 'elasticsearch'
+    }
+  }
+}
 
 dependencies {
 

distribution/build.gradle

@@ -157,6 +157,19 @@ subprojects {
       MavenFilteringHack.filter(it, expansions)
     }
   }
+
+  /*****************************************************************************
+   *                           Publishing setup                                *
+   *****************************************************************************/
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+  publishing {
+    publications {
+      nebula {
+        artifactId 'elasticsearch'
+      }
+    }
+  }
 }
 
 /*****************************************************************************

distribution/deb/build.gradle

@@ -36,7 +36,14 @@ task buildDeb(type: Deb) {
 
 artifacts {
   'default' buildDeb
-  archives buildDeb
+}
+
+publishing {
+  publications {
+    nebula {
+      artifact buildDeb
+    }
+  }
 }
 
 integTest {

distribution/integ-test-zip/build.gradle

@@ -24,7 +24,14 @@ task buildZip(type: Zip) {
 
 artifacts {
   'default' buildZip
-  archives buildZip
+}
+
+publishing {
+  publications {
+    nebula {
+      artifact buildZip
+    }
+  }
 }
 
 integTest.dependsOn buildZip

distribution/rpm/build.gradle

@@ -33,7 +33,14 @@ task buildRpm(type: Rpm) {
 
 artifacts {
   'default' buildRpm
-  archives buildRpm
+}
+
+publishing {
+  publications {
+    nebula {
+      artifact buildRpm
+    }
+  }
 }
 
 integTest {

distribution/tar/build.gradle

@@ -26,5 +26,12 @@ task buildTar(type: Tar) {
 
 artifacts {
   'default' buildTar
-  archives buildTar
+}
+
+publishing {
+  publications {
+    nebula {
+      artifact buildTar
+    }
+  }
 }

distribution/zip/build.gradle

@@ -24,7 +24,14 @@ task buildZip(type: Zip) {
 
 artifacts {
   'default' buildZip
-  archives buildZip
+}
+
+publishing {
+  publications {
+    nebula {
+      artifact buildZip
+    }
+  }
 }
 
 integTest.dependsOn buildZip

modules/build.gradle

@@ -40,8 +40,4 @@ subprojects {
       throw new InvalidModelException("Modules cannot disable isolation")
     }
   }
-
-  // these are implementation details of our build, no need to publish them!
-  install.enabled = false
-  uploadArchives.enabled = false
 }

plugins/build.gradle

@@ -27,5 +27,7 @@ configure(subprojects.findAll { it.parent.path == project.path }) {
   esplugin {
     // for local ES plugins, the name of the plugin is the same as the directory
     name project.name
+    // only publish non examples
+    publish project.name.contains('example') == false
   }
 }

settings.gradle

@@ -1,6 +1,7 @@
 rootProject.name = 'elasticsearch'
 
 List projects = [
+  'build-tools',
   'rest-api-spec',
   'core',
   'distribution:integ-test-zip',
@@ -58,6 +59,8 @@ if (isEclipse) {
 
 include projects.toArray(new String[0])
 
+project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
+
 if (isEclipse) {
   project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main')
   project(":core").buildFileName = 'eclipse-build.gradle'

test/build.gradle

@@ -41,4 +41,9 @@ subprojects {
   // TODO: why is the test framework pulled in...
   forbiddenApisMain.enabled = false
   jarHell.enabled = false
+
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+  apply plugin: 'nebula.source-jar'
+  apply plugin: 'nebula.javadoc-jar'
 }