From 884c3fd7caaf38b33fe4458ba62c7d3f9247287c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 10 Apr 2019 10:43:01 -0700 Subject: [PATCH] Use separate tasks per bwc artifact (#40703) (#40826) This commit changes the bwc builds from a single task for a branch to a task for each bwc artifact. This reduces the bwc build time when only needing a specific artifact, for example when running cluster restart tests on a mac, the windows artifacts or rpm/debs are not needed. --- distribution/bwc/build.gradle | 115 ++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 2a581cf1480..8285d8dae2b 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -120,46 +120,8 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased } } - Map artifactFiles = [:] - List projectDirs = [] - List projects = ['deb', 'rpm'] - if (bwcVersion.onOrAfter('7.0.0')) { - projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) - } else { - projects.add('zip') - } - - for (String projectName : projects) { - String baseDir = "distribution" - String classifier = "" - String extension = projectName - if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { - int index = projectName.indexOf('-') - classifier = "-${projectName.substring(0, index)}-x86_64" - extension = projectName.substring(index + 1) - if (extension.equals('tar')) { - extension += '.gz' - } - } - if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) { - classifier = "-amd64" - } - if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) { - classifier = "-x86_64" - } - if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' - // add oss variant first - projectDirs.add("${baseDir}/oss-${projectName}") - artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) - } - projectDirs.add("${baseDir}/${projectName}") - artifactFiles.put(projectName, - file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) - } - Closure createRunBwcGradleTask = { name, extraConfig -> - task "$name"(type: Exec) { + return tasks.create(name: "$name", type: Exec) { dependsOn checkoutBwcBranch, writeBuildMetadata workingDir = checkoutDir doFirst { @@ -217,28 +179,75 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased } } - createRunBwcGradleTask("buildBwcVersion") { - for (String dir : projectDirs) { - args ":${dir.replace('/', ':')}:assemble" - } - doLast { - List missing = artifactFiles.values().grep { file -> - false == file.exists() - } - if (false == missing.empty) { - throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected files ${missing}") - } - } + Closure buildBwcTaskName = { projectName -> + return "buildBwc${projectName.replaceAll(/-\w/){ it[1].toUpperCase() }.capitalize()}" } + task buildBwc {} + + Closure createBuildBwcTask = { projectName, projectDir, projectArtifact -> + Task bwcTask = createRunBwcGradleTask(buildBwcTaskName(projectName)) { + args ":${projectDir.replace('/', ':')}:assemble" + doLast { + if (projectArtifact.exists() == false) { + throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}") + } + } + } + buildBwc.dependsOn bwcTask + } + + Map artifactFiles = [:] + List projectDirs = [] + List projects = ['deb', 'rpm'] + if (bwcVersion.onOrAfter('7.0.0')) { + projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) + } else { + projects.add('zip') + } + + for (String projectName : projects) { + String baseDir = "distribution" + String classifier = "" + String extension = projectName + if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { + int index = projectName.indexOf('-') + classifier = "-${projectName.substring(0, index)}-x86_64" + extension = projectName.substring(index + 1) + if (extension.equals('tar')) { + extension += '.gz' + } + } + if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) { + classifier = "-amd64" + } + if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) { + classifier = "-x86_64" + } + if (bwcVersion.onOrAfter('6.3.0')) { + baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' + // add oss variant first + projectDirs.add("${baseDir}/oss-${projectName}") + File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}") + artifactFiles.put("oss-" + projectName, ossProjectArtifact) + createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact) + } + projectDirs.add("${baseDir}/${projectName}") + File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}") + artifactFiles.put(projectName, projectArtifact) + + createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact) + } + + createRunBwcGradleTask("resolveAllBwcDependencies") { args 'resolveAllDependencies' } - resolveAllDependencies.dependsOn resolveAllBwcDependencies for (e in artifactFiles) { String projectName = e.key + String buildBwcTask = buildBwcTaskName(projectName) File artifactFile = e.value String artifactFileName = artifactFile.name String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' @@ -251,7 +260,7 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased } configurations.create(projectName) artifacts { - it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcVersion]) + it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask]) } } // make sure no dependencies were added to assemble; we want it to be a no-op