Extract distribution archive checks into plugin (7.x backport) (#61567)

- Added test coverage
- Removes build script cluttering
- Splits archive building and archive checking logic
- only rely on boost for now for ML licenses(tbd)
- Use Gradle build-in untar and unzip support

* Handle dynamic versions in func tests assertions
This commit is contained in:
Rene Groeschke 2020-08-26 15:04:12 +02:00 committed by GitHub
parent fac66a7528
commit 3a8cfdc1f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 490 additions and 102 deletions

View File

@ -52,12 +52,14 @@ abstract class AbstractGradleFuncTest extends Specification {
}
def assertOutputContains(String givenOutput, String expected) {
assert normalizedString(givenOutput).contains(normalizedString(expected))
assert normalizedOutput(givenOutput).contains(normalizedOutput(expected))
true
}
String normalizedString(String input) {
return input.readLines().join("\n")
String normalizedOutput(String input) {
return input.readLines()
.collect {it.replaceAll(testProjectDir.root.canonicalPath, ".") }
.join("\n")
}
File file(String path) {

View File

@ -0,0 +1,173 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.internal
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
import org.gradle.testkit.runner.TaskOutcome
import spock.lang.Unroll
class InternalDistributionArchiveCheckPluginFuncTest extends AbstractGradleFuncTest {
def setup() {
["darwin-zip", 'darwin-tar'].each { projName ->
settingsFile << """
include ':${projName}'
"""
file("${projName}/build.gradle") << """
plugins {
id 'elasticsearch.internal-distribution-archive-check'
}"""
}
file("SomeFile.txt") << """
some dummy txt file
"""
buildFile << """
allprojects {
apply plugin:'base'
ext.elasticLicenseUrl = "http://foo.bar"
}
tasks.register("buildDarwinTar", Tar) {
compression = Compression.GZIP
from 'SomeFile.class'
}
tasks.register("buildDarwinZip", Zip) {
from 'SomeFile.txt'
}"""
}
@Unroll
def "plain class files in distribution #archiveType archives are detected"() {
given:
file("SomeFile.class") << """
some dummy class file
"""
buildFile << """
tasks.withType(AbstractArchiveTask).configureEach {
from 'SomeFile.class'
}
"""
when:
def result = gradleRunner(":darwin-${archiveType}:check", '--stacktrace').buildAndFail()
then:
result.task(":darwin-${archiveType}:checkExtraction").outcome == TaskOutcome.FAILED
result.output.contains("Detected class file in distribution ('SomeFile.class')")
where:
archiveType << ["zip", 'tar']
}
def "fails on unexpected license content"() {
given:
elasticLicense()
file("LICENSE.txt") << """elastic license coorp stuff line 1
unknown license content line 2
"""
buildFile << """
tasks.withType(AbstractArchiveTask).configureEach {
into("elasticsearch-${VersionProperties.getElasticsearch()}") {
from 'LICENSE.txt'
from 'SomeFile.txt'
}
}
"""
when:
def result = gradleRunner(":darwin-tar:check").buildAndFail()
then:
result.task(":darwin-tar:checkLicense").outcome == TaskOutcome.FAILED
normalizedOutput(result.output).contains("> expected line [2] in " +
"[./darwin-tar/build/tar-extracted/elasticsearch-${VersionProperties.getElasticsearch()}/LICENSE.txt] " +
"to be [elastic license coorp stuff line 2] but was [unknown license content line 2]")
}
def "fails on unexpected notice content"() {
given:
elasticLicense()
elasticLicense(file("LICENSE.txt"))
file("NOTICE.txt").text = """Elasticsearch
Copyright 2009-2018 Acme Coorp"""
buildFile << """
apply plugin:'base'
tasks.withType(AbstractArchiveTask).configureEach {
into("elasticsearch-${VersionProperties.getElasticsearch()}") {
from 'LICENSE.txt'
from 'SomeFile.txt'
from 'NOTICE.txt'
}
}
"""
when:
def result = gradleRunner(":darwin-tar:checkNotice").buildAndFail()
then:
result.task(":darwin-tar:checkNotice").outcome == TaskOutcome.FAILED
normalizedOutput(result.output).contains("> expected line [2] in " +
"[./darwin-tar/build/tar-extracted/elasticsearch-${VersionProperties.getElasticsearch()}/NOTICE.txt] " +
"to be [Copyright 2009-2018 Elasticsearch] but was [Copyright 2009-2018 Acme Coorp]")
}
def "fails on unexpected ml notice content"() {
given:
elasticLicense()
elasticLicense(file("LICENSE.txt"))
file("NOTICE.txt").text = """Elasticsearch
Copyright 2009-2018 Elasticsearch"""
file("ml/NOTICE.txt").text = "Boost Software License - Version 1.0 - August 17th, 2003"
file('darwin-tar/build.gradle') << """
distributionArchiveCheck {
expectedMlLicenses.add('foo license')
}
"""
buildFile << """
apply plugin:'base'
tasks.withType(AbstractArchiveTask).configureEach {
into("elasticsearch-${VersionProperties.getElasticsearch()}") {
from 'LICENSE.txt'
from 'SomeFile.txt'
from 'NOTICE.txt'
into('modules/x-pack-ml') {
from 'ml/NOTICE.txt'
}
}
}
"""
when:
def result = gradleRunner(":darwin-tar:check").buildAndFail()
then:
result.task(":darwin-tar:checkMlCppNotice").outcome == TaskOutcome.FAILED
normalizedOutput(result.output)
.contains("> expected [./darwin-tar/build/tar-extracted/elasticsearch-" +
"${VersionProperties.getElasticsearch()}/modules/x-pack-ml/NOTICE.txt " +
"to contain [foo license] but it did not")
}
void elasticLicense(File file = file("licenses/ELASTIC-LICENSE.txt")) {
file << """elastic license coorp stuff line 1
elastic license coorp stuff line 2
elastic license coorp stuff line 3
"""
}
}

View File

@ -17,8 +17,9 @@
* under the License.
*/
package org.elasticsearch.gradle
package org.elasticsearch.gradle.internal
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
import org.gradle.testkit.runner.GradleRunner
import org.gradle.testkit.runner.TaskOutcome
@ -107,7 +108,6 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
given:
internalBuild()
bwcMinorProjectSetup()
def distroVersion = "8.1.0"
buildFile << """
apply plugin: 'elasticsearch.internal-distribution-download'

View File

@ -0,0 +1,32 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.internal;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.ListProperty;
public class DistributionArchiveCheckExtension {
ListProperty<String> expectedMlLicenses;
public DistributionArchiveCheckExtension(ObjectFactory factory) {
this.expectedMlLicenses = factory.listProperty(String.class);
}
}

View File

@ -0,0 +1,253 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.internal;
import org.elasticsearch.gradle.VersionProperties;
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.file.ArchiveOperations;
import org.gradle.api.plugins.BasePlugin;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.TaskProvider;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
public class InternalDistributionArchiveCheckPlugin implements Plugin<Project> {
private ArchiveOperations archiveOperations;
@Inject
public InternalDistributionArchiveCheckPlugin(ArchiveOperations archiveOperations) {
this.archiveOperations = archiveOperations;
}
@Override
public void apply(Project project) {
project.getPlugins().apply(BasePlugin.class);
String buildTaskName = calculateBuildTask(project.getName());
TaskProvider<Task> buildDistTask = project.getParent().getTasks().named(buildTaskName);
DistributionArchiveCheckExtension distributionArchiveCheckExtension = project.getExtensions()
.create("distributionArchiveCheck", DistributionArchiveCheckExtension.class);
File archiveExtractionDir = calculateArchiveExtractionDir(project);
// sanity checks if archives can be extracted
TaskProvider<Copy> checkExtraction = registerCheckExtractionTask(project, buildDistTask, archiveExtractionDir);
TaskProvider<Task> checkLicense = registerCheckLicenseTask(project, checkExtraction);
TaskProvider<Task> checkNotice = registerCheckNoticeTask(project, checkExtraction);
TaskProvider<Task> checkTask = project.getTasks().named("check");
checkTask.configure(task -> {
task.dependsOn(checkExtraction);
task.dependsOn(checkLicense);
task.dependsOn(checkNotice);
});
if (project.getName().contains("zip") || project.getName().contains("tar")) {
project.getExtensions().add("licenseName", "Elastic License");
project.getExtensions().add("licenseUrl", project.getExtensions().getExtraProperties().get("elasticLicenseUrl"));
TaskProvider<Task> checkMlCppNoticeTask = registerCheckMlCppNoticeTask(
project,
checkExtraction,
distributionArchiveCheckExtension
);
checkTask.configure(task -> task.dependsOn(checkMlCppNoticeTask));
}
}
private File calculateArchiveExtractionDir(Project project) {
if (project.getName().contains("tar")) {
return new File(project.getBuildDir(), "tar-extracted");
}
if (project.getName().contains("zip") == false) {
throw new GradleException("Expecting project name containing 'zip' or 'tar'.");
}
return new File(project.getBuildDir(), "zip-extracted");
}
private static TaskProvider<Task> registerCheckMlCppNoticeTask(
Project project,
TaskProvider<Copy> checkExtraction,
DistributionArchiveCheckExtension extension
) {
TaskProvider<Task> checkMlCppNoticeTask = project.getTasks().register("checkMlCppNotice", task -> {
task.dependsOn(checkExtraction);
task.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
// this is just a small sample from the C++ notices,
// the idea being that if we've added these lines we've probably added all the required lines
final List<String> expectedLines = extension.expectedMlLicenses.get();
final Path noticePath = checkExtraction.get()
.getDestinationDir()
.toPath()
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt");
final List<String> actualLines;
try {
actualLines = Files.readAllLines(noticePath);
for (final String expectedLine : expectedLines) {
if (actualLines.contains(expectedLine) == false) {
throw new GradleException("expected [" + noticePath + " to contain [" + expectedLine + "] but it did not");
}
}
} catch (IOException ioException) {
ioException.printStackTrace();
}
}
});
});
return checkMlCppNoticeTask;
}
private TaskProvider<Task> registerCheckNoticeTask(Project project, TaskProvider<Copy> checkExtraction) {
return project.getTasks().register("checkNotice", task -> {
task.dependsOn(checkExtraction);
task.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2018 Elasticsearch");
final Path noticePath = checkExtraction.get()
.getDestinationDir()
.toPath()
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt");
assertLinesInFile(noticePath, noticeLines);
}
});
});
}
private TaskProvider<Task> registerCheckLicenseTask(Project project, TaskProvider<Copy> checkExtraction) {
TaskProvider<Task> checkLicense = project.getTasks().register("checkLicense", task -> {
task.dependsOn(checkExtraction);
task.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
String licenseFilename = null;
if (project.getName().contains("oss-") || project.getName().equals("integ-test-zip")) {
licenseFilename = "APACHE-LICENSE-2.0.txt";
} else {
licenseFilename = "ELASTIC-LICENSE.txt";
}
final List<String> licenseLines;
try {
licenseLines = Files.readAllLines(project.getRootDir().toPath().resolve("licenses/" + licenseFilename));
final Path licensePath = checkExtraction.get()
.getDestinationDir()
.toPath()
.resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt");
assertLinesInFile(licensePath, licenseLines);
} catch (IOException ioException) {
ioException.printStackTrace();
}
}
});
});
return checkLicense;
}
private TaskProvider<Copy> registerCheckExtractionTask(Project project, TaskProvider<Task> buildDistTask, File archiveExtractionDir) {
return project.getTasks().register("checkExtraction", Copy.class, t -> {
t.dependsOn(buildDistTask);
if (project.getName().contains("tar")) {
t.from(archiveOperations.tarTree(distTaskOutput(buildDistTask)));
} else {
t.from(archiveOperations.zipTree(distTaskOutput(buildDistTask)));
}
t.into(archiveExtractionDir);
// common sanity checks on extracted archive directly as part of checkExtraction
t.eachFile(fileCopyDetails -> assertNoClassFile(fileCopyDetails.getFile()));
});
}
private static void assertLinesInFile(Path path, List<String> expectedLines) {
try {
final List<String> actualLines = Files.readAllLines(path);
int line = 0;
for (final String expectedLine : expectedLines) {
final String actualLine = actualLines.get(line);
if (expectedLine.equals(actualLine) == false) {
throw new GradleException(
"expected line [" + (line + 1) + "] in [" + path + "] to be [" + expectedLine + "] but was [" + actualLine + "]"
);
}
line++;
}
} catch (IOException ioException) {
throw new GradleException("Unable to read from file " + path, ioException);
}
}
private static boolean toolExists(Project project) {
if (project.getName().contains("tar")) {
return tarExists();
} else {
assert project.getName().contains("zip");
return zipExists();
}
}
private static void assertNoClassFile(File file) {
if (file.getName().endsWith(".class")) {
throw new GradleException("Detected class file in distribution ('" + file.getName() + "')");
}
}
private static boolean zipExists() {
return new File("/bin/unzip").exists() || new File("/usr/bin/unzip").exists() || new File("/usr/local/bin/unzip").exists();
}
private static boolean tarExists() {
return new File("/bin/tar").exists() || new File("/usr/bin/tar").exists() || new File("/usr/local/bin/tar").exists();
}
private Object distTaskOutput(TaskProvider<Task> buildDistTask) {
return new Callable<File>() {
@Override
public File call() {
return buildDistTask.get().getOutputs().getFiles().getSingleFile();
}
@Override
public String toString() {
return call().getAbsolutePath();
}
};
}
private String calculateBuildTask(String projectName) {
return "build" + Arrays.stream(projectName.split("-")).map(f -> capitalize(f)).collect(Collectors.joining());
}
private static String capitalize(String str) {
if (str == null) return str;
return str.substring(0, 1).toUpperCase() + str.substring(1);
}
}

View File

@ -0,0 +1,20 @@
#
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
implementation-class=org.elasticsearch.gradle.internal.InternalDistributionArchiveCheckPlugin

View File

@ -213,9 +213,6 @@ tasks.register('buildOssNoJdkLinuxTar', SymbolicLinkPreservingTar) {
with archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, false)
}
Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
Closure unzipExists = { it -> new File('/bin/unzip').exists() || new File('/usr/bin/unzip').exists() || new File('/usr/local/bin/unzip').exists() }
// This configures the default artifact for the distribution specific
// subprojects. We have subprojects for two reasons:
// 1. Gradle project substitutions can only bind to the default
@ -224,106 +221,17 @@ Closure unzipExists = { it -> new File('/bin/unzip').exists() || new File('/usr/
// filename, so they must be placed in different directories.
subprojects {
apply plugin: 'distribution'
apply plugin: 'elasticsearch.internal-distribution-archive-check'
distributionArchiveCheck {
expectedMlLicenses.add("Boost Software License - Version 1.0 - August 17th, 2003")
}
String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
ext.buildDist = parent.tasks.named(buildTask)
artifacts {
'default' buildDist
}
// sanity checks if archives can be extracted
File archiveExtractionDir
if (project.name.contains('tar')) {
archiveExtractionDir = new File(buildDir, 'tar-extracted')
} else {
assert project.name.contains('zip')
archiveExtractionDir = new File(buildDir, 'zip-extracted')
}
def checkExtraction = tasks.register('checkExtraction', LoggedExec) {
dependsOn buildDist
doFirst {
project.delete(archiveExtractionDir)
archiveExtractionDir.mkdirs()
}
// common sanity checks on extracted archive directly as part of checkExtraction
doLast {
// check no plain class files are packaged
archiveExtractionDir.eachFileRecurse (FileType.FILES) { file ->
assert file.name.endsWith(".class") == false
}
}
}
tasks.named('check').configure { dependsOn checkExtraction }
if (project.name.contains('tar')) {
checkExtraction.configure {
onlyIf tarExists
commandLine 'tar', '-xvzf', "${-> buildDist.get().outputs.files.singleFile}", '-C', archiveExtractionDir
}
} else {
assert project.name.contains('zip')
checkExtraction.configure {
onlyIf unzipExists
commandLine 'unzip', "${-> buildDist.get().outputs.files.singleFile}", '-d', archiveExtractionDir
}
}
Closure toolExists
if (project.name.contains('tar')) {
toolExists = tarExists
} else {
assert project.name.contains('zip')
toolExists = unzipExists
}
tasks.register('checkLicense') {
dependsOn buildDist, checkExtraction
onlyIf toolExists
doLast {
String licenseFilename = null
if (project.name.contains('oss-') || project.name == 'integ-test-zip') {
licenseFilename = "APACHE-LICENSE-2.0.txt"
} else {
licenseFilename = "ELASTIC-LICENSE.txt"
}
final List<String> licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename))
final Path licensePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/LICENSE.txt")
assertLinesInFile(licensePath, licenseLines)
}
}
tasks.named('check').configure { dependsOn checkLicense }
tasks.register('checkNotice') {
dependsOn buildDist, checkExtraction
onlyIf toolExists
doLast {
final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2018 Elasticsearch")
final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/NOTICE.txt")
assertLinesInFile(noticePath, noticeLines)
}
}
tasks.named('check').configure { dependsOn checkNotice }
if (project.name == 'zip' || project.name == 'tar') {
project.ext.licenseName = 'Elastic License'
project.ext.licenseUrl = ext.elasticLicenseUrl
def checkMlCppNotice = tasks.register('checkMlCppNotice') {
dependsOn buildDist, checkExtraction
onlyIf toolExists
doLast {
// this is just a small sample from the C++ notices, the idea being that if we've added these lines we've probably added all the required lines
final List<String> expectedLines = Arrays.asList("Apache log4cxx", "Boost Software License - Version 1.0 - August 17th, 2003")
final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack-ml/NOTICE.txt")
final List<String> actualLines = Files.readAllLines(noticePath)
for (final String expectedLine : expectedLines) {
if (actualLines.contains(expectedLine) == false) {
throw new GradleException("expected [${noticePath}] to contain [${expectedLine}] but it did not")
}
}
}
}
tasks.named('check').configure { dependsOn checkMlCppNotice }
}
}
subprojects {