[PURIFY] update build.gradle files to ensure build completes; gradle check fails (#7)

Signed-off-by: Peter Nied <petern@amazon.com>
This commit is contained in:
Nick Knize 2021-01-29 23:39:22 -06:00 committed by Peter Nied
parent e96624565a
commit 3a52e9ddc1
13 changed files with 19 additions and 269 deletions

View File

@ -356,15 +356,7 @@ allprojects {
}
File licenseHeaderFile
String prefix = ':x-pack'
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
prefix = prefix.replace(':', '_')
}
if (eclipse.project.name.startsWith(prefix)) {
licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/elastic-license-header.txt')
} else {
licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
}
licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
@ -467,11 +459,7 @@ allprojects {
def checkPart1 = tasks.register('checkPart1')
def checkPart2 = tasks.register('checkPart2')
plugins.withId('lifecycle-base') {
if (project.path.startsWith(":x-pack:")) {
checkPart2.configure { dependsOn 'check' }
} else {
checkPart1.configure { dependsOn 'check' }
}
checkPart1.configure { dependsOn 'check' }
}
}

View File

@ -76,35 +76,10 @@ class InternalDistributionArchiveCheckPluginFuncTest extends AbstractGradleFuncT
archiveType << ["zip", 'tar']
}
def "fails on unexpected license content"() {
given:
elasticLicense()
file("LICENSE.txt") << """elastic license coorp stuff line 1
unknown license content line 2
"""
buildFile << """
tasks.withType(AbstractArchiveTask).configureEach {
into("elasticsearch-${VersionProperties.getElasticsearch()}") {
from 'LICENSE.txt'
from 'SomeFile.txt'
}
}
"""
when:
def result = gradleRunner(":darwin-tar:check").buildAndFail()
then:
result.task(":darwin-tar:checkLicense").outcome == TaskOutcome.FAILED
normalizedOutput(result.output).contains("> expected line [2] in " +
"[./darwin-tar/build/tar-extracted/elasticsearch-${VersionProperties.getElasticsearch()}/LICENSE.txt] " +
"to be [elastic license coorp stuff line 2] but was [unknown license content line 2]")
}
def "fails on unexpected notice content"() {
given:
elasticLicense()
elasticLicense(file("LICENSE.txt"))
file("NOTICE.txt").text = """Elasticsearch
license(file("LICENSE.txt"))
file("NOTICE.txt").text = """OSS Search
Copyright 2009-2018 Acme Coorp"""
buildFile << """
apply plugin:'base'
@ -126,47 +101,10 @@ Copyright 2009-2018 Acme Coorp"""
"to be [Copyright 2009-2018 Elasticsearch] but was [Copyright 2009-2018 Acme Coorp]")
}
def "fails on unexpected ml notice content"() {
given:
elasticLicense()
elasticLicense(file("LICENSE.txt"))
file("NOTICE.txt").text = """Elasticsearch
Copyright 2009-2018 Elasticsearch"""
file("ml/NOTICE.txt").text = "Boost Software License - Version 1.0 - August 17th, 2003"
file('darwin-tar/build.gradle') << """
distributionArchiveCheck {
expectedMlLicenses.add('foo license')
}
"""
buildFile << """
apply plugin:'base'
tasks.withType(AbstractArchiveTask).configureEach {
into("elasticsearch-${VersionProperties.getElasticsearch()}") {
from 'LICENSE.txt'
from 'SomeFile.txt'
from 'NOTICE.txt'
into('modules/x-pack-ml') {
from 'ml/NOTICE.txt'
}
}
}
"""
when:
def result = gradleRunner(":darwin-tar:check").buildAndFail()
then:
result.task(":darwin-tar:checkMlCppNotice").outcome == TaskOutcome.FAILED
normalizedOutput(result.output)
.contains("> expected [./darwin-tar/build/tar-extracted/elasticsearch-" +
"${VersionProperties.getElasticsearch()}/modules/x-pack-ml/NOTICE.txt " +
"to contain [foo license] but it did not")
}
void elasticLicense(File file = file("licenses/ELASTIC-LICENSE.txt")) {
file << """elastic license coorp stuff line 1
elastic license coorp stuff line 2
elastic license coorp stuff line 3
void license(File file = file("licenses/APACHE-LICENSE-2.0.txt")) {
file << """license coorp stuff line 1
license coorp stuff line 2
license coorp stuff line 3
"""
}

View File

@ -68,12 +68,6 @@ public class TestWithSslPlugin implements Plugin<Project> {
.getExtensions()
.getByName(TestClustersPlugin.EXTENSION_NAME);
clusters.all(c -> {
// ceremony to set up ssl
c.setting("xpack.security.transport.ssl.keystore.path", "test-node.jks");
c.setting("xpack.security.http.ssl.keystore.path", "test-node.jks");
c.keystore("xpack.security.transport.ssl.keystore.secure_password", "keypass");
c.keystore("xpack.security.http.ssl.keystore.secure_password", "keypass");
// copy keystores & certs into config/
c.extraConfigFile(nodeKeystore.getName(), nodeKeystore);
c.extraConfigFile(clientKeyStore.getName(), clientKeyStore);

View File

@ -60,19 +60,15 @@ import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask;
public class CopyRestApiTask extends DefaultTask {
private static final String REST_API_PREFIX = "rest-api-spec/api";
final ListProperty<String> includeCore = getProject().getObjects().listProperty(String.class);
final ListProperty<String> includeXpack = getProject().getObjects().listProperty(String.class);
String sourceSetName;
boolean skipHasRestTestCheck;
Configuration coreConfig;
Configuration xpackConfig;
Configuration additionalConfig;
private final PatternFilterable corePatternSet;
private final PatternFilterable xpackPatternSet;
public CopyRestApiTask() {
corePatternSet = getPatternSetFactory().create();
xpackPatternSet = getPatternSetFactory().create();
}
@Inject
@ -95,11 +91,6 @@ public class CopyRestApiTask extends DefaultTask {
return includeCore;
}
@Input
public ListProperty<String> getIncludeXpack() {
return includeXpack;
}
@Input
String getSourceSetName() {
return sourceSetName;
@ -114,11 +105,6 @@ public class CopyRestApiTask extends DefaultTask {
@InputFiles
public FileTree getInputDir() {
FileTree coreFileTree = null;
FileTree xpackFileTree = null;
if (includeXpack.get().isEmpty() == false) {
xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
xpackFileTree = xpackConfig.getAsFileTree().matching(xpackPatternSet);
}
boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();
if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) {
if (BuildParams.isInternal()) {
@ -130,13 +116,11 @@ public class CopyRestApiTask extends DefaultTask {
}
ConfigurableFileCollection fileCollection = additionalConfig == null
? getProject().files(coreFileTree, xpackFileTree)
: getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree());
? getProject().files(coreFileTree)
: getProject().files(coreFileTree, additionalConfig.getAsFileTree());
// if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
? fileCollection.getAsFileTree()
: null;
return projectHasYamlRestTests || includeCore.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
}
@OutputDirectory
@ -179,15 +163,6 @@ public class CopyRestApiTask extends DefaultTask {
}
});
}
// only copy x-pack specs if explicitly instructed
if (includeXpack.get().isEmpty() == false) {
getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath);
getFileSystemOperations().copy(c -> {
c.from(xpackConfig.getSingleFile());
c.into(getOutputDir());
c.include(xpackPatternSet.getIncludes());
});
}
// TODO: once https://github.com/elastic/elasticsearch/pull/62968 lands ensure that this uses `getFileSystemOperations()`
// copy any additional config
if (additionalConfig != null) {

View File

@ -57,19 +57,15 @@ import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask;
public class CopyRestTestsTask extends DefaultTask {
private static final String REST_TEST_PREFIX = "rest-api-spec/test";
final ListProperty<String> includeCore = getProject().getObjects().listProperty(String.class);
final ListProperty<String> includeXpack = getProject().getObjects().listProperty(String.class);
String sourceSetName;
Configuration coreConfig;
Configuration xpackConfig;
Configuration additionalConfig;
private final PatternFilterable corePatternSet;
private final PatternFilterable xpackPatternSet;
public CopyRestTestsTask() {
corePatternSet = getPatternSetFactory().create();
xpackPatternSet = getPatternSetFactory().create();
}
@Inject
@ -92,11 +88,6 @@ public class CopyRestTestsTask extends DefaultTask {
return includeCore;
}
@Input
public ListProperty<String> getIncludeXpack() {
return includeXpack;
}
@Input
String getSourceSetName() {
return sourceSetName;
@ -106,11 +97,6 @@ public class CopyRestTestsTask extends DefaultTask {
@InputFiles
public FileTree getInputDir() {
FileTree coreFileTree = null;
FileTree xpackFileTree = null;
if (includeXpack.get().isEmpty() == false) {
xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
xpackFileTree = xpackConfig.getAsFileTree().matching(xpackPatternSet);
}
if (includeCore.get().isEmpty() == false) {
if (BuildParams.isInternal()) {
corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
@ -120,13 +106,11 @@ public class CopyRestTestsTask extends DefaultTask {
}
}
ConfigurableFileCollection fileCollection = additionalConfig == null
? getProject().files(coreFileTree, xpackFileTree)
: getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree());
? getProject().files(coreFileTree)
: getProject().files(coreFileTree, additionalConfig.getAsFileTree());
// copy tests only if explicitly requested
return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false || additionalConfig != null
? fileCollection.getAsFileTree()
: null;
return includeCore.get().isEmpty() == false || additionalConfig != null ? fileCollection.getAsFileTree() : null;
}
@OutputDirectory
@ -167,15 +151,6 @@ public class CopyRestTestsTask extends DefaultTask {
});
}
}
// only copy x-pack tests if explicitly instructed
if (includeXpack.get().isEmpty() == false) {
getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", projectPath);
getFileSystemOperations().copy(c -> {
c.from(xpackConfig.getAsFileTree());
c.into(getOutputDir());
c.include(xpackPatternSet.getIncludes());
});
}
// copy any additional config
if (additionalConfig != null) {
getFileSystemOperations().copy(c -> {

View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.gradle.test.rest;
import org.elasticsearch.gradle.info.BuildParams;
import org.gradle.api.Action;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.ListProperty;
@ -50,30 +49,17 @@ public class RestResourcesExtension {
static class RestResourcesSpec {
private final ListProperty<String> includeCore;
private final ListProperty<String> includeXpack;
RestResourcesSpec(ObjectFactory objects) {
includeCore = objects.listProperty(String.class);
includeXpack = objects.listProperty(String.class);
}
public void includeCore(String... include) {
this.includeCore.addAll(include);
}
public void includeXpack(String... include) {
if (BuildParams.isInternal() == false) {
throw new IllegalStateException("Can not include x-pack rest resources from an external build.");
}
this.includeXpack.addAll(include);
}
public ListProperty<String> getIncludeCore() {
return includeCore;
}
public ListProperty<String> getIncludeXpack() {
return includeXpack;
}
}
}

View File

@ -37,18 +37,7 @@ import java.util.Map;
* </p>
* <strong>Rest API specification:</strong> <br>
* When the {@link RestResourcesPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification
* if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly
* declared to be copied.
* <br>
* <i>For example:</i>
* <pre>
* restResources {
* restApi {
* includeXpack 'enrich'
* }
* }
* </pre>
* Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*.
* if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output.
* It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching
* behavior.
* <i>For example:</i>
@ -56,27 +45,13 @@ import java.util.Map;
* restResources {
* restApi {
* includeCore 'index', 'cat'
* includeXpack 'enrich'
* }
* }
* </pre>
* <br>
* <strong>Rest YAML tests :</strong> <br>
* When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly
* configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension.
* <i>For example:</i>
* <pre>
* restResources {
* restApi {
* includeXpack 'graph'
* }
* restTests {
* includeXpack 'graph'
* }
* }
* </pre>
* Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core
* Rest API specification.
* configured with `includeCore` through the `restResources.restTests` extension.
*
* Additionally you can specify which sourceSetName resources should be copied to. The default is the yamlRestTest source set.
* @see CopyRestApiTask
@ -92,13 +67,10 @@ public class RestResourcesPlugin implements Plugin<Project> {
// tests
Configuration testConfig = project.getConfigurations().create("restTestConfig");
Configuration xpackTestConfig = project.getConfigurations().create("restXpackTestConfig");
project.getConfigurations().create("restTests");
project.getConfigurations().create("restXpackTests");
Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
.register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
task.includeCore.set(extension.restTests.getIncludeCore());
task.includeXpack.set(extension.restTests.getIncludeXpack());
task.coreConfig = testConfig;
task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
if (BuildParams.isInternal()) {
@ -106,12 +78,6 @@ public class RestResourcesPlugin implements Plugin<Project> {
Dependency restTestdependency = project.getDependencies()
.project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
// x-pack
task.xpackConfig = xpackTestConfig;
Dependency restXPackTestdependency = project.getDependencies()
.project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency);
task.dependsOn(task.xpackConfig);
} else {
Dependency dependency = project.getDependencies()
.create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
@ -122,13 +88,10 @@ public class RestResourcesPlugin implements Plugin<Project> {
// api
Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
Configuration xpackSpecConfig = project.getConfigurations().create("restXpackSpec");
project.getConfigurations().create("restSpecs");
project.getConfigurations().create("restXpackSpecs");
Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
.register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
task.includeCore.set(extension.restApi.getIncludeCore());
task.includeXpack.set(extension.restApi.getIncludeXpack());
task.dependsOn(copyRestYamlTestTask);
task.coreConfig = specConfig;
task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
@ -136,11 +99,6 @@ public class RestResourcesPlugin implements Plugin<Project> {
Dependency restSpecDependency = project.getDependencies()
.project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
task.xpackConfig = xpackSpecConfig;
Dependency restXpackSpecDependency = project.getDependencies()
.project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency);
task.dependsOn(task.xpackConfig);
} else {
Dependency dependency = project.getDependencies()
.create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());

View File

@ -50,12 +50,6 @@ dependencies {
testImplementation "junit:junit:${versions.junit}"
//this is needed to make RestHighLevelClientTests#testApiNamingConventions work from IDEs
testImplementation project(":rest-api-spec")
// Needed for serialization tests:
// (In order to serialize a server side class to a client side class or the other way around)
testImplementation(project(':x-pack:plugin:core')) {
exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client'
}
testImplementation(project(':x-pack:plugin:eql'))
}
tasks.named('forbiddenApisMain').configure {

View File

@ -276,20 +276,6 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
restTestExpansions['expected.modules.count'] += 1
}
// use licenses from each of the bundled xpack plugins
Project xpack = project(':x-pack:plugin')
xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
File licenses = new File(xpackModule.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNoticeTaskProvider.configure {
licensesDir licenses
source xpackModule.file('src/main/java')
}
}
copyModule(processDefaultOutputsTaskProvider, xpackModule)
copyLog4jProperties(buildDefaultLog4jConfigTaskProvider, xpackModule)
}
copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd'))
project(':test:external-modules').subprojects.each { Project testModule ->
@ -341,7 +327,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
libsPluginCli project(':distribution:tools:plugin-cli')
libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
libsSecurityCli project(':x-pack:plugin:security:cli')
}
project.ext {
@ -391,12 +376,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
excludePlatforms = []
}
from(buildModules) {
// geo registers the geo_shape mapper that is overridden by
// the geo_shape mapper registered in the x-pack-spatial plugin
if (oss == false) {
exclude "**/geo/**"
}
for (String excludePlatform : excludePlatforms) {
exclude "**/platform/${excludePlatform}/**"
}

View File

@ -155,16 +155,6 @@ def createAndSetWritable(Object... locations) {
}
}
tasks.register("copyKeystore", Sync) {
from project(':x-pack:plugin:core')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into "${buildDir}/certs"
doLast {
file("${buildDir}/certs").setReadable(true, false)
file("${buildDir}/certs/testnode.jks").setReadable(true, false)
}
}
elasticsearch_distributions {
Architecture.values().each { eachArchitecture ->
Flavor.values().each { distroFlavor ->
@ -181,7 +171,6 @@ elasticsearch_distributions {
tasks.named("preProcessFixture").configure {
dependsOn elasticsearch_distributions.docker_default, elasticsearch_distributions.docker_oss
dependsOn "copyKeystore"
doLast {
// tests expect to have an empty repo
project.delete(
@ -199,11 +188,6 @@ tasks.named("preProcessFixture").configure {
}
}
tasks.named("processTestResources").configure {
from project(':x-pack:plugin:core')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
}
tasks.register("integTest", Test) {
outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
maxParallelForks = '1'

View File

@ -29,8 +29,6 @@ configure(project('painless-whitelist')) {
configure(project('security-authorization-engine')) {
configurations.all {
resolutionStrategy.dependencySubstitution {
substitute module('org.elasticsearch.plugin:x-pack-core') with project(':x-pack:plugin:core')
substitute module('org.elasticsearch.client:x-pack-transport') with project(':x-pack:transport-client')
substitute module('org.elasticsearch.test:logger-usage') with project(':test:logger-usage')
}
}

View File

@ -5,15 +5,11 @@ esplugin {
name 'security-authorization-engine'
description 'An example spi extension plugin for security that implements an Authorization Engine'
classname 'org.elasticsearch.example.AuthorizationEnginePlugin'
extendedPlugins = ['x-pack-security']
licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
noticeFile rootProject.file('NOTICE.txt')
}
dependencies {
compileOnly "org.elasticsearch.plugin:x-pack-core:${versions.elasticsearch}"
javaRestTestImplementation "org.elasticsearch.client:x-pack-transport:${versions.elasticsearch}"
javaRestTestImplementation "org.elasticsearch.plugin:x-pack-core:${versions.elasticsearch}"
// let the javaRestTest see the classpath of main
javaRestTestImplementation project.sourceSets.main.runtimeClasspath
}

View File

@ -31,16 +31,6 @@ dependencies {
testImplementation project(':client:rest-high-level')
}
tasks.register("copyKeystore", Sync) {
from project(':x-pack:plugin:core')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into "${buildDir}/certs"
doLast {
file("${buildDir}/certs").setReadable(true, false)
file("${buildDir}/certs/testnode.jks").setReadable(true, false)
}
}
elasticsearch_distributions {
docker {
type = 'docker'
@ -52,7 +42,7 @@ elasticsearch_distributions {
}
preProcessFixture {
dependsOn "copyKeystore", elasticsearch_distributions.docker
dependsOn elasticsearch_distributions.docker
doLast {
// tests expect to have an empty repo
project.delete(
@ -87,11 +77,6 @@ def createAndSetWritable(Object... locations) {
}
}
tasks.named("processTestResources").configure {
from project(':x-pack:plugin:core')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
}
tasks.register("integTest", Test) {
outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
maxParallelForks = '1'