honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Converting DependencyLicensesTask and UpdateShasTask to java (#41921)

This commit is contained in:
Gabryel Monteiro 2019-06-13 06:12:02 -03:00 committed by Alpar Torok
parent afbb791969
commit 47d861c23a
7 changed files with 824 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/build.gradle
View File

@ -81,6 +81,8 @@ repositories {
dependencies {
compile localGroovy()
compile 'commons-codec:commons-codec:1.12'
compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'

268
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
View File

@ -1,268 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
import java.security.MessageDigest
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* A task to check licenses for dependencies.
*
* There are two parts to the check:
* <ul>
* <li>LICENSE and NOTICE files</li>
* <li>SHA checksums for each dependency jar</li>
* </ul>
*
* The directory to find the license and sha files in defaults to the dir @{code licenses}
* in the project directory for this task. You can override this directory:
* <pre>
* dependencyLicenses {
* licensesDir = project.file('mybetterlicensedir')
* }
* </pre>
*
* The jar files to check default to the dependencies from the default configuration. You
* can override this, for example, to only check compile dependencies:
* <pre>
* dependencyLicenses {
* dependencies = project.configurations.compile
* }
* </pre>
*
* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
* automatically using the {@code updateShas} helper task that is created along
* with this task. It will add {@code .sha1} files for new jars that are in dependencies
* and remove old {@code .sha1} files that are no longer needed.
*
* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
* LICENSE and NOTICE files by mapping a pattern to the same name.
* <pre>
* dependencyLicenses {
* mapping from: &#47;lucene-.*&#47;, to: 'lucene'
* }
* </pre>
*/
public class DependencyLicensesTask extends DefaultTask {
static final String SHA_EXTENSION = '.sha1'
// TODO: we should be able to default this to eg compile deps, but we need to move the licenses
// check from distribution to core (ie this should only be run on java projects)
/** A collection of jar files that should be checked. */
@InputFiles
public FileCollection dependencies
/** The directory to find the license and sha files in. */
@InputDirectory
public File licensesDir = new File(project.projectDir, 'licenses')
/** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
private LinkedHashMap<String, String> mappings = new LinkedHashMap<>()
/** Names of dependencies whose shas should not exist. */
private Set<String> ignoreShas = new HashSet<>()
/**
* Add a mapping from a regex pattern for the jar name, to a prefix to find
* the LICENSE and NOTICE file for that jar.
*/
@Input
public void mapping(Map<String, String> props) {
String from = props.remove('from')
if (from == null) {
throw new InvalidUserDataException('Missing "from" setting for license name mapping')
}
String to = props.remove('to')
if (to == null) {
throw new InvalidUserDataException('Missing "to" setting for license name mapping')
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: ${props.keySet()}")
}
mappings.put(from, to)
}
public LinkedHashMap<String, String> getMappings() {
return new LinkedHashMap<>(mappings)
}
/**
* Add a rule which will skip SHA checking for the given dependency name. This should be used for
* locally build dependencies, which cause the sha to change constantly.
*/
@Input
public void ignoreSha(String dep) {
ignoreShas.add(dep)
}
@TaskAction
public void checkDependencies() {
if (dependencies.isEmpty()) {
if (licensesDir.exists()) {
throw new GradleException("Licenses dir ${licensesDir} exists, but there are no dependencies")
}
return // no dependencies to check
} else if (licensesDir.exists() == false) {
throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies")
}
Map<String, Integer> licenses = new HashMap<>()
Map<String, Integer> notices = new HashMap<>()
Set<File> shaFiles = new HashSet<File>()
licensesDir.eachFile {
String name = it.getName()
if (name.endsWith(SHA_EXTENSION)) {
shaFiles.add(it)
} else if (name.endsWith('-LICENSE') || name.endsWith('-LICENSE.txt')) {
// TODO: why do we support suffix of LICENSE *and* LICENSE.txt??
licenses.put(name, 0)
} else if (name.contains('-NOTICE') || name.contains('-NOTICE.txt')) {
notices.put(name, 0)
}
}
for (File dependency : dependencies) {
String jarName = dependency.getName()
String depName = jarName - ~/\-v?\d+.*/
if (ignoreShas.contains(depName)) {
// local deps should not have sha files!
if (getShaFile(jarName).exists()) {
throw new GradleException("SHA file ${getShaFile(jarName)} exists for ignored dependency ${depName}")
}
} else {
logger.info("Checking sha for " + jarName)
checkSha(dependency, jarName, shaFiles)
}
final String dependencyName = getDependencyName(mappings, depName)
logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check")
checkFile(dependencyName, jarName, licenses, 'LICENSE')
checkFile(dependencyName, jarName, notices, 'NOTICE')
}
licenses.each { license, count ->
if (count == 0) {
throw new GradleException("Unused license ${license}")
}
}
notices.each { notice, count ->
if (count == 0) {
throw new GradleException("Unused notice ${notice}")
}
}
if (shaFiles.isEmpty() == false) {
throw new GradleException("Unused sha files found: \n${shaFiles.join('\n')}")
}
}
public static String getDependencyName(final LinkedHashMap<String, String> mappings, final String dependencyName) {
// order is the same for keys and values iteration since we use a linked hashmap
List<String> mapped = new ArrayList<>(mappings.values())
Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
Matcher match = mappingsPattern.matcher(dependencyName)
if (match.matches()) {
int i = 0
while (i < match.groupCount() && match.group(i + 1) == null) ++i;
return mapped.get(i)
}
return dependencyName
}
private File getShaFile(String jarName) {
return new File(licensesDir, jarName + SHA_EXTENSION)
}
private void checkSha(File jar, String jarName, Set<File> shaFiles) {
File shaFile = getShaFile(jarName)
if (shaFile.exists() == false) {
throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create")
}
// TODO: shouldn't have to trim, sha files should not have trailing newline
String expectedSha = shaFile.getText('UTF-8').trim()
String sha = MessageDigest.getInstance("SHA-1").digest(jar.getBytes()).encodeHex().toString()
if (expectedSha.equals(sha) == false) {
throw new GradleException("SHA has changed! Expected ${expectedSha} for ${jarName} but got ${sha}. " +
"\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." +
"\nEither wipe your cache, fix the upstream artifact, or delete ${shaFile} and run updateShas")
}
shaFiles.remove(shaFile)
}
private void checkFile(String name, String jarName, Map<String, Integer> counters, String type) {
String fileName = "${name}-${type}"
Integer count = counters.get(fileName)
if (count == null) {
// try the other suffix...TODO: get rid of this, just support ending in .txt
fileName = "${fileName}.txt"
counters.get(fileName)
}
count = counters.get(fileName)
if (count == null) {
throw new GradleException("Missing ${type} for ${jarName}, expected in ${fileName}")
}
counters.put(fileName, count + 1)
}
/** A helper task to update the sha files in the license dir. */
public static class UpdateShasTask extends DefaultTask {
private DependencyLicensesTask parentTask
@TaskAction
public void updateShas() {
Set<File> shaFiles = new HashSet<File>()
parentTask.licensesDir.eachFile {
String name = it.getName()
if (name.endsWith(SHA_EXTENSION)) {
shaFiles.add(it)
}
}
for (File dependency : parentTask.dependencies) {
String jarName = dependency.getName()
String depName = jarName - ~/\-\d+.*/
if (parentTask.ignoreShas.contains(depName)) {
continue
}
File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION)
if (shaFile.exists() == false) {
logger.lifecycle("Adding sha for ${jarName}")
String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString()
shaFile.setText(sha, 'UTF-8')
} else {
shaFiles.remove(shaFile)
}
}
shaFiles.each { shaFile ->
logger.lifecycle("Removing unused sha ${shaFile.getName()}")
Files.delete(shaFile.toPath())
}
}
}
}

66
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy
View File

@ -1,66 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
import java.security.MessageDigest
/**
* A task to update shas used by {@code DependencyLicensesCheck}
*/
public class UpdateShasTask extends DefaultTask {
/** The parent dependency licenses task to use configuration from */
public DependencyLicensesTask parentTask
public UpdateShasTask() {
description = 'Updates the sha files for the dependencyLicenses check'
onlyIf { parentTask.licensesDir.exists() }
}
@TaskAction
public void updateShas() {
Set<File> shaFiles = new HashSet<File>()
parentTask.licensesDir.eachFile {
String name = it.getName()
if (name.endsWith(DependencyLicensesTask.SHA_EXTENSION)) {
shaFiles.add(it)
}
}
for (File dependency : parentTask.dependencies) {
String jarName = dependency.getName()
File shaFile = new File(parentTask.licensesDir, jarName + DependencyLicensesTask.SHA_EXTENSION)
if (shaFile.exists() == false) {
logger.lifecycle("Adding sha for ${jarName}")
String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString()
shaFile.setText(sha, 'UTF-8')
} else {
shaFiles.remove(shaFile)
}
}
shaFiles.each { shaFile ->
logger.lifecycle("Removing unused sha ${shaFile.getName()}")
Files.delete(shaFile.toPath())
}
}
}

328
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java Normal file
View File

@ -0,0 +1,328 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit;
import org.apache.commons.codec.binary.Hex;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.InvalidUserDataException;
import org.gradle.api.file.FileCollection;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.TaskAction;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* A task to check licenses for dependencies.
*
* There are two parts to the check:
* <ul>
* <li>LICENSE and NOTICE files</li>
* <li>SHA checksums for each dependency jar</li>
* </ul>
*
* The directory to find the license and sha files in defaults to the dir @{code licenses}
* in the project directory for this task. You can override this directory:
* <pre>
* dependencyLicenses {
* licensesDir = getProject().file("mybetterlicensedir")
* }
* </pre>
*
* The jar files to check default to the dependencies from the default configuration. You
* can override this, for example, to only check compile dependencies:
* <pre>
* dependencyLicenses {
* dependencies = getProject().configurations.compile
* }
* </pre>
*
* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
* automatically using the {@code updateShas} helper task that is created along
* with this task. It will add {@code .sha1} files for new jars that are in dependencies
* and remove old {@code .sha1} files that are no longer needed.
*
* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
* LICENSE and NOTICE files by mapping a pattern to the same name.
* <pre>
* dependencyLicenses {
* mapping from: &#47;lucene-.*&#47;, to: "lucene"
* }
* </pre>
*/
public class DependencyLicensesTask extends DefaultTask {
private final Pattern regex = Pattern.compile("-v?\\d+.*");
private final Logger logger = Logging.getLogger(getClass());
private static final String SHA_EXTENSION = ".sha1";
// TODO: we should be able to default this to eg compile deps, but we need to move the licenses
// check from distribution to core (ie this should only be run on java projects)
/** A collection of jar files that should be checked. */
private FileCollection dependencies;
/** The directory to find the license and sha files in. */
private File licensesDir = new File(getProject().getProjectDir(), "licenses");
/** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
private Map<String, String> mappings = new LinkedHashMap<>();
/** Names of dependencies whose shas should not exist. */
private Set<String> ignoreShas = new HashSet<>();
/**
* Add a mapping from a regex pattern for the jar name, to a prefix to find
* the LICENSE and NOTICE file for that jar.
*/
public void mapping(Map<String, String> props) {
String from = props.remove("from");
if (from == null) {
throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
}
String to = props.remove("to");
if (to == null) {
throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
}
mappings.put(from, to);
}
@InputFiles
public FileCollection getDependencies() {
return dependencies;
}
public void setDependencies(FileCollection dependencies) {
this.dependencies = dependencies;
}
@Optional
@InputDirectory
public File getLicensesDir() {
if (licensesDir.exists()) {
return licensesDir;
}
return null;
}
public void setLicensesDir(File licensesDir) {
this.licensesDir = licensesDir;
}
/**
* Add a rule which will skip SHA checking for the given dependency name. This should be used for
* locally build dependencies, which cause the sha to change constantly.
*/
public void ignoreSha(String dep) {
ignoreShas.add(dep);
}
@TaskAction
public void checkDependencies() throws IOException, NoSuchAlgorithmException {
if (dependencies == null) {
throw new GradleException("No dependencies variable defined.");
}
if (dependencies.isEmpty()) {
if (licensesDir.exists()) {
throw new GradleException("Licenses dir " + licensesDir + " exists, but there are no dependencies");
}
return; // no dependencies to check
} else if (licensesDir.exists() == false) {
throw new GradleException("Licences dir " + licensesDir + " does not exist, but there are dependencies");
}
Map<String, Boolean> licenses = new HashMap<>();
Map<String, Boolean> notices = new HashMap<>();
Set<File> shaFiles = new HashSet<>();
for (File file : licensesDir.listFiles()) {
String name = file.getName();
if (name.endsWith(SHA_EXTENSION)) {
shaFiles.add(file);
} else if (name.endsWith("-LICENSE") || name.endsWith("-LICENSE.txt")) {
// TODO: why do we support suffix of LICENSE *and* LICENSE.txt??
licenses.put(name, false);
} else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) {
notices.put(name, false);
}
}
checkDependencies(licenses, notices, shaFiles);
licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));
notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice"));
if (shaFiles.isEmpty() == false) {
throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles));
}
}
private void failIfAnyMissing(String item, Boolean exists, String type) {
if (exists == false) {
throw new GradleException("Unused " + type + " " + item);
}
}
private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolean> notices, Set<File> shaFiles)
throws NoSuchAlgorithmException, IOException {
for (File dependency : dependencies) {
String jarName = dependency.getName();
String depName = regex.matcher(jarName).replaceFirst("");
validateSha(shaFiles, dependency, jarName, depName);
String dependencyName = getDependencyName(mappings, depName);
logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
checkFile(dependencyName, jarName, licenses, "LICENSE");
checkFile(dependencyName, jarName, notices, "NOTICE");
}
}
private void validateSha(Set<File> shaFiles, File dependency, String jarName, String depName)
throws NoSuchAlgorithmException, IOException {
if (ignoreShas.contains(depName)) {
// local deps should not have sha files!
if (getShaFile(jarName).exists()) {
throw new GradleException("SHA file " + getShaFile(jarName) + " exists for ignored dependency " + depName);
}
} else {
logger.info("Checking sha for {}", jarName);
checkSha(dependency, jarName, shaFiles);
}
}
private String joinFilenames(Set<File> shaFiles) {
List<String> names = shaFiles.stream().map(File::getName).collect(Collectors.toList());
return String.join("\n", names);
}
public static String getDependencyName(Map<String, String> mappings, String dependencyName) {
// order is the same for keys and values iteration since we use a linked hashmap
List<String> mapped = new ArrayList<>(mappings.values());
Pattern mappingsPattern = Pattern.compile("(" + String.join(")|(", mappings.keySet()) + ")");
Matcher match = mappingsPattern.matcher(dependencyName);
if (match.matches()) {
int i = 0;
while (i < match.groupCount() && match.group(i + 1) == null) {
++i;
}
return mapped.get(i);
}
return dependencyName;
}
private void checkSha(File jar, String jarName, Set<File> shaFiles) throws NoSuchAlgorithmException, IOException {
File shaFile = getShaFile(jarName);
if (shaFile.exists() == false) {
throw new GradleException("Missing SHA for " + jarName + ". Run \"gradle updateSHAs\" to create them");
}
// TODO: shouldn't have to trim, sha files should not have trailing newline
byte[] fileBytes = Files.readAllBytes(shaFile.toPath());
String expectedSha = new String(fileBytes, StandardCharsets.UTF_8).trim();
String sha = getSha1(jar);
if (expectedSha.equals(sha) == false) {
throw new GradleException(
"SHA has changed! Expected " + expectedSha + " for " + jarName + " but got " + sha + ". " +
"\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." +
"\nEither wipe your cache, fix the upstream artifact, or delete " + shaFile + " and run updateShas");
}
shaFiles.remove(shaFile);
}
private void checkFile(String name, String jarName, Map<String, Boolean> counters, String type) {
String fileName = getFileName(name, counters, type);
if (counters.containsKey(fileName) == false) {
throw new GradleException("Missing " + type + " for " + jarName + ", expected in " + fileName);
}
counters.put(fileName, true);
}
private String getFileName(String name, Map<String, ?> counters, String type) {
String fileName = name + "-" + type;
if (counters.containsKey(fileName) == false) {
// try the other suffix...TODO: get rid of this, just support ending in .txt
return fileName + ".txt";
}
return fileName;
}
@Input
public LinkedHashMap<String, String> getMappings() {
return new LinkedHashMap<>(mappings);
}
File getShaFile(String jarName) {
return new File(licensesDir, jarName + SHA_EXTENSION);
}
Set<File> getShaFiles() {
File[] array = licensesDir.listFiles();
if (array == null) {
throw new GradleException("\"" + licensesDir.getPath() + "\" isn't a valid directory");
}
return Arrays.stream(array)
.filter(file -> file.getName().endsWith(SHA_EXTENSION))
.collect(Collectors.toSet());
}
String getSha1(File file) throws IOException, NoSuchAlgorithmException {
byte[] bytes = Files.readAllBytes(file.toPath());
MessageDigest digest = MessageDigest.getInstance("SHA-1");
char[] encoded = Hex.encodeHex(digest.digest(bytes));
return String.copyValueOf(encoded);
}
}

86
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit;
import org.gradle.api.DefaultTask;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.tasks.TaskAction;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.security.NoSuchAlgorithmException;
import java.util.Set;
/**
* A task to update shas used by {@code DependencyLicensesCheck}
*/
public class UpdateShasTask extends DefaultTask {
private final Logger logger = Logging.getLogger(getClass());
/** The parent dependency licenses task to use configuration from */
private DependencyLicensesTask parentTask;
public UpdateShasTask() {
setDescription("Updates the sha files for the dependencyLicenses check");
setOnlyIf(element -> parentTask.getLicensesDir() != null);
}
@TaskAction
public void updateShas() throws NoSuchAlgorithmException, IOException {
Set<File> shaFiles = parentTask.getShaFiles();
for (File dependency : parentTask.getDependencies()) {
String jarName = dependency.getName();
File shaFile = parentTask.getShaFile(jarName);
if (shaFile.exists() == false) {
createSha(dependency, jarName, shaFile);
} else {
shaFiles.remove(shaFile);
}
}
for (File shaFile : shaFiles) {
logger.lifecycle("Removing unused sha " + shaFile.getName());
shaFile.delete();
}
}
private void createSha(File dependency, String jarName, File shaFile) throws IOException, NoSuchAlgorithmException {
logger.lifecycle("Adding sha for " + jarName);
String sha = parentTask.getSha1(dependency);
Files.write(shaFile.toPath(), sha.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
}
public DependencyLicensesTask getParentTask() {
return parentTask;
}
public void setParentTask(DependencyLicensesTask parentTask) {
this.parentTask = parentTask;
}
}

268
buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java Normal file
View File

@ -0,0 +1,268 @@
package org.elasticsearch.gradle.precommit;
import org.elasticsearch.gradle.test.GradleUnitTestCase;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.testfixtures.ProjectBuilder;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.containsString;
public class DependencyLicensesTaskTests extends GradleUnitTestCase {
@Rule
public ExpectedException expectedException = ExpectedException.none();
private UpdateShasTask updateShas;
private DependencyLicensesTask task;
private Project project;
private Dependency dependency;
@Before
public void prepare() {
project = createProject();
task = createDependencyLicensesTask(project);
updateShas = createUpdateShasTask(project, task);
dependency = project.getDependencies().localGroovy();
}
@Test
public void givenProjectWithLicensesDirButNoDependenciesThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("exists, but there are no dependencies"));
getLicensesDir(project).mkdir();
task.checkDependencies();
}
@Test
public void givenProjectWithoutLicensesDirButWithDependenciesThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("does not exist, but there are dependencies"));
project.getDependencies().add("compile", dependency);
task.checkDependencies();
}
@Test
public void givenProjectWithoutLicensesDirNorDependenciesThenShouldReturnSilently() throws Exception {
task.checkDependencies();
}
@Test
public void givenProjectWithDependencyButNoShaFileThenShouldReturnException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Missing SHA for "));
File licensesDir = getLicensesDir(project);
createFileIn(licensesDir, "groovy-all-LICENSE.txt", "");
createFileIn(licensesDir, "groovy-all-NOTICE.txt", "");
project.getDependencies().add("compile", project.getDependencies().localGroovy());
task.checkDependencies();
}
@Test
public void givenProjectWithDependencyButNoLicenseFileThenShouldReturnException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Missing LICENSE for "));
project.getDependencies().add("compile", project.getDependencies().localGroovy());
getLicensesDir(project).mkdir();
updateShas.updateShas();
task.checkDependencies();
}
@Test
public void givenProjectWithDependencyButNoNoticeFileThenShouldReturnException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Missing NOTICE for "));
project.getDependencies().add("compile", dependency);
createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", "");
updateShas.updateShas();
task.checkDependencies();
}
@Test
public void givenProjectWithDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception {
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy-all");
task.checkDependencies();
}
@Test
public void givenProjectWithALicenseButWithoutTheDependencyThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Unused license "));
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy-all");
createFileIn(licensesDir, "non-declared-LICENSE.txt", "");
task.checkDependencies();
}
@Test
public void givenProjectWithANoticeButWithoutTheDependencyThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Unused notice "));
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy-all");
createFileIn(licensesDir, "non-declared-NOTICE.txt", "");
task.checkDependencies();
}
@Test
public void givenProjectWithAShaButWithoutTheDependencyThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("Unused sha files found: \n"));
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy-all");
createFileIn(licensesDir, "non-declared.sha1", "");
task.checkDependencies();
}
@Test
public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() throws Exception {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("SHA has changed! Expected "));
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy-all");
Path groovySha = Files
.list(licensesDir.toPath())
.filter(file -> file.toFile().getName().contains("sha"))
.findFirst().get();
Files.write(groovySha, new byte[] { 1 }, StandardOpenOption.CREATE);
task.checkDependencies();
}
@Test
public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws Exception {
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createAllDefaultDependencyFiles(licensesDir, "groovy");
Map<String, String> mappings = new HashMap<>();
mappings.put("from", "groovy-all");
mappings.put("to", "groovy");
task.mapping(mappings);
task.checkDependencies();
}
@Test
public void givenProjectWithAIgnoreShaConfigurationAndNoShaFileThenShouldReturnSilently() throws Exception {
project.getDependencies().add("compile", dependency);
File licensesDir = getLicensesDir(project);
createFileIn(licensesDir, "groovy-all-LICENSE.txt", "");
createFileIn(licensesDir, "groovy-all-NOTICE.txt", "");
task.ignoreSha("groovy-all");
task.checkDependencies();
}
@Test
public void givenProjectWithoutLicensesDirWhenAskingForShaFilesThenShouldThrowException() {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("isn't a valid directory"));
task.getShaFiles();
}
private Project createProject() {
Project project = ProjectBuilder.builder().build();
project.getPlugins().apply(JavaPlugin.class);
return project;
}
private void createAllDefaultDependencyFiles(File licensesDir, String dependencyName) throws IOException, NoSuchAlgorithmException {
createFileIn(licensesDir, dependencyName + "-LICENSE.txt", "");
createFileIn(licensesDir, dependencyName + "-NOTICE.txt", "");
updateShas.updateShas();
}
private File getLicensesDir(Project project) {
return getFile(project, "licenses");
}
private File getFile(Project project, String fileName) {
return project.getProjectDir().toPath().resolve(fileName).toFile();
}
private void createFileIn(File parent, String name, String content) throws IOException {
parent.mkdir();
Path file = parent.toPath().resolve(name);
file.toFile().createNewFile();
Files.write(file, content.getBytes(StandardCharsets.UTF_8));
}
private UpdateShasTask createUpdateShasTask(Project project, DependencyLicensesTask dependencyLicensesTask) {
UpdateShasTask task = project.getTasks()
.register("updateShas", UpdateShasTask.class)
.get();
task.setParentTask(dependencyLicensesTask);
return task;
}
private DependencyLicensesTask createDependencyLicensesTask(Project project) {
DependencyLicensesTask task = project.getTasks()
.register("dependencyLicenses", DependencyLicensesTask.class)
.get();
task.setDependencies(getDependencies(project));
return task;
}
private FileCollection getDependencies(Project project) {
return project.getConfigurations().getByName("compile");
}
}

140
buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java Normal file
View File

@ -0,0 +1,140 @@
package org.elasticsearch.gradle.precommit;
import org.apache.commons.io.FileUtils;
import org.elasticsearch.gradle.test.GradleUnitTestCase;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.testfixtures.ProjectBuilder;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.NoSuchAlgorithmException;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
public class UpdateShasTaskTests extends GradleUnitTestCase {
@Rule
public ExpectedException expectedException = ExpectedException.none();
private UpdateShasTask task;
private Project project;
private Dependency dependency;
@Before
public void prepare() throws IOException {
project = createProject();
task = createUpdateShasTask(project);
dependency = project.getDependencies().localGroovy();
}
@Test
public void whenDependencyDoesntExistThenShouldDeleteDependencySha()
throws IOException, NoSuchAlgorithmException {
File unusedSha = createFileIn(getLicensesDir(project), "test.sha1", "");
task.updateShas();
assertFalse(unusedSha.exists());
}
@Test
public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile()
throws IOException, NoSuchAlgorithmException {
project.getDependencies().add("compile", dependency);
getLicensesDir(project).mkdir();
task.updateShas();
Path groovySha = Files
.list(getLicensesDir(project).toPath())
.findFirst().get();
assertTrue(groovySha.toFile().getName().startsWith("groovy-all"));
}
@Test
public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile()
throws IOException, NoSuchAlgorithmException {
project.getDependencies().add("compile", dependency);
File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next();
String groovyShaName = groovyJar.getName() + ".sha1";
File groovySha = createFileIn(getLicensesDir(project), groovyShaName, "content");
task.updateShas();
assertThat(FileUtils.readFileToString(groovySha), equalTo("content"));
}
@Test
public void whenLicensesDirDoesntExistThenShouldThrowException()
throws IOException, NoSuchAlgorithmException {
expectedException.expect(GradleException.class);
expectedException.expectMessage(containsString("isn't a valid directory"));
task.updateShas();
}
private Project createProject() {
Project project = ProjectBuilder.builder().build();
project.getPlugins().apply(JavaPlugin.class);
return project;
}
private File getLicensesDir(Project project) {
return getFile(project, "licenses");
}
private File getFile(Project project, String fileName) {
return project.getProjectDir().toPath().resolve(fileName).toFile();
}
private File createFileIn(File parent, String name, String content) throws IOException {
parent.mkdir();
Path path = parent.toPath().resolve(name);
File file = path.toFile();
Files.write(path, content.getBytes(), StandardOpenOption.CREATE);
return file;
}
private UpdateShasTask createUpdateShasTask(Project project) {
UpdateShasTask task = project.getTasks()
.register("updateShas", UpdateShasTask.class)
.get();
task.setParentTask(createDependencyLicensesTask(project));
return task;
}
private DependencyLicensesTask createDependencyLicensesTask(Project project) {
DependencyLicensesTask task = project.getTasks()
.register("dependencyLicenses", DependencyLicensesTask.class)
.get();
task.setDependencies(getDependencies(project));
return task;
}
private FileCollection getDependencies(Project project) {
return project.getConfigurations().getByName("compile");
}
}