From 47d861c23a13f6c80d1c07adb01f8a3dc933789a Mon Sep 17 00:00:00 2001 From: Gabryel Monteiro Date: Thu, 13 Jun 2019 06:12:02 -0300 Subject: [PATCH] Converting DependencyLicensesTask and UpdateShasTask to java (#41921) --- buildSrc/build.gradle | 2 + .../precommit/DependencyLicensesTask.groovy | 268 -------------- .../gradle/precommit/UpdateShasTask.groovy | 66 ---- .../precommit/DependencyLicensesTask.java | 328 ++++++++++++++++++ .../gradle/precommit/UpdateShasTask.java | 86 +++++ .../DependencyLicensesTaskTests.java | 268 ++++++++++++++ .../gradle/precommit/UpdateShasTaskTests.java | 140 ++++++++ 7 files changed, 824 insertions(+), 334 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index e974ac41038..339f1a75875 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -81,6 +81,8 @@ repositories { dependencies { compile localGroovy() + compile 'commons-codec:commons-codec:1.12' + compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3' compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4' compile 'com.netflix.nebula:gradle-info-plugin:3.0.3' diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy deleted file mode 100644 index 04fb023e205..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.precommit - -import org.gradle.api.DefaultTask -import org.gradle.api.GradleException -import org.gradle.api.InvalidUserDataException -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputDirectory -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.TaskAction - -import java.nio.file.Files -import java.security.MessageDigest -import java.util.regex.Matcher -import java.util.regex.Pattern - -/** - * A task to check licenses for dependencies. - * - * There are two parts to the check: - * - * - * The directory to find the license and sha files in defaults to the dir @{code licenses} - * in the project directory for this task. You can override this directory: - * 
- *   dependencyLicenses {
- *     licensesDir = project.file('mybetterlicensedir')
- *   }
- *
- * - * The jar files to check default to the dependencies from the default configuration. You - * can override this, for example, to only check compile dependencies: - * 
- *   dependencyLicenses {
- *     dependencies = project.configurations.compile
- *   }
- *
- * - * Every jar must have a {@code .sha1} file in the licenses dir. These can be managed - * automatically using the {@code updateShas} helper task that is created along - * with this task. It will add {@code .sha1} files for new jars that are in dependencies - * and remove old {@code .sha1} files that are no longer needed. - * - * Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share - * LICENSE and NOTICE files by mapping a pattern to the same name. - * 
- *   dependencyLicenses {
- *     mapping from: /lucene-.*/, to: 'lucene'
- *   }
- *
- */ -public class DependencyLicensesTask extends DefaultTask { - static final String SHA_EXTENSION = '.sha1' - - // TODO: we should be able to default this to eg compile deps, but we need to move the licenses - // check from distribution to core (ie this should only be run on java projects) - /** A collection of jar files that should be checked. */ - @InputFiles - public FileCollection dependencies - - /** The directory to find the license and sha files in. */ - @InputDirectory - public File licensesDir = new File(project.projectDir, 'licenses') - - /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */ - private LinkedHashMap mappings = new LinkedHashMap<>() - - /** Names of dependencies whose shas should not exist. */ - private Set ignoreShas = new HashSet<>() - - /** - * Add a mapping from a regex pattern for the jar name, to a prefix to find - * the LICENSE and NOTICE file for that jar. - */ - @Input - public void mapping(Map props) { - String from = props.remove('from') - if (from == null) { - throw new InvalidUserDataException('Missing "from" setting for license name mapping') - } - String to = props.remove('to') - if (to == null) { - throw new InvalidUserDataException('Missing "to" setting for license name mapping') - } - if (props.isEmpty() == false) { - throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: ${props.keySet()}") - } - mappings.put(from, to) - } - - public LinkedHashMap getMappings() { - return new LinkedHashMap<>(mappings) - } - - /** - * Add a rule which will skip SHA checking for the given dependency name. This should be used for - * locally build dependencies, which cause the sha to change constantly. - */ - @Input - public void ignoreSha(String dep) { - ignoreShas.add(dep) - } - - @TaskAction - public void checkDependencies() { - if (dependencies.isEmpty()) { - if (licensesDir.exists()) { - throw new GradleException("Licenses dir ${licensesDir} exists, but there are no dependencies") - } - return // no dependencies to check - } else if (licensesDir.exists() == false) { - throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies") - } - - Map licenses = new HashMap<>() - Map notices = new HashMap<>() - Set shaFiles = new HashSet() - - licensesDir.eachFile { - String name = it.getName() - if (name.endsWith(SHA_EXTENSION)) { - shaFiles.add(it) - } else if (name.endsWith('-LICENSE') || name.endsWith('-LICENSE.txt')) { - // TODO: why do we support suffix of LICENSE *and* LICENSE.txt?? - licenses.put(name, 0) - } else if (name.contains('-NOTICE') || name.contains('-NOTICE.txt')) { - notices.put(name, 0) - } - } - - for (File dependency : dependencies) { - String jarName = dependency.getName() - String depName = jarName - ~/\-v?\d+.*/ - if (ignoreShas.contains(depName)) { - // local deps should not have sha files! - if (getShaFile(jarName).exists()) { - throw new GradleException("SHA file ${getShaFile(jarName)} exists for ignored dependency ${depName}") - } - } else { - logger.info("Checking sha for " + jarName) - checkSha(dependency, jarName, shaFiles) - } - - final String dependencyName = getDependencyName(mappings, depName) - logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check") - checkFile(dependencyName, jarName, licenses, 'LICENSE') - checkFile(dependencyName, jarName, notices, 'NOTICE') - } - - licenses.each { license, count -> - if (count == 0) { - throw new GradleException("Unused license ${license}") - } - } - notices.each { notice, count -> - if (count == 0) { - throw new GradleException("Unused notice ${notice}") - } - } - if (shaFiles.isEmpty() == false) { - throw new GradleException("Unused sha files found: \n${shaFiles.join('\n')}") - } - } - - public static String getDependencyName(final LinkedHashMap mappings, final String dependencyName) { - // order is the same for keys and values iteration since we use a linked hashmap - List mapped = new ArrayList<>(mappings.values()) - Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') - Matcher match = mappingsPattern.matcher(dependencyName) - if (match.matches()) { - int i = 0 - while (i < match.groupCount() && match.group(i + 1) == null) ++i; - return mapped.get(i) - } - return dependencyName - } - - private File getShaFile(String jarName) { - return new File(licensesDir, jarName + SHA_EXTENSION) - } - - private void checkSha(File jar, String jarName, Set shaFiles) { - File shaFile = getShaFile(jarName) - if (shaFile.exists() == false) { - throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create") - } - // TODO: shouldn't have to trim, sha files should not have trailing newline - String expectedSha = shaFile.getText('UTF-8').trim() - String sha = MessageDigest.getInstance("SHA-1").digest(jar.getBytes()).encodeHex().toString() - if (expectedSha.equals(sha) == false) { - throw new GradleException("SHA has changed! Expected ${expectedSha} for ${jarName} but got ${sha}. " + - "\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." + - "\nEither wipe your cache, fix the upstream artifact, or delete ${shaFile} and run updateShas") - } - shaFiles.remove(shaFile) - } - - private void checkFile(String name, String jarName, Map counters, String type) { - String fileName = "${name}-${type}" - Integer count = counters.get(fileName) - if (count == null) { - // try the other suffix...TODO: get rid of this, just support ending in .txt - fileName = "${fileName}.txt" - counters.get(fileName) - } - count = counters.get(fileName) - if (count == null) { - throw new GradleException("Missing ${type} for ${jarName}, expected in ${fileName}") - } - counters.put(fileName, count + 1) - } - - /** A helper task to update the sha files in the license dir. */ - public static class UpdateShasTask extends DefaultTask { - private DependencyLicensesTask parentTask - - @TaskAction - public void updateShas() { - Set shaFiles = new HashSet() - parentTask.licensesDir.eachFile { - String name = it.getName() - if (name.endsWith(SHA_EXTENSION)) { - shaFiles.add(it) - } - } - for (File dependency : parentTask.dependencies) { - String jarName = dependency.getName() - String depName = jarName - ~/\-\d+.*/ - if (parentTask.ignoreShas.contains(depName)) { - continue - } - File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION) - if (shaFile.exists() == false) { - logger.lifecycle("Adding sha for ${jarName}") - String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString() - shaFile.setText(sha, 'UTF-8') - } else { - shaFiles.remove(shaFile) - } - } - shaFiles.each { shaFile -> - logger.lifecycle("Removing unused sha ${shaFile.getName()}") - Files.delete(shaFile.toPath()) - } - } - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy deleted file mode 100644 index 4a174688aa1..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle.precommit - -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.TaskAction - -import java.nio.file.Files -import java.security.MessageDigest - -/** - * A task to update shas used by {@code DependencyLicensesCheck} - */ -public class UpdateShasTask extends DefaultTask { - - /** The parent dependency licenses task to use configuration from */ - public DependencyLicensesTask parentTask - - public UpdateShasTask() { - description = 'Updates the sha files for the dependencyLicenses check' - onlyIf { parentTask.licensesDir.exists() } - } - - @TaskAction - public void updateShas() { - Set shaFiles = new HashSet() - parentTask.licensesDir.eachFile { - String name = it.getName() - if (name.endsWith(DependencyLicensesTask.SHA_EXTENSION)) { - shaFiles.add(it) - } - } - for (File dependency : parentTask.dependencies) { - String jarName = dependency.getName() - File shaFile = new File(parentTask.licensesDir, jarName + DependencyLicensesTask.SHA_EXTENSION) - if (shaFile.exists() == false) { - logger.lifecycle("Adding sha for ${jarName}") - String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString() - shaFile.setText(sha, 'UTF-8') - } else { - shaFiles.remove(shaFile) - } - } - shaFiles.each { shaFile -> - logger.lifecycle("Removing unused sha ${shaFile.getName()}") - Files.delete(shaFile.toPath()) - } - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java new file mode 100644 index 00000000000..d884207d590 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java @@ -0,0 +1,328 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.precommit; + +import org.apache.commons.codec.binary.Hex; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.file.FileCollection; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * A task to check licenses for dependencies. + * + * There are two parts to the check: + *
    + *
  • LICENSE and NOTICE files
    • + *
  • SHA checksums for each dependency jar
    • + *
+ * + * The directory to find the license and sha files in defaults to the dir @{code licenses} + * in the project directory for this task. You can override this directory: + * 
+ *   dependencyLicenses {
+ *     licensesDir = getProject().file("mybetterlicensedir")
+ *   }
+ *
+ * + * The jar files to check default to the dependencies from the default configuration. You + * can override this, for example, to only check compile dependencies: + * 
+ *   dependencyLicenses {
+ *     dependencies = getProject().configurations.compile
+ *   }
+ *
+ * + * Every jar must have a {@code .sha1} file in the licenses dir. These can be managed + * automatically using the {@code updateShas} helper task that is created along + * with this task. It will add {@code .sha1} files for new jars that are in dependencies + * and remove old {@code .sha1} files that are no longer needed. + * + * Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share + * LICENSE and NOTICE files by mapping a pattern to the same name. + * 
+ *   dependencyLicenses {
+ *     mapping from: /lucene-.*/, to: "lucene"
+ *   }
+ *
+ */ +public class DependencyLicensesTask extends DefaultTask { + + private final Pattern regex = Pattern.compile("-v?\\d+.*"); + + private final Logger logger = Logging.getLogger(getClass()); + + private static final String SHA_EXTENSION = ".sha1"; + + // TODO: we should be able to default this to eg compile deps, but we need to move the licenses + // check from distribution to core (ie this should only be run on java projects) + /** A collection of jar files that should be checked. */ + private FileCollection dependencies; + + /** The directory to find the license and sha files in. */ + private File licensesDir = new File(getProject().getProjectDir(), "licenses"); + + /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */ + private Map mappings = new LinkedHashMap<>(); + + /** Names of dependencies whose shas should not exist. */ + private Set ignoreShas = new HashSet<>(); + + /** + * Add a mapping from a regex pattern for the jar name, to a prefix to find + * the LICENSE and NOTICE file for that jar. + */ + public void mapping(Map props) { + String from = props.remove("from"); + if (from == null) { + throw new InvalidUserDataException("Missing \"from\" setting for license name mapping"); + } + String to = props.remove("to"); + if (to == null) { + throw new InvalidUserDataException("Missing \"to\" setting for license name mapping"); + } + if (props.isEmpty() == false) { + throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet()); + } + mappings.put(from, to); + } + + @InputFiles + public FileCollection getDependencies() { + return dependencies; + } + + public void setDependencies(FileCollection dependencies) { + this.dependencies = dependencies; + } + + @Optional + @InputDirectory + public File getLicensesDir() { + if (licensesDir.exists()) { + return licensesDir; + } + + return null; + } + + public void setLicensesDir(File licensesDir) { + this.licensesDir = licensesDir; + } + + /** + * Add a rule which will skip SHA checking for the given dependency name. This should be used for + * locally build dependencies, which cause the sha to change constantly. + */ + public void ignoreSha(String dep) { + ignoreShas.add(dep); + } + + @TaskAction + public void checkDependencies() throws IOException, NoSuchAlgorithmException { + if (dependencies == null) { + throw new GradleException("No dependencies variable defined."); + } + + if (dependencies.isEmpty()) { + if (licensesDir.exists()) { + throw new GradleException("Licenses dir " + licensesDir + " exists, but there are no dependencies"); + } + return; // no dependencies to check + } else if (licensesDir.exists() == false) { + throw new GradleException("Licences dir " + licensesDir + " does not exist, but there are dependencies"); + } + + Map licenses = new HashMap<>(); + Map notices = new HashMap<>(); + Set shaFiles = new HashSet<>(); + + for (File file : licensesDir.listFiles()) { + String name = file.getName(); + if (name.endsWith(SHA_EXTENSION)) { + shaFiles.add(file); + } else if (name.endsWith("-LICENSE") || name.endsWith("-LICENSE.txt")) { + // TODO: why do we support suffix of LICENSE *and* LICENSE.txt?? + licenses.put(name, false); + } else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) { + notices.put(name, false); + } + } + + checkDependencies(licenses, notices, shaFiles); + + licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license")); + + notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice")); + + if (shaFiles.isEmpty() == false) { + throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles)); + } + } + + private void failIfAnyMissing(String item, Boolean exists, String type) { + if (exists == false) { + throw new GradleException("Unused " + type + " " + item); + } + } + + private void checkDependencies(Map licenses, Map notices, Set shaFiles) + throws NoSuchAlgorithmException, IOException { + for (File dependency : dependencies) { + String jarName = dependency.getName(); + String depName = regex.matcher(jarName).replaceFirst(""); + + validateSha(shaFiles, dependency, jarName, depName); + + String dependencyName = getDependencyName(mappings, depName); + logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName); + checkFile(dependencyName, jarName, licenses, "LICENSE"); + checkFile(dependencyName, jarName, notices, "NOTICE"); + } + } + + private void validateSha(Set shaFiles, File dependency, String jarName, String depName) + throws NoSuchAlgorithmException, IOException { + if (ignoreShas.contains(depName)) { + // local deps should not have sha files! + if (getShaFile(jarName).exists()) { + throw new GradleException("SHA file " + getShaFile(jarName) + " exists for ignored dependency " + depName); + } + } else { + logger.info("Checking sha for {}", jarName); + checkSha(dependency, jarName, shaFiles); + } + } + + private String joinFilenames(Set shaFiles) { + List names = shaFiles.stream().map(File::getName).collect(Collectors.toList()); + return String.join("\n", names); + } + + public static String getDependencyName(Map mappings, String dependencyName) { + // order is the same for keys and values iteration since we use a linked hashmap + List mapped = new ArrayList<>(mappings.values()); + Pattern mappingsPattern = Pattern.compile("(" + String.join(")|(", mappings.keySet()) + ")"); + Matcher match = mappingsPattern.matcher(dependencyName); + if (match.matches()) { + int i = 0; + while (i < match.groupCount() && match.group(i + 1) == null) { + ++i; + } + return mapped.get(i); + } + return dependencyName; + } + + private void checkSha(File jar, String jarName, Set shaFiles) throws NoSuchAlgorithmException, IOException { + File shaFile = getShaFile(jarName); + if (shaFile.exists() == false) { + throw new GradleException("Missing SHA for " + jarName + ". Run \"gradle updateSHAs\" to create them"); + } + + // TODO: shouldn't have to trim, sha files should not have trailing newline + byte[] fileBytes = Files.readAllBytes(shaFile.toPath()); + String expectedSha = new String(fileBytes, StandardCharsets.UTF_8).trim(); + + String sha = getSha1(jar); + + if (expectedSha.equals(sha) == false) { + throw new GradleException( + "SHA has changed! Expected " + expectedSha + " for " + jarName + " but got " + sha + ". " + + "\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." + + "\nEither wipe your cache, fix the upstream artifact, or delete " + shaFile + " and run updateShas"); + } + shaFiles.remove(shaFile); + } + + private void checkFile(String name, String jarName, Map counters, String type) { + String fileName = getFileName(name, counters, type); + + if (counters.containsKey(fileName) == false) { + throw new GradleException("Missing " + type + " for " + jarName + ", expected in " + fileName); + } + + counters.put(fileName, true); + } + + private String getFileName(String name, Map counters, String type) { + String fileName = name + "-" + type; + + if (counters.containsKey(fileName) == false) { + // try the other suffix...TODO: get rid of this, just support ending in .txt + return fileName + ".txt"; + } + + return fileName; + } + + @Input + public LinkedHashMap getMappings() { + return new LinkedHashMap<>(mappings); + } + + File getShaFile(String jarName) { + return new File(licensesDir, jarName + SHA_EXTENSION); + } + + Set getShaFiles() { + File[] array = licensesDir.listFiles(); + if (array == null) { + throw new GradleException("\"" + licensesDir.getPath() + "\" isn't a valid directory"); + } + + return Arrays.stream(array) + .filter(file -> file.getName().endsWith(SHA_EXTENSION)) + .collect(Collectors.toSet()); + } + + String getSha1(File file) throws IOException, NoSuchAlgorithmException { + byte[] bytes = Files.readAllBytes(file.toPath()); + + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + char[] encoded = Hex.encodeHex(digest.digest(bytes)); + return String.copyValueOf(encoded); + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java new file mode 100644 index 00000000000..db3148da696 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.precommit; + +import org.gradle.api.DefaultTask; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.security.NoSuchAlgorithmException; +import java.util.Set; + +/** + * A task to update shas used by {@code DependencyLicensesCheck} + */ +public class UpdateShasTask extends DefaultTask { + + private final Logger logger = Logging.getLogger(getClass()); + + /** The parent dependency licenses task to use configuration from */ + private DependencyLicensesTask parentTask; + + public UpdateShasTask() { + setDescription("Updates the sha files for the dependencyLicenses check"); + setOnlyIf(element -> parentTask.getLicensesDir() != null); + } + + @TaskAction + public void updateShas() throws NoSuchAlgorithmException, IOException { + Set shaFiles = parentTask.getShaFiles(); + + for (File dependency : parentTask.getDependencies()) { + String jarName = dependency.getName(); + File shaFile = parentTask.getShaFile(jarName); + + if (shaFile.exists() == false) { + createSha(dependency, jarName, shaFile); + } else { + shaFiles.remove(shaFile); + } + } + + for (File shaFile : shaFiles) { + logger.lifecycle("Removing unused sha " + shaFile.getName()); + shaFile.delete(); + } + } + + private void createSha(File dependency, String jarName, File shaFile) throws IOException, NoSuchAlgorithmException { + logger.lifecycle("Adding sha for " + jarName); + + String sha = parentTask.getSha1(dependency); + + Files.write(shaFile.toPath(), sha.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + } + + public DependencyLicensesTask getParentTask() { + return parentTask; + } + + public void setParentTask(DependencyLicensesTask parentTask) { + this.parentTask = parentTask; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java new file mode 100644 index 00000000000..397c5938fba --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java @@ -0,0 +1,268 @@ +package org.elasticsearch.gradle.precommit; + +import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.file.FileCollection; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.testfixtures.ProjectBuilder; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.containsString; + +public class DependencyLicensesTaskTests extends GradleUnitTestCase { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private UpdateShasTask updateShas; + + private DependencyLicensesTask task; + + private Project project; + + private Dependency dependency; + + @Before + public void prepare() { + project = createProject(); + task = createDependencyLicensesTask(project); + updateShas = createUpdateShasTask(project, task); + dependency = project.getDependencies().localGroovy(); + } + + @Test + public void givenProjectWithLicensesDirButNoDependenciesThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("exists, but there are no dependencies")); + + getLicensesDir(project).mkdir(); + task.checkDependencies(); + } + + @Test + public void givenProjectWithoutLicensesDirButWithDependenciesThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("does not exist, but there are dependencies")); + + project.getDependencies().add("compile", dependency); + task.checkDependencies(); + } + + @Test + public void givenProjectWithoutLicensesDirNorDependenciesThenShouldReturnSilently() throws Exception { + task.checkDependencies(); + } + + @Test + public void givenProjectWithDependencyButNoShaFileThenShouldReturnException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Missing SHA for ")); + + File licensesDir = getLicensesDir(project); + createFileIn(licensesDir, "groovy-all-LICENSE.txt", ""); + createFileIn(licensesDir, "groovy-all-NOTICE.txt", ""); + + project.getDependencies().add("compile", project.getDependencies().localGroovy()); + task.checkDependencies(); + } + + @Test + public void givenProjectWithDependencyButNoLicenseFileThenShouldReturnException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Missing LICENSE for ")); + + project.getDependencies().add("compile", project.getDependencies().localGroovy()); + + getLicensesDir(project).mkdir(); + updateShas.updateShas(); + task.checkDependencies(); + } + + @Test + public void givenProjectWithDependencyButNoNoticeFileThenShouldReturnException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Missing NOTICE for ")); + + project.getDependencies().add("compile", dependency); + + createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", ""); + + updateShas.updateShas(); + task.checkDependencies(); + } + + @Test + public void givenProjectWithDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception { + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + + createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + task.checkDependencies(); + } + + @Test + public void givenProjectWithALicenseButWithoutTheDependencyThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Unused license ")); + + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createFileIn(licensesDir, "non-declared-LICENSE.txt", ""); + + task.checkDependencies(); + } + + @Test + public void givenProjectWithANoticeButWithoutTheDependencyThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Unused notice ")); + + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createFileIn(licensesDir, "non-declared-NOTICE.txt", ""); + + task.checkDependencies(); + } + + @Test + public void givenProjectWithAShaButWithoutTheDependencyThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("Unused sha files found: \n")); + + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createFileIn(licensesDir, "non-declared.sha1", ""); + + task.checkDependencies(); + } + + @Test + public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() throws Exception { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("SHA has changed! Expected ")); + + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + + Path groovySha = Files + .list(licensesDir.toPath()) + .filter(file -> file.toFile().getName().contains("sha")) + .findFirst().get(); + + Files.write(groovySha, new byte[] { 1 }, StandardOpenOption.CREATE); + + task.checkDependencies(); + } + + @Test + public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws Exception { + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createAllDefaultDependencyFiles(licensesDir, "groovy"); + + Map mappings = new HashMap<>(); + mappings.put("from", "groovy-all"); + mappings.put("to", "groovy"); + + task.mapping(mappings); + task.checkDependencies(); + } + + @Test + public void givenProjectWithAIgnoreShaConfigurationAndNoShaFileThenShouldReturnSilently() throws Exception { + project.getDependencies().add("compile", dependency); + + File licensesDir = getLicensesDir(project); + createFileIn(licensesDir, "groovy-all-LICENSE.txt", ""); + createFileIn(licensesDir, "groovy-all-NOTICE.txt", ""); + + task.ignoreSha("groovy-all"); + task.checkDependencies(); + } + + @Test + public void givenProjectWithoutLicensesDirWhenAskingForShaFilesThenShouldThrowException() { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("isn't a valid directory")); + + task.getShaFiles(); + } + + private Project createProject() { + Project project = ProjectBuilder.builder().build(); + project.getPlugins().apply(JavaPlugin.class); + + return project; + } + + private void createAllDefaultDependencyFiles(File licensesDir, String dependencyName) throws IOException, NoSuchAlgorithmException { + createFileIn(licensesDir, dependencyName + "-LICENSE.txt", ""); + createFileIn(licensesDir, dependencyName + "-NOTICE.txt", ""); + + updateShas.updateShas(); + } + + private File getLicensesDir(Project project) { + return getFile(project, "licenses"); + } + + private File getFile(Project project, String fileName) { + return project.getProjectDir().toPath().resolve(fileName).toFile(); + } + + private void createFileIn(File parent, String name, String content) throws IOException { + parent.mkdir(); + + Path file = parent.toPath().resolve(name); + file.toFile().createNewFile(); + + Files.write(file, content.getBytes(StandardCharsets.UTF_8)); + } + + private UpdateShasTask createUpdateShasTask(Project project, DependencyLicensesTask dependencyLicensesTask) { + UpdateShasTask task = project.getTasks() + .register("updateShas", UpdateShasTask.class) + .get(); + + task.setParentTask(dependencyLicensesTask); + return task; + } + + private DependencyLicensesTask createDependencyLicensesTask(Project project) { + DependencyLicensesTask task = project.getTasks() + .register("dependencyLicenses", DependencyLicensesTask.class) + .get(); + + task.setDependencies(getDependencies(project)); + return task; + } + + private FileCollection getDependencies(Project project) { + return project.getConfigurations().getByName("compile"); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java new file mode 100644 index 00000000000..62ac9600a83 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java @@ -0,0 +1,140 @@ +package org.elasticsearch.gradle.precommit; + +import org.apache.commons.io.FileUtils; +import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.file.FileCollection; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.testfixtures.ProjectBuilder; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.NoSuchAlgorithmException; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; + +public class UpdateShasTaskTests extends GradleUnitTestCase { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private UpdateShasTask task; + + private Project project; + + private Dependency dependency; + + @Before + public void prepare() throws IOException { + project = createProject(); + task = createUpdateShasTask(project); + dependency = project.getDependencies().localGroovy(); + + } + + @Test + public void whenDependencyDoesntExistThenShouldDeleteDependencySha() + throws IOException, NoSuchAlgorithmException { + + File unusedSha = createFileIn(getLicensesDir(project), "test.sha1", ""); + task.updateShas(); + + assertFalse(unusedSha.exists()); + } + + @Test + public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile() + throws IOException, NoSuchAlgorithmException { + project.getDependencies().add("compile", dependency); + + getLicensesDir(project).mkdir(); + task.updateShas(); + + Path groovySha = Files + .list(getLicensesDir(project).toPath()) + .findFirst().get(); + + assertTrue(groovySha.toFile().getName().startsWith("groovy-all")); + } + + @Test + public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile() + throws IOException, NoSuchAlgorithmException { + project.getDependencies().add("compile", dependency); + + File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next(); + String groovyShaName = groovyJar.getName() + ".sha1"; + + File groovySha = createFileIn(getLicensesDir(project), groovyShaName, "content"); + task.updateShas(); + + assertThat(FileUtils.readFileToString(groovySha), equalTo("content")); + } + + @Test + public void whenLicensesDirDoesntExistThenShouldThrowException() + throws IOException, NoSuchAlgorithmException { + expectedException.expect(GradleException.class); + expectedException.expectMessage(containsString("isn't a valid directory")); + + task.updateShas(); + } + + private Project createProject() { + Project project = ProjectBuilder.builder().build(); + project.getPlugins().apply(JavaPlugin.class); + + return project; + } + + private File getLicensesDir(Project project) { + return getFile(project, "licenses"); + } + + private File getFile(Project project, String fileName) { + return project.getProjectDir().toPath().resolve(fileName).toFile(); + } + + private File createFileIn(File parent, String name, String content) throws IOException { + parent.mkdir(); + + Path path = parent.toPath().resolve(name); + File file = path.toFile(); + + Files.write(path, content.getBytes(), StandardOpenOption.CREATE); + + return file; + } + + private UpdateShasTask createUpdateShasTask(Project project) { + UpdateShasTask task = project.getTasks() + .register("updateShas", UpdateShasTask.class) + .get(); + + task.setParentTask(createDependencyLicensesTask(project)); + return task; + } + + private DependencyLicensesTask createDependencyLicensesTask(Project project) { + DependencyLicensesTask task = project.getTasks() + .register("dependencyLicenses", DependencyLicensesTask.class) + .get(); + + task.setDependencies(getDependencies(project)); + return task; + } + + private FileCollection getDependencies(Project project) { + return project.getConfigurations().getByName("compile"); + } +}