Merge pull request #15545 from rmuir/rat

add gradle licenseHeaders to precommit

build.gradle

@@ -128,10 +128,11 @@ subprojects {
   // Only if your buildscript and Ant's optional task need the same library would you have to define it twice.
   // https://docs.gradle.org/current/userguide/organizing_build_logic.html
   configurations {
-    forbiddenApis
+    buildTools
   }
   dependencies {
-    forbiddenApis 'de.thetaphi:forbiddenapis:2.0'
+    buildTools 'de.thetaphi:forbiddenapis:2.0'
+    buildTools 'org.apache.rat:apache-rat:0.11'
   }
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit
+
+import java.nio.file.Files
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.SourceSet
+import org.gradle.api.tasks.TaskAction
+
+import groovy.xml.NamespaceBuilder
+import groovy.xml.NamespaceBuilderSupport
+
+/**
+ * Checks files for license headers.
+ * <p>
+ * This is a port of the apache lucene check
+ */
+public class LicenseHeadersTask extends DefaultTask {
+
+    LicenseHeadersTask() {
+        description = "Checks sources for missing, incorrect, or unacceptable license headers"
+    }
+
+    @TaskAction
+    public void check() {
+        // load rat tasks
+        AntBuilder ant = new AntBuilder()
+        ant.typedef(resource:  "org/apache/rat/anttasks/antlib.xml",
+                    uri:       "antlib:org.apache.rat.anttasks",
+                    classpath: project.configurations.buildTools.asPath)
+        NamespaceBuilderSupport rat = NamespaceBuilder.newInstance(ant, "antlib:org.apache.rat.anttasks")
+
+        // create a file for the log to go to under reports/
+        File reportDir = new File(project.buildDir, "reports/licenseHeaders")
+        reportDir.mkdirs()
+        File reportFile = new File(reportDir, "rat.log")
+        Files.deleteIfExists(reportFile.toPath())
+                     
+        // run rat, going to the file
+        rat.report(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
+               // checks all the java sources (allJava)
+               for (SourceSet set : project.sourceSets) {
+                   for (File dir : set.allJava.srcDirs) {
+                       // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
+                       if (dir.exists()) {
+                           ant.fileset(dir: dir)
+                       }
+                   }
+               }
+
+               // BSD 4-clause stuff (is disallowed below)
+               substringMatcher(licenseFamilyCategory: "BSD4 ",
+                                licenseFamilyName:     "Original BSD License (with advertising clause)") {
+                   pattern(substring: "All advertising materials")
+               }
+
+               // BSD-like stuff
+               substringMatcher(licenseFamilyCategory: "BSD  ",
+                                licenseFamilyName:     "Modified BSD License") {
+                   // brics automaton
+                   pattern(substring: "Copyright (c) 2001-2009 Anders Moeller")
+                   // snowball
+                   pattern(substring: "Copyright (c) 2001, Dr Martin Porter")
+                   // UMASS kstem
+                   pattern(substring: "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS")
+                   // Egothor
+                   pattern(substring: "Egothor Software License version 1.00")
+                   // JaSpell
+                   pattern(substring: "Copyright (c) 2005 Bruno Martins")
+                   // d3.js
+                   pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS")
+                   // highlight.js
+                   pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS")
+               }
+
+               // MIT-like
+               substringMatcher(licenseFamilyCategory: "MIT  ",
+                                licenseFamilyName:     "The MIT License") {
+                   // ICU license
+                   pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy")
+               }
+
+               // Apache
+               substringMatcher(licenseFamilyCategory: "AL   ",
+                                licenseFamilyName:     "Apache") {
+                   // Apache license (ES)
+                   pattern(substring: "Licensed to Elasticsearch under one or more contributor")
+                   // Apache license (ASF)
+                   pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
+                   // this is the old-school one under some files
+                   pattern(substring: "Licensed under the Apache License, Version 2.0 (the \"License\")")
+               }
+
+               // Generated resources
+               substringMatcher(licenseFamilyCategory: "GEN  ",
+                                licenseFamilyName:     "Generated") {
+                   // svg files generated by gnuplot
+                   pattern(substring: "Produced by GNUPLOT")
+                   // snowball stemmers generated by snowball compiler
+                   pattern(substring: "This file was generated automatically by the Snowball to Java compiler")
+                   // uima tests generated by JCasGen
+                   pattern(substring: "First created by JCasGen")
+                   // parsers generated by antlr
+                   pattern(substring: "ANTLR GENERATED CODE")
+               }
+                
+               // approved categories
+               approvedLicense(familyName: "Apache")
+               approvedLicense(familyName: "The MIT License")
+               approvedLicense(familyName: "Modified BSD License")
+               approvedLicense(familyName: "Generated") 
+        }
+        
+        // check the license file for any errors, this should be fast.
+        boolean zeroUnknownLicenses = false
+        boolean foundProblemsWithFiles = false
+        reportFile.eachLine('UTF-8') { line ->
+            if (line.startsWith("0 Unknown Licenses")) {
+                zeroUnknownLicenses = true
+            }
+            
+            if (line.startsWith(" !")) {
+                foundProblemsWithFiles = true
+            }
+        }
+        
+        if (zeroUnknownLicenses == false || foundProblemsWithFiles) {
+            // print the unapproved license section, usually its all you need to fix problems.
+            int sectionNumber = 0
+            reportFile.eachLine('UTF-8') { line ->
+                if (line.startsWith("*******************************")) {
+                    sectionNumber++
+                } else {
+                    if (sectionNumber == 2) {
+                        logger.error(line)
+                    }
+                }
+            }
+            throw new IllegalStateException("License header problems were found! Full details: " + reportFile.absolutePath)
+        }
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -34,6 +34,7 @@ class PrecommitTasks {
         List<Task> precommitTasks = [
             configureForbiddenApis(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
+            project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
             project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy

@@ -114,7 +114,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
         
         ant.taskdef(name:      "thirdPartyAudit",
                     classname: "de.thetaphi.forbiddenapis.ant.AntTask",
-                    classpath: project.configurations.forbiddenApis.asPath)
+                    classpath: project.configurations.buildTools.asPath)
         
         // print which jars we are going to scan, always
         // this is not the time to try to be succinct! Forbidden will print plenty on its own!

core/src/main/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandler.java

@@ -1,5 +1,27 @@
 package org.elasticsearch.http.netty.pipelining;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// this file is from netty-http-pipelining, under apache 2.0 license
+// see github.com/typesafehub/netty-http-pipelining
+
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.jboss.netty.channel.*;

core/src/main/java/org/elasticsearch/http/netty/pipelining/OrderedDownstreamChannelEvent.java

@@ -1,5 +1,27 @@
 package org.elasticsearch.http.netty.pipelining;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// this file is from netty-http-pipelining, under apache 2.0 license
+// see github.com/typesafehub/netty-http-pipelining
+
 import org.jboss.netty.channel.*;
 
 /**

core/src/main/java/org/elasticsearch/http/netty/pipelining/OrderedUpstreamMessageEvent.java

@@ -1,5 +1,27 @@
 package org.elasticsearch.http.netty.pipelining;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// this file is from netty-http-pipelining, under apache 2.0 license
+// see github.com/typesafehub/netty-http-pipelining
+
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.UpstreamMessageEvent;
 

core/src/main/java/org/elasticsearch/search/profile/InternalProfileShardResults.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.search.profile;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;

core/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.action.support.master;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;

core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.cluster.routing;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;

core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.cluster.routing.allocation;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaImplTests.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.mapper.attachments;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.test.ESTestCase;
 
 public class TikaImplTests extends ESTestCase {

plugins/repository-hdfs/src/main/java/org/elasticsearch/plugin/hadoop/hdfs/Utils.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.plugin.hadoop.hdfs;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import java.net.URL;
 import java.security.AccessControlContext;
 import java.security.AccessController;
@@ -81,4 +100,4 @@ public abstract class Utils {
 
         return base;
     }
-}
+}

plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsRepositoryRestIT.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.plugin.hadoop.hdfs;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import java.io.IOException;
 import java.util.Collection;
 

plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/HdfsTestPlugin.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.plugin.hadoop.hdfs;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import java.net.URL;
 import java.util.Collections;
 import java.util.List;

plugins/repository-hdfs/src/test/java/org/elasticsearch/plugin/hadoop/hdfs/UtilsTests.java

@@ -1,5 +1,24 @@
 package org.elasticsearch.plugin.hadoop.hdfs;
 
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import org.elasticsearch.test.ESTestCase;
 
 public class UtilsTests extends ESTestCase {