Fix license check in travis and make it optional (#8049)

* Fix license check in travis and make it optional

* debug

* fix build

* too loud maven

* move MAVEN_OPTS to top and add comments

* adjust script

* remove mvn option from python script

.travis.yml

@@ -50,8 +50,8 @@ matrix:
         - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
         - pip3 install wheel # install wheel first explicitly
         - pip3 install pyyaml
-      install: true
+      install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B
-      script: MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts
+      script: ./build.sh
 
       # processing module test
     - env:

build.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MAVEN_OPTS='-Xmx3000m'
+
+mkdir -p target
+
+# Generate dependency reports and checks they are valid.
+docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer
+
+docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports
+
+# Build binary distribution. Note that the below command internally runs 'docs/_bin/generate-license.py' without license check which overwrites LICENSES.BINARY file generated by the above command. This must be fine since both commands are supposed to generate the same contents.
+mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts

distribution/pom.xml

@@ -131,22 +131,7 @@
                                 </configuration>
                             </execution>
                             <execution>
-                                <id>generate-dependency-reports</id>
+                                <id>generate-license</id>
-                                <phase>initialize</phase>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <executable>${project.parent.basedir}/docs/_bin/generate-license-dependency-reports.py</executable>
-                                    <arguments>
-                                        <argument>${project.basedir}/../</argument>
-                                        <argument>${project.basedir}/target</argument>
-                                        <argument>--clean-maven-artifact-transfer</argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>verify-and-generate-license</id>
                                 <phase>initialize</phase>
                                 <goals>
                                     <goal>exec</goal>
@@ -156,7 +141,6 @@
                                     <arguments>
                                         <argument>${project.basedir}/../licenses/APACHE2</argument>
                                         <argument>${project.basedir}/../licenses.yaml</argument>
-                                        <argument>${project.basedir}/target/license-reports</argument>
                                         <argument>${project.basedir}/../LICENSE.BINARY</argument>
                                     </arguments>
                                 </configuration>

docs/_bin/generate-license.py

@@ -20,6 +20,7 @@ import json
 import os
 import sys
 from html.parser import HTMLParser
+import argparse
 
 
 class DependencyReportParser(HTMLParser):
@@ -544,16 +545,24 @@ def generate_license(apache_license_v2, license_yaml):
                 print_outfile("")
 
 
-# TODO: add options: debug mode
+if __name__ == "__main__":
-if len(sys.argv) != 5:
+    try:
-    sys.stderr.write("usage: {} <path to apache license file> <path to license.yaml> <root to maven dependency reports> <path to output file>".format(sys.argv[0]))
+        parser = argparse.ArgumentParser(description='Check and generate license file.')
-    sys.exit(1)
+        parser.add_argument('apache_license', metavar='<path to apache license file>', type=str)
+        parser.add_argument('license_yaml', metavar='<path to license.yaml>', type=str)
+        parser.add_argument('out_path', metavar='<path to output file>', type=str)
+        parser.add_argument('--dependency-reports', dest='dependency_reports_root', type=str, default=None, metavar='<root to maven dependency reports>')
+        args = parser.parse_args()
         
-with open(sys.argv[1]) as apache_license_file:
+        with open(args.apache_license) as apache_license_file:
-    apache_license_v2 = apache_license_file.read()
+            apache_license_v2 = apache_license_file.read()
-license_yaml = sys.argv[2]
+        license_yaml = args.license_yaml
-dependency_reports_root = sys.argv[3]
+        dependency_reports_root = args.dependency_reports_root
 
-with open(sys.argv[4], "w") as outfile:
+        with open(args.out_path, "w") as outfile:
-    check_licenses(license_yaml, dependency_reports_root)
+            if dependency_reports_root is not None:
-    generate_license(apache_license_v2, license_yaml)
+                check_licenses(license_yaml, dependency_reports_root)
+            generate_license(apache_license_v2, license_yaml)
+
+    except KeyboardInterrupt:
+        print('Interrupted, closing.')