From 0a3538b5693fb20a1d631bc111a95718a32271e6 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 19:35:29 -0700 Subject: [PATCH] Fix license check in travis and make it optional (#8049) * Fix license check in travis and make it optional * debug * fix build * too loud maven * move MAVEN_OPTS to top and add comments * adjust script * remove mvn option from python script --- .travis.yml | 4 ++-- build.sh | 27 +++++++++++++++++++++++++++ distribution/pom.xml | 18 +----------------- docs/_bin/generate-license.py | 31 ++++++++++++++++++++----------- 4 files changed, 50 insertions(+), 30 deletions(-) create mode 100755 build.sh diff --git a/.travis.yml b/.travis.yml index 1659b7cb0bd..0b8c577730d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,8 +50,8 @@ matrix: - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y - pip3 install wheel # install wheel first explicitly - pip3 install pyyaml - install: true - script: MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts + install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B + script: ./build.sh # processing module test - env: diff --git a/build.sh b/build.sh new file mode 100755 index 00000000000..bf539b41df8 --- /dev/null +++ b/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MAVEN_OPTS='-Xmx3000m' + +mkdir -p target + +# Generate dependency reports and checks they are valid. +docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer + +docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports + +# Build binary distribution. Note that the below command internally runs 'docs/_bin/generate-license.py' without license check which overwrites LICENSES.BINARY file generated by the above command. This must be fine since both commands are supposed to generate the same contents. +mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts diff --git a/distribution/pom.xml b/distribution/pom.xml index 2933b02dc65..cbfc489b4fd 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -131,22 +131,7 @@ - generate-dependency-reports - initialize - - exec - - - ${project.parent.basedir}/docs/_bin/generate-license-dependency-reports.py - - ${project.basedir}/../ - ${project.basedir}/target - --clean-maven-artifact-transfer - - - - - verify-and-generate-license + generate-license initialize exec @@ -156,7 +141,6 @@ ${project.basedir}/../licenses/APACHE2 ${project.basedir}/../licenses.yaml - ${project.basedir}/target/license-reports ${project.basedir}/../LICENSE.BINARY diff --git a/docs/_bin/generate-license.py b/docs/_bin/generate-license.py index 1ccc72f5df4..6c5a6433032 100755 --- a/docs/_bin/generate-license.py +++ b/docs/_bin/generate-license.py @@ -20,6 +20,7 @@ import json import os import sys from html.parser import HTMLParser +import argparse class DependencyReportParser(HTMLParser): @@ -544,16 +545,24 @@ def generate_license(apache_license_v2, license_yaml): print_outfile("") -# TODO: add options: debug mode -if len(sys.argv) != 5: - sys.stderr.write("usage: {} ".format(sys.argv[0])) - sys.exit(1) +if __name__ == "__main__": + try: + parser = argparse.ArgumentParser(description='Check and generate license file.') + parser.add_argument('apache_license', metavar='', type=str) + parser.add_argument('license_yaml', metavar='', type=str) + parser.add_argument('out_path', metavar='', type=str) + parser.add_argument('--dependency-reports', dest='dependency_reports_root', type=str, default=None, metavar='') + args = parser.parse_args() + + with open(args.apache_license) as apache_license_file: + apache_license_v2 = apache_license_file.read() + license_yaml = args.license_yaml + dependency_reports_root = args.dependency_reports_root -with open(sys.argv[1]) as apache_license_file: - apache_license_v2 = apache_license_file.read() -license_yaml = sys.argv[2] -dependency_reports_root = sys.argv[3] + with open(args.out_path, "w") as outfile: + if dependency_reports_root is not None: + check_licenses(license_yaml, dependency_reports_root) + generate_license(apache_license_v2, license_yaml) -with open(sys.argv[4], "w") as outfile: - check_licenses(license_yaml, dependency_reports_root) - generate_license(apache_license_v2, license_yaml) + except KeyboardInterrupt: + print('Interrupted, closing.')