druid/distribution/bin/check-licenses.py

#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import yaml
import os
import sys
from html.parser import HTMLParser
import argparse

class DependencyReportParser(HTMLParser):
    # This class parses the given html file to find all dependency reports under "Project dependencies"
    # and "Projection transparent dependencies" sections.
    # The parser works based on the state machine and its state is updated whenever it reads a new tag.
    # The state changes as below:
    #
    # none -> h2_start -> project_dependencies_start -> h3_start -> compile_start -> table_start -> row_start -> th_start / td_start -> th_end / td_end -> row_end -> table_end -> compile_end -> h3_end -> project_dependencies_end -> h2_end -> none

    attr_index = 0
    group_id = None
    artifact_id = None
    version = None
    classifier = None
    dep_type = None
    license = None
    state = "none"
    dep_to_license = None
    compatible_license_names = None
    include_classifier = False
    druid_module_name = None

    def __init__(self, druid_module_name, compatible_license_names):
        HTMLParser.__init__(self)
        self.state = "none"
        self.druid_module_name = druid_module_name
        self.compatible_license_names = compatible_license_names

    def parse(self, f):
        self.dep_to_license = {}
        self.feed(f.read())
        return self.dep_to_license

    def handle_starttag(self, tag, attrs):
        # print("current: {}, start tag: {}, attrs:{} ".format(self.state, tag, attrs))
        if self.state == "none":
            if tag == "h2":
                self.state = "h2_start"

        if self.state == "h2_start":
            if tag == "a":
                for attr in attrs:
                    if attr[0] == "name" and (attr[1] == "Project_Dependencies" or attr[1] == "Project_Transitive_Dependencies"):
                        self.state = "project_dependencies_start"
                        self.include_classifier = False

        if self.state == "h2_end":
            if tag == "h3":
                self.state = "h3_start"

        if self.state == "h3_start":
            if tag == "a":
                for attr in attrs:
                    if attr[0] == "name" and attr[1] == "compile":
                        self.state = "compile_start"

        if self.state == "h3_end":
            if tag == "table":
                self.state = "table_start"

        if self.state == "table_start":
            if tag == "tr":
                self.state = "row_start"
                self.clear_attr()

        if self.state == "row_end":
            if tag == "tr":
                self.state = "row_start"
                self.clear_attr()

        if self.state == "row_start":
            if tag == "td":
                self.state = "td_start"
            elif tag == "th":
                self.state = "th_start"

        if self.state == "th_end":
            if tag == "th":
                self.state = "th_start"

        if self.state == "td_end":
            if tag == "td":
                self.state = "td_start"

    def handle_endtag(self, tag):
        # print("current: {}, end tag: {}".format(self.state, tag))
        if self.state == "project_dependencies_start":
            if tag == "a":
                self.state = "project_dependencies_end"

        if self.state == "h2_start":
            if tag == "h2":
                self.state = "h2_end"

        if self.state == "project_dependencies_end":
            if tag == "h2":
                self.state = "h2_end"

        if self.state == "compile_start":
            if tag == "a":
                self.state = "compile_end"

        if self.state == "compile_end":
            if tag == "h3":
                self.state = "h3_end"

        if self.state == "table_start":
            if tag == "table":
                self.state = "none"

        if self.state == "td_start":
            if tag == "td":
                self.state = "td_end"
                self.attr_index = self.attr_index + 1

        if self.state == "th_start":
            if tag == "th":
                self.state = "th_end"

        if self.state == "row_start":
            if tag == "tr":
                self.state = "row_end"

        if self.state == "th_end":
            if tag == "tr":
                self.state = "row_end"

        if self.state == "td_end":
            if tag == "tr":
                self.state = "row_end"
                # print(json.dumps({"groupId": self.group_id, "artifactId": self.artifact_id, "version": self.version, "classifier": self.classifier, "type": self.dep_type, "license": self.license}))
                if self.group_id.find("org.apache.druid") < 0:
                    self.dep_to_license[get_dep_key(self.group_id, self.artifact_id, self.version)] = (self.license, self.druid_module_name)

        if self.state == "row_end":
            if tag == "table":
                self.state = "none"

    def handle_data(self, data):
        if self.state == "td_start":
            self.set_attr(data)
        elif self.state == "th_start":
            if data.lower() == "classifier":
                self.include_classifier = True

    def clear_attr(self):
        self.group_id = None
        self.artifact_id = None
        self.version = None
        self.classifier = None
        self.dep_type = None
        self.license = None
        self.attr_index = 0

    def set_attr(self, data):
        #print("set data: {}".format(data))
        if self.attr_index == 0:
            self.group_id = data
        elif self.attr_index == 1:
            self.artifact_id = data
        elif self.attr_index == 2:
            self.version = get_version_string(data)
        elif self.attr_index == 3:
            if self.include_classifier:
                self.classifier = data
            else:
                self.dep_type = data
        elif self.attr_index == 4:
            if self.include_classifier:
                self.dep_type = data
            else:
                self.set_license(data)
        elif self.attr_index == 5:
            if self.include_classifier:
                self.set_license(data)
            else:
                raise Exception("Unknown attr_index [{}]".format(self.attr_index))
        else:
            raise Exception("Unknown attr_index [{}]".format(self.attr_index))

    def set_license(self, data):
        if data.upper().find("GPL") < 0:
            if self.license != 'Apache License version 2.0':
                self.license = self.compatible_license_names[data]


def print_log_to_stderr(string):
    print(string, file=sys.stderr)

def build_compatible_license_names():
    compatible_licenses = {}
    compatible_licenses['Apache License, Version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['The Apache Software License, Version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache-2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache 2'] = 'Apache License version 2.0'
    compatible_licenses['Apache License 2'] = 'Apache License version 2.0'
    compatible_licenses['Apache License 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache Software License - Version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['The Apache License, Version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache License version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache License Version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache License Version 2'] = 'Apache License version 2.0'
    compatible_licenses['Apache License v2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache License, 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache License, version 2.0'] = 'Apache License version 2.0'
    compatible_licenses['Apache 2.0 License'] = 'Apache License version 2.0'
    compatible_licenses['Apache License, 2.0'] = 'Apache License version 2.0'

    compatible_licenses['Public Domain'] = 'Public Domain'

    compatible_licenses['BSD-2-Clause License'] = 'BSD-2-Clause License'
    compatible_licenses['BSD-2-Clause'] = 'BSD-2-Clause License'

    compatible_licenses['BSD-3-Clause License'] = 'BSD-3-Clause License'
    compatible_licenses['New BSD license'] = 'BSD-3-Clause License'
    compatible_licenses['BSD'] = 'BSD-3-Clause License'
    compatible_licenses['The BSD License'] = 'BSD-3-Clause License'
    compatible_licenses['BSD licence'] = 'BSD-3-Clause License'
    compatible_licenses['BSD License'] = 'BSD-3-Clause License'
    compatible_licenses['BSD-like'] = 'BSD-3-Clause License'
    compatible_licenses['BSD 3-clause'] = 'BSD-3-Clause License'
    compatible_licenses['The BSD 3-Clause License'] = 'BSD-3-Clause License'
    compatible_licenses['Revised BSD'] = 'BSD-3-Clause License'
    compatible_licenses['New BSD License'] = 'BSD-3-Clause License'
    compatible_licenses['3-Clause BSD License'] = 'BSD-3-Clause License'
    compatible_licenses['BSD 3-Clause'] = 'BSD-3-Clause License'

    compatible_licenses['ICU License'] = 'ICU License'

    compatible_licenses['SIL Open Font License 1.1'] = 'SIL Open Font License 1.1'

    compatible_licenses['CDDL 1.1'] = 'CDDL 1.1'
    compatible_licenses['CDDL/GPLv2+CE'] = 'CDDL 1.1'
    compatible_licenses['CDDL + GPLv2 with classpath exception'] = 'CDDL 1.1'
    compatible_licenses['CDDL License'] = 'CDDL 1.1'
    compatible_licenses['COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0'] = 'CDDL 1.0'

    compatible_licenses['Eclipse Public License 1.0'] = 'Eclipse Public License 1.0'
    compatible_licenses['The Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0'
    compatible_licenses['Eclipse Public License - Version 1.0'] = 'Eclipse Public License 1.0'
    compatible_licenses['Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0'
    compatible_licenses['Eclipse Public License v1.0'] = 'Eclipse Public License 1.0'
    compatible_licenses['EPL 1.0'] = 'Eclipse Public License 1.0'

    compatible_licenses['Eclipse Public License 2.0'] = 'Eclipse Public License 2.0'
    compatible_licenses['The Eclipse Public License, Version 2.0'] = 'Eclipse Public License 2.0'
    compatible_licenses['Eclipse Public License - Version 2.0'] = 'Eclipse Public License 2.0'
    compatible_licenses['Eclipse Public License, Version 2.0'] = 'Eclipse Public License 2.0'
    compatible_licenses['Eclipse Public License v2.0'] = 'Eclipse Public License 2.0'
    compatible_licenses['EPL 2.0'] = 'Eclipse Public License 2.0'

    compatible_licenses['Eclipse Distribution License 1.0'] = 'Eclipse Distribution License 1.0'
    compatible_licenses['Eclipse Distribution License - v 1.0'] = 'Eclipse Distribution License 1.0'
    compatible_licenses['Eclipse Distribution License v. 1.0'] = 'Eclipse Distribution License 1.0'
    compatible_licenses['EDL 1.0'] = 'Eclipse Distribution License 1.0'

    compatible_licenses['Mozilla Public License Version 2.0'] = 'Mozilla Public License Version 2.0'
    compatible_licenses['Mozilla Public License, Version 2.0'] = 'Mozilla Public License Version 2.0'

    compatible_licenses['Creative Commons Attribution 2.5'] = 'Creative Commons Attribution 2.5'

    compatible_licenses['Creative Commons CC0'] = 'Creative Commons CC0'
    compatible_licenses['CC0'] = 'Creative Commons CC0'

    compatible_licenses['The MIT License'] = 'MIT License'
    compatible_licenses['MIT License'] = 'MIT License'
    compatible_licenses['The MIT License (MIT)'] = 'MIT License'
    compatible_licenses['Bouncy Castle Licence'] = 'MIT License'

    compatible_licenses['-'] = '-'
    return compatible_licenses

def get_dep_key(group_id, artifact_id, version):
    return (group_id, artifact_id, version)

def get_version_string(version):
    if type(version) == str:
        return version
    else:
        return str(version)

def find_druid_module_name(dirpath):
    ext_start = dirpath.find("/ext/")
    if ext_start > 0:
        # Found an extension
        subpath = dirpath[(len("/ext/") + ext_start):]
        ext_name_end = subpath.find("/")
        if ext_name_end < 0:
            raise Exception("Can't determine extension name from [{}]".format(dirpath))
        else:
            return subpath[0:ext_name_end]
    else:
        # Druid core
        return "core"

def check_licenses(license_yaml, dependency_reports_root):
    # Build a dictionary to facilitate comparing reported licenses and registered ones.
    # These dictionaries are the mapping of (group_id, artifact_id, version) to license_name.

    # Build reported license dictionary.
    reported_dep_to_licenses = {}
    compatible_license_names = build_compatible_license_names()
    for dirpath, dirnames, filenames in os.walk(dependency_reports_root):
        for filename in filenames:
            if filename == "dependencies.html":
                full_path = os.path.join(dirpath, filename)
                # Determine if it's druid core or an extension
                druid_module_name = find_druid_module_name(dirpath)
                print_log_to_stderr("Parsing {}".format(full_path))
                with open(full_path, encoding="utf-8") as report_file:
                    parser = DependencyReportParser(druid_module_name, compatible_license_names)
                    reported_dep_to_licenses.update(parser.parse(report_file))

    if len(reported_dep_to_licenses) == 0:
        raise Exception("No dependency reports are found")

    print_log_to_stderr("Found {} reported licenses\n".format(len(reported_dep_to_licenses)))

    # Build registered license dictionary.
    registered_dep_to_licenses = {}
    skipping_licenses = {}
    with open(license_yaml, encoding='utf-8') as registry_file:
        licenses_list = list(yaml.load_all(registry_file, Loader=yaml.FullLoader))
    for license in licenses_list:
        if 'libraries' in license:
            for library in license['libraries']:
                if type(library) is not dict:
                    raise Exception("Expected dict but got {}[{}]".format(type(library), library))
                if len(library) > 1:
                    raise Exception("Expected 1 groupId and artifactId, but got [{}]".format(library))
                for group_id, artifact_id in library.items():
                    if 'version' not in license:
                        raise Exception("version is missing in {}".format(license))
                    if 'license_name' not in license:
                        raise Exception("name is missing in {}".format(license))
                    if 'skip_dependency_report_check' in license and license['skip_dependency_report_check']:
                        if 'version' not in license:
                            version = "-"
                        else:
                            version = get_version_string(license['version'])
                        skipping_licenses[get_dep_key(group_id, artifact_id, version)] = license
                    else:
                        registered_dep_to_licenses[get_dep_key(group_id, artifact_id, get_version_string(license['version']))] = compatible_license_names[license['license_name']]

    if len(registered_dep_to_licenses) == 0:
        raise Exception("No registered licenses are found")

    # Compare licenses in registry and those in dependency reports.
    mismatched_licenses = []
    missing_licenses = []
    unchecked_licenses = []
    # Iterate through registered licenses and check if its license is same with the reported one.
    for key, registered_license in registered_dep_to_licenses.items():
        if key in reported_dep_to_licenses: # key is (group_id, artifact_id, version)
            reported_license_druid_module = reported_dep_to_licenses[key]
            reported_license = reported_license_druid_module[0]
            druid_module = reported_license_druid_module[1]
            if reported_license is not None and reported_license != "-" and reported_license != registered_license:
                group_id = key[0]
                artifact_id = key[1]
                version = key[2]
                mismatched_licenses.append((druid_module, group_id, artifact_id, version, reported_license, registered_license))

    # If we find any mismatched license, stop immediately.
    if len(mismatched_licenses) > 0:
        print_log_to_stderr("Error: found {} mismatches between reported licenses and registered licenses".format(len(mismatched_licenses)))
        for mismatched_license in mismatched_licenses:
            print_log_to_stderr("druid_module: {}, groupId: {}, artifactId: {}, version: {}, reported_license: {}, registered_license: {}".format(mismatched_license[0], mismatched_license[1], mismatched_license[2], mismatched_license[3], mismatched_license[4], mismatched_license[5]))
        print_log_to_stderr("")

    # Let's find missing licenses, which are reported but missing in the registry.
    for key, reported_license_druid_module in reported_dep_to_licenses.items():
        if reported_license_druid_module[0] != "-" and key not in registered_dep_to_licenses and key not in skipping_licenses:
            missing_licenses.append((reported_license_druid_module[1], key[0], key[1], key[2], reported_license_druid_module[0]))

    if len(missing_licenses) > 0:
        print_log_to_stderr("Error: found {} missing licenses. These licenses are reported, but missing in the registry".format(len(missing_licenses)))
        for missing_license in missing_licenses:
            print_log_to_stderr("druid_module: {}, groupId: {}, artifactId: {}, version: {}, license: {}".format(missing_license[0], missing_license[1], missing_license[2], missing_license[3], missing_license[4]))
        print_log_to_stderr("")

    # Let's find unchecked licenses, which are registered but missing in the report.
    # These licenses should be checked manually.
    for key, registered_license in registered_dep_to_licenses.items():
        if key not in reported_dep_to_licenses:
            unchecked_licenses.append((key[0], key[1], key[2], registered_license))
        elif reported_dep_to_licenses[key][0] == "-":
            unchecked_licenses.append((key[0], key[1], key[2], registered_license))

    if len(unchecked_licenses) > 0:
        print_log_to_stderr("Warn: found {} unchecked licenses. These licenses are registered, but not found in dependency reports.".format(len(unchecked_licenses)))
        print_log_to_stderr("These licenses must be checked manually.")
        for unchecked_license in unchecked_licenses:
            print_log_to_stderr("groupId: {}, artifactId: {}, version: {}, reported_license: {}".format(unchecked_license[0], unchecked_license[1], unchecked_license[2], unchecked_license[3]))
    print_log_to_stderr("")

    if len(mismatched_licenses) > 0 or len(missing_licenses) > 0:
        sys.exit(1)


if __name__ == "__main__":
    try:
        parser = argparse.ArgumentParser(description='Check and generate license file.')
        parser.add_argument('license_yaml', metavar='<path to license.yaml>', type=str)
        parser.add_argument('dependency_reports_root', metavar='<root to maven dependency reports>', type=str)
        args = parser.parse_args()

        license_yaml = args.license_yaml
        dependency_reports_root = args.dependency_reports_root

        check_licenses(license_yaml, dependency_reports_root)

    except KeyboardInterrupt:
        print('Interrupted, closing.')