mirror of
https://github.com/apache/druid.git
synced 2025-02-09 11:34:54 +00:00
ef6811ef88
* Improved Java 17 support and Java runtime docs. 1) Add a "Java runtime" doc page with information about supported Java versions, garbage collection, and strong encapsulation.. 2) Update asm and equalsverifier to versions that support Java 17. 3) Add additional "--add-opens" lines to surefire configuration, so tests can pass successfully under Java 17. 4) Switch openjdk15 tests to openjdk17. 5) Update FrameFile to specifically mention Java runtime incompatibility as the cause of not being able to use Memory.map. 6) Update SegmentLoadDropHandler to log an error for Errors too, not just Exceptions. This is important because an IllegalAccessError is encountered when the correct "--add-opens" line is not provided, which would otherwise be silently ignored. 7) Update example configs to use druid.indexer.runner.javaOptsArray instead of druid.indexer.runner.javaOpts. (The latter is deprecated.) * Adjustments. * Use run-java in more places. * Add run-java. * Update .gitignore. * Exclude hadoop-client-api. Brought in when building on Java 17. * Swap one more usage of java. * Fix the run-java script. * Fix flag. * Include link to Temurin. * Spelling. * Update examples/bin/run-java Co-authored-by: Xavier Léauté <xl+github@xvrl.net> Co-authored-by: Xavier Léauté <xl+github@xvrl.net>
441 lines
20 KiB
Python
Executable File
441 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import yaml
|
|
import os
|
|
import sys
|
|
from html.parser import HTMLParser
|
|
import argparse
|
|
|
|
class DependencyReportParser(HTMLParser):
|
|
# This class parses the given html file to find all dependency reports under "Project dependencies"
|
|
# and "Projection transparent dependencies" sections.
|
|
# The parser works based on the state machine and its state is updated whenever it reads a new tag.
|
|
# The state changes as below:
|
|
#
|
|
# none -> h2_start -> project_dependencies_start -> h3_start -> compile_start -> table_start -> row_start -> th_start / td_start -> th_end / td_end -> row_end -> table_end -> compile_end -> h3_end -> project_dependencies_end -> h2_end -> none
|
|
|
|
attr_index = 0
|
|
group_id = None
|
|
artifact_id = None
|
|
version = None
|
|
classifier = None
|
|
dep_type = None
|
|
license = None
|
|
state = "none"
|
|
dep_to_license = None
|
|
compatible_license_names = None
|
|
include_classifier = False
|
|
druid_module_name = None
|
|
|
|
def __init__(self, druid_module_name, compatible_license_names):
|
|
HTMLParser.__init__(self)
|
|
self.state = "none"
|
|
self.druid_module_name = druid_module_name
|
|
self.compatible_license_names = compatible_license_names
|
|
|
|
def parse(self, f):
|
|
self.dep_to_license = {}
|
|
self.feed(f.read())
|
|
return self.dep_to_license
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
# print("current: {}, start tag: {}, attrs:{} ".format(self.state, tag, attrs))
|
|
if self.state == "none":
|
|
if tag == "h2":
|
|
self.state = "h2_start"
|
|
|
|
if self.state == "h2_start":
|
|
if tag == "a":
|
|
for attr in attrs:
|
|
if attr[0] == "name" and (attr[1] == "Project_Dependencies" or attr[1] == "Project_Transitive_Dependencies"):
|
|
self.state = "project_dependencies_start"
|
|
self.include_classifier = False
|
|
|
|
if self.state == "h2_end":
|
|
if tag == "h3":
|
|
self.state = "h3_start"
|
|
|
|
if self.state == "h3_start":
|
|
if tag == "a":
|
|
for attr in attrs:
|
|
if attr[0] == "name" and attr[1] == "compile":
|
|
self.state = "compile_start"
|
|
|
|
if self.state == "h3_end":
|
|
if tag == "table":
|
|
self.state = "table_start"
|
|
|
|
if self.state == "table_start":
|
|
if tag == "tr":
|
|
self.state = "row_start"
|
|
self.clear_attr()
|
|
|
|
if self.state == "row_end":
|
|
if tag == "tr":
|
|
self.state = "row_start"
|
|
self.clear_attr()
|
|
|
|
if self.state == "row_start":
|
|
if tag == "td":
|
|
self.state = "td_start"
|
|
elif tag == "th":
|
|
self.state = "th_start"
|
|
|
|
if self.state == "th_end":
|
|
if tag == "th":
|
|
self.state = "th_start"
|
|
|
|
if self.state == "td_end":
|
|
if tag == "td":
|
|
self.state = "td_start"
|
|
|
|
def handle_endtag(self, tag):
|
|
# print("current: {}, end tag: {}".format(self.state, tag))
|
|
if self.state == "project_dependencies_start":
|
|
if tag == "a":
|
|
self.state = "project_dependencies_end"
|
|
|
|
if self.state == "h2_start":
|
|
if tag == "h2":
|
|
self.state = "h2_end"
|
|
|
|
if self.state == "project_dependencies_end":
|
|
if tag == "h2":
|
|
self.state = "h2_end"
|
|
|
|
if self.state == "compile_start":
|
|
if tag == "a":
|
|
self.state = "compile_end"
|
|
|
|
if self.state == "compile_end":
|
|
if tag == "h3":
|
|
self.state = "h3_end"
|
|
|
|
if self.state == "table_start":
|
|
if tag == "table":
|
|
self.state = "none"
|
|
|
|
if self.state == "td_start":
|
|
if tag == "td":
|
|
self.state = "td_end"
|
|
self.attr_index = self.attr_index + 1
|
|
|
|
if self.state == "th_start":
|
|
if tag == "th":
|
|
self.state = "th_end"
|
|
|
|
if self.state == "row_start":
|
|
if tag == "tr":
|
|
self.state = "row_end"
|
|
|
|
if self.state == "th_end":
|
|
if tag == "tr":
|
|
self.state = "row_end"
|
|
|
|
if self.state == "td_end":
|
|
if tag == "tr":
|
|
self.state = "row_end"
|
|
# print(json.dumps({"groupId": self.group_id, "artifactId": self.artifact_id, "version": self.version, "classifier": self.classifier, "type": self.dep_type, "license": self.license}))
|
|
if self.group_id.find("org.apache.druid") < 0:
|
|
self.dep_to_license[get_dep_key(self.group_id, self.artifact_id, self.version)] = (self.license, self.druid_module_name)
|
|
|
|
if self.state == "row_end":
|
|
if tag == "table":
|
|
self.state = "none"
|
|
|
|
def handle_data(self, data):
|
|
if self.state == "td_start":
|
|
self.set_attr(data)
|
|
elif self.state == "th_start":
|
|
if data.lower() == "classifier":
|
|
self.include_classifier = True
|
|
|
|
def clear_attr(self):
|
|
self.group_id = None
|
|
self.artifact_id = None
|
|
self.version = None
|
|
self.classifier = None
|
|
self.dep_type = None
|
|
self.license = None
|
|
self.attr_index = 0
|
|
|
|
def set_attr(self, data):
|
|
#print("set data: {}".format(data))
|
|
if self.attr_index == 0:
|
|
self.group_id = data
|
|
elif self.attr_index == 1:
|
|
self.artifact_id = data
|
|
elif self.attr_index == 2:
|
|
self.version = get_version_string(data)
|
|
elif self.attr_index == 3:
|
|
if self.include_classifier:
|
|
self.classifier = data
|
|
else:
|
|
self.dep_type = data
|
|
elif self.attr_index == 4:
|
|
if self.include_classifier:
|
|
self.dep_type = data
|
|
else:
|
|
self.set_license(data)
|
|
elif self.attr_index == 5:
|
|
if self.include_classifier:
|
|
self.set_license(data)
|
|
else:
|
|
raise Exception("Unknown attr_index [{}]".format(self.attr_index))
|
|
else:
|
|
raise Exception("Unknown attr_index [{}]".format(self.attr_index))
|
|
|
|
def set_license(self, data):
|
|
if data.upper().find("GPL") < 0:
|
|
if self.license != 'Apache License version 2.0':
|
|
self.license = self.compatible_license_names[data]
|
|
|
|
|
|
def print_log_to_stderr(string):
|
|
print(string, file=sys.stderr)
|
|
|
|
def build_compatible_license_names():
|
|
compatible_licenses = {}
|
|
compatible_licenses['Apache License, Version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['The Apache Software License, Version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache-2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache 2'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License 2'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache Software License - Version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['The Apache License, Version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License Version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License Version 2'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License v2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License, 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License, version 2.0'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache 2.0 License'] = 'Apache License version 2.0'
|
|
compatible_licenses['Apache License, 2.0'] = 'Apache License version 2.0'
|
|
|
|
compatible_licenses['Public Domain'] = 'Public Domain'
|
|
|
|
compatible_licenses['BSD-2-Clause License'] = 'BSD-2-Clause License'
|
|
compatible_licenses['BSD-2-Clause'] = 'BSD-2-Clause License'
|
|
compatible_licenses['BSD 2-Clause license'] = 'BSD-2-Clause License'
|
|
compatible_licenses['BSD 2-Clause License'] = 'BSD-2-Clause License'
|
|
|
|
compatible_licenses['BSD-3-Clause License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['New BSD license'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD'] = 'BSD-3-Clause License'
|
|
compatible_licenses['The BSD License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD licence'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD-like'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD 3-clause'] = 'BSD-3-Clause License'
|
|
compatible_licenses['The BSD 3-Clause License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['Revised BSD'] = 'BSD-3-Clause License'
|
|
compatible_licenses['New BSD License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['3-Clause BSD License'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD 3-Clause'] = 'BSD-3-Clause License'
|
|
compatible_licenses['BSD-3-Clause'] = 'BSD-3-Clause License'
|
|
|
|
compatible_licenses['ICU License'] = 'ICU License'
|
|
|
|
compatible_licenses['SIL Open Font License 1.1'] = 'SIL Open Font License 1.1'
|
|
|
|
compatible_licenses['CDDL 1.1'] = 'CDDL 1.1'
|
|
compatible_licenses['CDDL/GPLv2+CE'] = 'CDDL 1.1'
|
|
compatible_licenses['CDDL + GPLv2 with classpath exception'] = 'CDDL 1.1'
|
|
compatible_licenses['CDDL License'] = 'CDDL 1.1'
|
|
compatible_licenses['COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0'] = 'CDDL 1.0'
|
|
|
|
compatible_licenses['Eclipse Public License 1.0'] = 'Eclipse Public License 1.0'
|
|
compatible_licenses['The Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0'
|
|
compatible_licenses['Eclipse Public License - Version 1.0'] = 'Eclipse Public License 1.0'
|
|
compatible_licenses['Eclipse Public License, Version 1.0'] = 'Eclipse Public License 1.0'
|
|
compatible_licenses['Eclipse Public License v1.0'] = 'Eclipse Public License 1.0'
|
|
compatible_licenses['EPL 1.0'] = 'Eclipse Public License 1.0'
|
|
|
|
compatible_licenses['Eclipse Public License 2.0'] = 'Eclipse Public License 2.0'
|
|
compatible_licenses['The Eclipse Public License, Version 2.0'] = 'Eclipse Public License 2.0'
|
|
compatible_licenses['Eclipse Public License - Version 2.0'] = 'Eclipse Public License 2.0'
|
|
compatible_licenses['Eclipse Public License, Version 2.0'] = 'Eclipse Public License 2.0'
|
|
compatible_licenses['Eclipse Public License v2.0'] = 'Eclipse Public License 2.0'
|
|
compatible_licenses['EPL 2.0'] = 'Eclipse Public License 2.0'
|
|
|
|
compatible_licenses['Eclipse Distribution License 1.0'] = 'Eclipse Distribution License 1.0'
|
|
compatible_licenses['Eclipse Distribution License - v 1.0'] = 'Eclipse Distribution License 1.0'
|
|
compatible_licenses['Eclipse Distribution License v. 1.0'] = 'Eclipse Distribution License 1.0'
|
|
compatible_licenses['EDL 1.0'] = 'Eclipse Distribution License 1.0'
|
|
|
|
compatible_licenses['Mozilla Public License Version 2.0'] = 'Mozilla Public License Version 2.0'
|
|
compatible_licenses['Mozilla Public License, Version 2.0'] = 'Mozilla Public License Version 2.0'
|
|
|
|
compatible_licenses['Creative Commons Attribution 2.5'] = 'Creative Commons Attribution 2.5'
|
|
|
|
compatible_licenses['Creative Commons CC0'] = 'Creative Commons CC0'
|
|
compatible_licenses['CC0'] = 'Creative Commons CC0'
|
|
|
|
compatible_licenses['The MIT License'] = 'MIT License'
|
|
compatible_licenses['MIT License'] = 'MIT License'
|
|
compatible_licenses['The MIT License (MIT)'] = 'MIT License'
|
|
compatible_licenses['Bouncy Castle Licence'] = 'MIT License'
|
|
|
|
compatible_licenses['-'] = '-'
|
|
return compatible_licenses
|
|
|
|
def get_dep_key(group_id, artifact_id, version):
|
|
return (group_id, artifact_id, version)
|
|
|
|
def get_version_string(version):
|
|
if type(version) == str:
|
|
return version
|
|
else:
|
|
return str(version)
|
|
|
|
def find_druid_module_name(dirpath):
|
|
ext_start = dirpath.find("/ext/")
|
|
if ext_start > 0:
|
|
# Found an extension
|
|
subpath = dirpath[(len("/ext/") + ext_start):]
|
|
ext_name_end = subpath.find("/")
|
|
if ext_name_end < 0:
|
|
raise Exception("Can't determine extension name from [{}]".format(dirpath))
|
|
else:
|
|
return subpath[0:ext_name_end]
|
|
else:
|
|
# Druid core
|
|
return "core"
|
|
|
|
def check_licenses(license_yaml, dependency_reports_root):
|
|
# Build a dictionary to facilitate comparing reported licenses and registered ones.
|
|
# These dictionaries are the mapping of (group_id, artifact_id, version) to license_name.
|
|
|
|
# Build reported license dictionary.
|
|
reported_dep_to_licenses = {}
|
|
compatible_license_names = build_compatible_license_names()
|
|
for dirpath, dirnames, filenames in os.walk(dependency_reports_root):
|
|
for filename in filenames:
|
|
if filename == "dependencies.html":
|
|
full_path = os.path.join(dirpath, filename)
|
|
# Determine if it's druid core or an extension
|
|
druid_module_name = find_druid_module_name(dirpath)
|
|
print_log_to_stderr("Parsing {}".format(full_path))
|
|
with open(full_path, encoding="utf-8") as report_file:
|
|
parser = DependencyReportParser(druid_module_name, compatible_license_names)
|
|
reported_dep_to_licenses.update(parser.parse(report_file))
|
|
|
|
if len(reported_dep_to_licenses) == 0:
|
|
raise Exception("No dependency reports are found")
|
|
|
|
print_log_to_stderr("Found {} reported licenses\n".format(len(reported_dep_to_licenses)))
|
|
|
|
# Build registered license dictionary.
|
|
registered_dep_to_licenses = {}
|
|
skipping_licenses = {}
|
|
with open(license_yaml, encoding='utf-8') as registry_file:
|
|
licenses_list = list(yaml.load_all(registry_file, Loader=yaml.FullLoader))
|
|
for license in licenses_list:
|
|
if 'libraries' in license:
|
|
for library in license['libraries']:
|
|
if type(library) is not dict:
|
|
raise Exception("Expected dict but got {}[{}]".format(type(library), library))
|
|
if len(library) > 1:
|
|
raise Exception("Expected 1 groupId and artifactId, but got [{}]".format(library))
|
|
for group_id, artifact_id in library.items():
|
|
if 'version' not in license:
|
|
raise Exception("version is missing in {}".format(license))
|
|
if 'license_name' not in license:
|
|
raise Exception("name is missing in {}".format(license))
|
|
if 'skip_dependency_report_check' in license and license['skip_dependency_report_check']:
|
|
if 'version' not in license:
|
|
version = "-"
|
|
else:
|
|
version = get_version_string(license['version'])
|
|
skipping_licenses[get_dep_key(group_id, artifact_id, version)] = license
|
|
else:
|
|
registered_dep_to_licenses[get_dep_key(group_id, artifact_id, get_version_string(license['version']))] = compatible_license_names[license['license_name']]
|
|
|
|
if len(registered_dep_to_licenses) == 0:
|
|
raise Exception("No registered licenses are found")
|
|
|
|
# Compare licenses in registry and those in dependency reports.
|
|
mismatched_licenses = []
|
|
missing_licenses = []
|
|
unchecked_licenses = []
|
|
# Iterate through registered licenses and check if its license is same with the reported one.
|
|
for key, registered_license in registered_dep_to_licenses.items():
|
|
if key in reported_dep_to_licenses: # key is (group_id, artifact_id, version)
|
|
reported_license_druid_module = reported_dep_to_licenses[key]
|
|
reported_license = reported_license_druid_module[0]
|
|
druid_module = reported_license_druid_module[1]
|
|
if reported_license is not None and reported_license != "-" and reported_license != registered_license:
|
|
group_id = key[0]
|
|
artifact_id = key[1]
|
|
version = key[2]
|
|
mismatched_licenses.append((druid_module, group_id, artifact_id, version, reported_license, registered_license))
|
|
|
|
# If we find any mismatched license, stop immediately.
|
|
if len(mismatched_licenses) > 0:
|
|
print_log_to_stderr("Error: found {} mismatches between reported licenses and registered licenses".format(len(mismatched_licenses)))
|
|
for mismatched_license in mismatched_licenses:
|
|
print_log_to_stderr("druid_module: {}, groupId: {}, artifactId: {}, version: {}, reported_license: {}, registered_license: {}".format(mismatched_license[0], mismatched_license[1], mismatched_license[2], mismatched_license[3], mismatched_license[4], mismatched_license[5]))
|
|
print_log_to_stderr("")
|
|
|
|
# Let's find missing licenses, which are reported but missing in the registry.
|
|
for key, reported_license_druid_module in reported_dep_to_licenses.items():
|
|
if reported_license_druid_module[0] != "-" and key not in registered_dep_to_licenses and key not in skipping_licenses:
|
|
missing_licenses.append((reported_license_druid_module[1], key[0], key[1], key[2], reported_license_druid_module[0]))
|
|
|
|
if len(missing_licenses) > 0:
|
|
print_log_to_stderr("Error: found {} missing licenses. These licenses are reported, but missing in the registry".format(len(missing_licenses)))
|
|
for missing_license in missing_licenses:
|
|
print_log_to_stderr("druid_module: {}, groupId: {}, artifactId: {}, version: {}, license: {}".format(missing_license[0], missing_license[1], missing_license[2], missing_license[3], missing_license[4]))
|
|
print_log_to_stderr("")
|
|
|
|
# Let's find unchecked licenses, which are registered but missing in the report.
|
|
# These licenses should be checked manually.
|
|
for key, registered_license in registered_dep_to_licenses.items():
|
|
if key not in reported_dep_to_licenses:
|
|
unchecked_licenses.append((key[0], key[1], key[2], registered_license))
|
|
elif reported_dep_to_licenses[key][0] == "-":
|
|
unchecked_licenses.append((key[0], key[1], key[2], registered_license))
|
|
|
|
if len(unchecked_licenses) > 0:
|
|
print_log_to_stderr("Warn: found {} unchecked licenses. These licenses are registered, but not found in dependency reports.".format(len(unchecked_licenses)))
|
|
print_log_to_stderr("These licenses must be checked manually.")
|
|
for unchecked_license in unchecked_licenses:
|
|
print_log_to_stderr("groupId: {}, artifactId: {}, version: {}, reported_license: {}".format(unchecked_license[0], unchecked_license[1], unchecked_license[2], unchecked_license[3]))
|
|
print_log_to_stderr("")
|
|
|
|
if len(mismatched_licenses) > 0 or len(missing_licenses) > 0:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
parser = argparse.ArgumentParser(description='Check and generate license file.')
|
|
parser.add_argument('license_yaml', metavar='<path to license.yaml>', type=str)
|
|
parser.add_argument('dependency_reports_root', metavar='<root to maven dependency reports>', type=str)
|
|
args = parser.parse_args()
|
|
|
|
license_yaml = args.license_yaml
|
|
dependency_reports_root = args.dependency_reports_root
|
|
|
|
check_licenses(license_yaml, dependency_reports_root)
|
|
|
|
except KeyboardInterrupt:
|
|
print('Interrupted, closing.')
|