From 4125c7682e9fdd41113d827e607356aee83a5bfe Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Fri, 30 May 2014 10:33:55 +0000 Subject: [PATCH] LUCENE-5442: ant check-lib-versions will fail the build if there are unexpected version conflicts between direct and transitive dependencies. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1598538 13f79535-47bb-0310-9956-ffa450edef68 --- dev-tools/idea/.idea/libraries/Ivy.xml | 9 + dev-tools/idea/lucene/tools/tools.iml | 1 + lucene/CHANGES.txt | 9 + lucene/build.xml | 6 +- lucene/ivy-ignore-conflicts.properties | 33 ++ lucene/ivy-versions.properties | 2 + lucene/licenses/ivy-2.3.0.jar.sha1 | 1 + lucene/licenses/ivy-LICENSE-ASL.txt | 258 +++++++++ lucene/licenses/ivy-NOTICE.txt | 16 + lucene/tools/custom-tasks.xml | 8 +- lucene/tools/ivy.xml | 1 + .../GetMavenDependenciesTask.java | 62 +- .../dependencies/InterpolatedProperties.java | 72 +++ .../validation/LibVersionsCheckTask.java | 546 +++++++++++++++--- .../validation/ivyde/IvyNodeElement.java | 180 ++++++ .../ivyde/IvyNodeElementAdapter.java | 137 +++++ 16 files changed, 1228 insertions(+), 113 deletions(-) create mode 100644 dev-tools/idea/.idea/libraries/Ivy.xml create mode 100644 lucene/ivy-ignore-conflicts.properties create mode 100644 lucene/licenses/ivy-2.3.0.jar.sha1 create mode 100644 lucene/licenses/ivy-LICENSE-ASL.txt create mode 100644 lucene/licenses/ivy-NOTICE.txt create mode 100644 lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java create mode 100644 lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElement.java create mode 100644 lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElementAdapter.java diff --git a/dev-tools/idea/.idea/libraries/Ivy.xml b/dev-tools/idea/.idea/libraries/Ivy.xml new file mode 100644 index 00000000000..798dc0f6375 --- /dev/null +++ b/dev-tools/idea/.idea/libraries/Ivy.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/dev-tools/idea/lucene/tools/tools.iml b/dev-tools/idea/lucene/tools/tools.iml index a1aee1f3182..27734f3bbc6 100644 --- a/dev-tools/idea/lucene/tools/tools.iml +++ b/dev-tools/idea/lucene/tools/tools.iml @@ -10,6 +10,7 @@ + diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c45922dc608..e18d3c923a2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -249,6 +249,15 @@ Test Framework * LUCENE-5619: Added backwards compatibility tests to ensure we can update existing indexes with doc-values updates. (Shai Erera, Robert Muir) + +Build + +* LUCENE-5442: The Ant check-lib-versions target now runs Ivy resolution + transitively, then fails the build when it finds a version conflict: when a + transitive dependency's version is more recent than the direct dependency's + version specified in lucene/ivy-versions.properties. Exceptions are + specifiable in lucene/ivy-ignore-conflicts.properties. + (Steve Rowe) ======================= Lucene 4.8.1 ======================= diff --git a/lucene/build.xml b/lucene/build.xml index 3237141e545..43acb751197 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -171,7 +171,11 @@ - + diff --git a/lucene/ivy-ignore-conflicts.properties b/lucene/ivy-ignore-conflicts.properties new file mode 100644 index 00000000000..0e3eacc6924 --- /dev/null +++ b/lucene/ivy-ignore-conflicts.properties @@ -0,0 +1,33 @@ +# The /org/name keys in this file must be kept lexically sorted. +# Blank lines, comment lines, and keys that aren't in /org/name format are ignored +# when the lexical sort check is performed by the ant check-lib-versions target. +# +# The format is: +# +# /org/name = [, [ ... ] ] +# +# where each is an indirect dependency version to ignore (i.e., not +# trigger a conflict) when the ant check-lib-versions target is run. + +/asm/asm = 3.2 +/com.google.guava/guava = 16.0.1 + +com.sun.jersey.ignore.versions = 1.9 +/com.sun.jersey.contribs/jersey-guice = ${com.sun.jersey.ignore.versions} +/com.sun.jersey/jersey-core = ${com.sun.jersey.ignore.versions} +/com.sun.jersey/jersey-json = ${com.sun.jersey.ignore.versions} +/com.sun.jersey/jersey-server = ${com.sun.jersey.ignore.versions} + +/com.sun.xml.bind/jaxb-impl = 2.2.3-1 +/commons-beanutils/commons-beanutils = 1.8.0, 1.8.3 +/commons-digester/commons-digester = 2.1 +/commons-io/commons-io = 2.3 +/commons-logging/commons-logging = 1.1.3 +/io.netty/netty = 3.7.0.Final +/javax.activation/activation = 1.1.1 +/javax.servlet/servlet-api = 2.5, 3.0-alpha-1 +/log4j/log4j = 1.2.17 +/org.apache.avro/avro = 1.7.5 +/org.ow2.asm/asm = 5.0_BETA +/org.tukaani/xz = 1.4 +/org.xerial.snappy/snappy-java = 1.0.5 diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 9f31ffdc8f6..56966a5fc58 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -113,6 +113,8 @@ org.apache.hadoop.version = 2.2.0 /org.apache.httpcomponents/httpcore = 4.3 /org.apache.httpcomponents/httpmime = 4.3.1 +/org.apache.ivy/ivy = 2.3.0 + org.apache.james.apache.mime4j.version = 0.7.2 /org.apache.james/apache-mime4j-core = ${org.apache.james.apache.mime4j.version} /org.apache.james/apache-mime4j-dom = ${org.apache.james.apache.mime4j.version} diff --git a/lucene/licenses/ivy-2.3.0.jar.sha1 b/lucene/licenses/ivy-2.3.0.jar.sha1 new file mode 100644 index 00000000000..f4b036fe046 --- /dev/null +++ b/lucene/licenses/ivy-2.3.0.jar.sha1 @@ -0,0 +1 @@ +c5ebf1c253ad4959a29f4acfe696ee48cdd9f473 diff --git a/lucene/licenses/ivy-LICENSE-ASL.txt b/lucene/licenses/ivy-LICENSE-ASL.txt new file mode 100644 index 00000000000..eb06170386b --- /dev/null +++ b/lucene/licenses/ivy-LICENSE-ASL.txt @@ -0,0 +1,258 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------------------------------------------------------------------ +License for JCraft JSch package +------------------------------------------------------------------------------ +Copyright (c) 2002,2003,2004,2005,2006,2007 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------------------------------------------------------------------ +License for jQuery +------------------------------------------------------------------------------ +Copyright (c) 2007 John Resig, http://jquery.com/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + \ No newline at end of file diff --git a/lucene/licenses/ivy-NOTICE.txt b/lucene/licenses/ivy-NOTICE.txt new file mode 100644 index 00000000000..33d7e07c961 --- /dev/null +++ b/lucene/licenses/ivy-NOTICE.txt @@ -0,0 +1,16 @@ +Apache Ivy (TM) +Copyright 2007-2013 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Portions of Ivy were originally developed by +Jayasoft SARL (http://www.jayasoft.fr/) +and are licensed to the Apache Software Foundation under the +"Software Grant License Agreement" + +SSH and SFTP support is provided by the JCraft JSch package, +which is open source software, available under +the terms of a BSD style license. +The original software and related information is available +at http://www.jcraft.com/jsch/. \ No newline at end of file diff --git a/lucene/tools/custom-tasks.xml b/lucene/tools/custom-tasks.xml index e38b0b137a4..6b4f8236731 100644 --- a/lucene/tools/custom-tasks.xml +++ b/lucene/tools/custom-tasks.xml @@ -88,6 +88,9 @@ + + + Lib versions check under: @{dir} - + diff --git a/lucene/tools/ivy.xml b/lucene/tools/ivy.xml index 6b8cafbe059..9f8d9681c68 100644 --- a/lucene/tools/ivy.xml +++ b/lucene/tools/ivy.xml @@ -20,6 +20,7 @@ + diff --git a/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java b/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java index 466fa8e38b8..2f1e9672cae 100644 --- a/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java +++ b/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java @@ -39,7 +39,6 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -83,7 +82,6 @@ public class GetMavenDependenciesTask extends Task { // lucene/build/core/classes/java private static final Pattern COMPILATION_OUTPUT_DIRECTORY_PATTERN = Pattern.compile("(lucene|solr)/build/(?:contrib/)?(.*)/classes/(?:java|test)"); - private static final Pattern PROPERTY_REFERENCE_PATTERN = Pattern.compile("\\$\\{([^}]+)\\}"); private static final String UNWANTED_INTERNAL_DEPENDENCIES = "/(?:test-)?lib/|test-framework/classes/java|/test-files|/resources"; private static final Pattern SHARED_EXTERNAL_DEPENDENCIES_PATTERN @@ -437,7 +435,13 @@ public class GetMavenDependenciesTask extends Task { private void appendAllExternalDependencies(StringBuilder dependenciesBuilder, Map versionsMap) { log("Loading centralized ivy versions from: " + centralizedVersionsFile, verboseLevel); ivyCacheDir = getIvyCacheDir(); - Properties versions = loadPropertiesFile(centralizedVersionsFile); + Properties versions = new InterpolatedProperties(); + try (InputStream inputStream = new FileInputStream(centralizedVersionsFile); + Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + versions.load(reader); + } catch (IOException e) { + throw new BuildException("Exception reading centralized versions file " + centralizedVersionsFile.getPath(), e); + } SortedSet sortedEntries = new TreeSet<>(new Comparator() { @Override public int compare(Map.Entry o1, Map.Entry o2) { return ((String)o1.getKey()).compareTo((String)o2.getKey()); @@ -510,7 +514,15 @@ public class GetMavenDependenciesTask extends Task { */ private void setInternalDependencyProperties() { log("Loading module dependencies from: " + moduleDependenciesPropertiesFile, verboseLevel); - Properties moduleDependencies = loadPropertiesFile(moduleDependenciesPropertiesFile); + Properties moduleDependencies = new Properties(); + try (InputStream inputStream = new FileInputStream(moduleDependenciesPropertiesFile); + Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + moduleDependencies.load(reader); + } catch (FileNotFoundException e) { + throw new BuildException("Properties file does not exist: " + moduleDependenciesPropertiesFile.getPath()); + } catch (IOException e) { + throw new BuildException("Exception reading properties file " + moduleDependenciesPropertiesFile.getPath(), e); + } Map> testScopeDependencies = new HashMap<>(); Map testScopePropertyKeys = new HashMap<>(); for (Map.Entry entry : moduleDependencies.entrySet()) { @@ -809,48 +821,6 @@ public class GetMavenDependenciesTask extends Task { return builder.toString().replace("solr-solr-", "solr-"); } - /** - * Parse the given properties file, performing non-recursive Ant-like - * property value interpolation, and return the resulting Properties. - */ - private Properties loadPropertiesFile(File file) { - final InputStream stream; - try { - stream = new FileInputStream(file); - } catch (FileNotFoundException e) { - throw new BuildException("Properties file does not exist: " + file.getPath()); - } - // Properties files are encoded as Latin-1 - final Reader reader = new InputStreamReader(stream, StandardCharsets.ISO_8859_1); - final Properties properties = new Properties(); - try { - properties.load(reader); - } catch (IOException e) { - throw new BuildException("Exception reading properties file " + file, e); - } finally { - try { - reader.close(); - } catch (IOException e) { - // do nothing - } - } - // Perform non-recursive Ant-like property value interpolation - StringBuffer buffer = new StringBuffer(); - for (Map.Entry entry : properties.entrySet()) { - buffer.setLength(0); - Matcher matcher = PROPERTY_REFERENCE_PATTERN.matcher((String)entry.getValue()); - while (matcher.find()) { - String interpolatedValue = properties.getProperty(matcher.group(1)); - if (null != interpolatedValue) { - matcher.appendReplacement(buffer, interpolatedValue); - } - } - matcher.appendTail(buffer); - properties.setProperty((String)entry.getKey(), buffer.toString()); - } - return properties; - } - /** * Appends a <dependency> snippet to the given builder. */ diff --git a/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java b/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java new file mode 100644 index 00000000000..c74dbc9c89e --- /dev/null +++ b/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java @@ -0,0 +1,72 @@ +package org.apache.lucene.dependencies; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parse a properties file, performing non-recursive Ant-like + * property value interpolation, and return the resulting Properties. + */ +public class InterpolatedProperties extends Properties { + private static final Pattern PROPERTY_REFERENCE_PATTERN = Pattern.compile("\\$\\{([^}]+)\\}"); + + /** + * Loads the properties file via {@link Properties#load(InputStream)}, + * then performs non-recursive Ant-like property value interpolation. + */ + @Override + public void load(InputStream inStream) throws IOException { + throw new UnsupportedOperationException("InterpolatedProperties.load(InputStream) is not supported."); + } + + /** + * Loads the properties file via {@link Properties#load(Reader)}, + * then performs non-recursive Ant-like property value interpolation. + */ + @Override + public void load(Reader reader) throws IOException { + super.load(reader); + interpolate(); + } + + /** + * Perform non-recursive Ant-like property value interpolation + */ + private void interpolate() { + StringBuffer buffer = new StringBuffer(); + for (Map.Entry entry : entrySet()) { + buffer.setLength(0); + Matcher matcher = PROPERTY_REFERENCE_PATTERN.matcher(entry.getValue().toString()); + while (matcher.find()) { + String interpolatedValue = getProperty(matcher.group(1)); + if (null != interpolatedValue) { + matcher.appendReplacement(buffer, interpolatedValue); + } + } + matcher.appendTail(buffer); + setProperty((String) entry.getKey(), buffer.toString()); + } + } +} diff --git a/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java b/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java index 9777df9a965..2120af0f32e 100644 --- a/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java +++ b/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java @@ -17,6 +17,15 @@ package org.apache.lucene.validation; * limitations under the License. */ +import org.apache.ivy.Ivy; +import org.apache.ivy.core.LogOptions; +import org.apache.ivy.core.report.ResolveReport; +import org.apache.ivy.core.resolve.ResolveOptions; +import org.apache.ivy.core.settings.IvySettings; +import org.apache.ivy.plugins.conflict.NoConflictManager; +import org.apache.lucene.dependencies.InterpolatedProperties; +import org.apache.lucene.validation.ivyde.IvyNodeElement; +import org.apache.lucene.validation.ivyde.IvyNodeElementAdapter; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Project; import org.apache.tools.ant.Task; @@ -24,7 +33,6 @@ import org.apache.tools.ant.types.Resource; import org.apache.tools.ant.types.ResourceCollection; import org.apache.tools.ant.types.resources.FileResource; import org.apache.tools.ant.types.resources.Resources; -import org.apache.tools.ant.util.FileNameMapper; import org.xml.sax.Attributes; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -33,23 +41,35 @@ import org.xml.sax.helpers.DefaultHandler; import org.xml.sax.helpers.XMLReaderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.Reader; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; +import java.io.StringWriter; +import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -62,7 +82,7 @@ import java.util.regex.Pattern; public class LibVersionsCheckTask extends Task { private static final String IVY_XML_FILENAME = "ivy.xml"; - private static final Pattern COORDINATE_KEY_PATTERN = Pattern.compile("(/[^/ \t\f]+/[^=:/ \t\f]+).*"); + private static final Pattern COORDINATE_KEY_PATTERN = Pattern.compile("(/([^/ \t\f]+)/([^=:/ \t\f]+))"); private static final Pattern BLANK_OR_COMMENT_LINE_PATTERN = Pattern.compile("[ \t\f]*(?:[#!].*)?"); private static final Pattern TRAILING_BACKSLASH_PATTERN = Pattern.compile("[^\\\\]*(\\\\+)$"); private static final Pattern LEADING_WHITESPACE_PATTERN = Pattern.compile("[ \t\f]+(.*)"); @@ -70,6 +90,10 @@ public class LibVersionsCheckTask extends Task { = Pattern.compile("[ \t\f]*(.*?)(?:(? referencedCoordinateKeys = new LinkedHashMap<>(); + private Map directDependencies = new LinkedHashMap<>(); + /** + * All /org/name keys found in ivy-ignore-conflicts.properties, + * mapped to the set of indirect dependency versions that will + * be ignored, i.e. not trigger a conflict. + */ + private Map> ignoreConflictVersions = new HashMap<>(); + + private class Versions { + String direct; + LinkedHashMap> conflictLocations = new LinkedHashMap<>(); // dependency path -> moduleNames + boolean directlyReferenced = false; + Versions(String direct) { + this.direct = direct; + } + } + /** * Adds a set of ivy.xml resources to check. */ @@ -117,6 +164,18 @@ public class LibVersionsCheckTask extends Task { centralizedVersionsFile = file; } + public void setIvySettingsFile(File file) { + ivySettingsFile = file; + } + + public void setCommonBuildDir(File file) { + commonBuildDir = file; + } + + public void setIgnoreConflictsFile(File file) { + ignoreConflictsFile = file; + } + /** * Execute the task. */ @@ -125,8 +184,21 @@ public class LibVersionsCheckTask extends Task { log("Starting scan.", verboseLevel); long start = System.currentTimeMillis(); - int errors = verifySortedCentralizedVersionsFile() ? 0 : 1; - int checked = 0; + setupIvy(); + + int numErrors = 0; + if ( ! verifySortedCoordinatesPropertiesFile(centralizedVersionsFile)) { + ++numErrors; + } + if ( ! verifySortedCoordinatesPropertiesFile(ignoreConflictsFile)) { + ++numErrors; + } + collectDirectDependencies(); + if ( ! collectVersionConflictsToIgnore()) { + ++numErrors; + } + + int numChecked = 0; @SuppressWarnings("unchecked") Iterator iter = (Iterator)ivyXmlResources.iterator(); @@ -142,89 +214,434 @@ public class LibVersionsCheckTask extends Task { File ivyXmlFile = ((FileResource)resource).getFile(); try { - if ( ! checkIvyXmlFile(ivyXmlFile) ) { - failures = true; - errors++; + if ( ! checkIvyXmlFile(ivyXmlFile)) { + ++numErrors; + } + if ( ! resolveTransitively(ivyXmlFile)) { + ++numErrors; } } catch (Exception e) { - throw new BuildException("Exception reading file " + ivyXmlFile.getPath(), e); + throw new BuildException("Exception reading file " + ivyXmlFile.getPath() + " - " + e.toString(), e); } - checked++; + ++numChecked; } + log("Checking for orphans in " + centralizedVersionsFile.getName(), verboseLevel); - for (Map.Entry entry : referencedCoordinateKeys.entrySet()) { + for (Map.Entry entry : directDependencies.entrySet()) { String coordinateKey = entry.getKey(); - boolean isReferenced = entry.getValue(); - if ( ! isReferenced) { + if ( ! entry.getValue().directlyReferenced) { log("ORPHAN coordinate key '" + coordinateKey + "' in " + centralizedVersionsFile.getName() + " is not found in any " + IVY_XML_FILENAME + " file.", Project.MSG_ERR); - failures = true; - errors++; + ++numErrors; } } - log(String.format(Locale.ROOT, "Checked that %s has lexically sorted " - + "'/org/name' keys and no duplicates or orphans, and scanned %d %s " - + "file(s) for rev=\"${/org/name}\" format (in %.2fs.), %d error(s).", - centralizedVersionsFile.getName(), checked, IVY_XML_FILENAME, - (System.currentTimeMillis() - start) / 1000.0, errors), - errors > 0 ? Project.MSG_ERR : Project.MSG_INFO); + int numConflicts = emitConflicts(); - if (failures) { + int messageLevel = numErrors > 0 ? Project.MSG_ERR : Project.MSG_INFO; + log("Checked that " + centralizedVersionsFile.getName() + " and " + ignoreConflictsFile.getName() + + " have lexically sorted '/org/name' keys and no duplicates or orphans.", + messageLevel); + log("Scanned " + numChecked + " " + IVY_XML_FILENAME + " files for rev=\"${/org/name}\" format.", + messageLevel); + log("Found " + numConflicts + " indirect dependency version conflicts."); + log(String.format(Locale.ROOT, "Completed in %.2fs., %d error(s).", + (System.currentTimeMillis() - start) / 1000.0, numErrors), + messageLevel); + + if (numConflicts > 0 || numErrors > 0) { throw new BuildException("Lib versions check failed. Check the logs."); } } /** - * Returns true if the "/org/name" coordinate keys in ivy-versions.properties - * are lexically sorted and are not duplicates. + * Collects indirect dependency version conflicts to ignore + * in ivy-ignore-conflicts.properties, and also checks for orphans + * (coordinates not included in ivy-versions.properties). + * + * Returns true if no orphans are found. */ - private boolean verifySortedCentralizedVersionsFile() { - log("Checking for lexically sorted non-duplicated '/org/name' keys in: " + centralizedVersionsFile, verboseLevel); - final InputStream stream; - try { - stream = new FileInputStream(centralizedVersionsFile); - } catch (FileNotFoundException e) { - throw new BuildException("Centralized versions file does not exist: " - + centralizedVersionsFile.getPath()); + private boolean collectVersionConflictsToIgnore() { + log("Checking for orphans in " + ignoreConflictsFile.getName(), verboseLevel); + boolean orphansFound = false; + InterpolatedProperties properties = new InterpolatedProperties(); + try (InputStream inputStream = new FileInputStream(ignoreConflictsFile); + Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + properties.load(reader); + } catch (IOException e) { + throw new BuildException("Exception reading " + ignoreConflictsFile + ": " + e.toString(), e); } - // Properties files are encoded as Latin-1 - final Reader reader = new InputStreamReader(stream, StandardCharsets.ISO_8859_1); - final BufferedReader bufferedReader = new BufferedReader(reader); - + for (Object obj : properties.keySet()) { + String coordinate = (String)obj; + if (COORDINATE_KEY_PATTERN.matcher(coordinate).matches()) { + if ( ! directDependencies.containsKey(coordinate)) { + orphansFound = true; + log("ORPHAN coordinate key '" + coordinate + "' in " + ignoreConflictsFile.getName() + + " is not found in " + centralizedVersionsFile.getName(), + Project.MSG_ERR); + } else { + String versionsToIgnore = properties.getProperty(coordinate); + List ignore = Arrays.asList(versionsToIgnore.trim().split("\\s*,\\s*|\\s+")); + ignoreConflictVersions.put(coordinate, new HashSet<>(ignore)); + } + } + } + return ! orphansFound; + } + + private void collectDirectDependencies() { + InterpolatedProperties properties = new InterpolatedProperties(); + try (InputStream inputStream = new FileInputStream(centralizedVersionsFile); + Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + properties.load(reader); + } catch (IOException e) { + throw new BuildException("Exception reading " + centralizedVersionsFile + ": " + e.toString(), e); + } + for (Object obj : properties.keySet()) { + String coordinate = (String)obj; + if (COORDINATE_KEY_PATTERN.matcher(coordinate).matches()) { + String direct = properties.getProperty(coordinate); + Versions versions = new Versions(direct); + directDependencies.put(coordinate, versions); + } + } + } + + /** + * Transitively resolves all dependencies in the given ivy.xml file, + * looking for indirect dependencies with versions that conflict + * with those of direct dependencies. Versions conflict when a + * direct dependency's version is older than that of an indirect + * dependency with the same /org/name. + * + * Returns true if no version conflicts are found and no resolution + * errors occurred, false otherwise. + */ + private boolean resolveTransitively(File ivyXmlFile) { + boolean success = true; + + ResolveOptions options = new ResolveOptions(); + options.setDownload(false); // Download only module descriptors, not artifacts + options.setTransitive(true); // Resolve transitively, if not already specified in the ivy.xml file + options.setUseCacheOnly(false); // Download the internet! + options.setOutputReport(false); // Don't print to the console + options.setLog(LogOptions.LOG_QUIET); // Don't log to the console + options.setConfs(new String[] {"*"}); // Resolve all configurations + + // Rewrite the ivy.xml, replacing all 'transitive="false"' with 'transitive="true"' + // The Ivy API is file-based, so we have to write the result to the filesystem. + String moduleName = "unknown"; + String ivyXmlContent = xmlToString(ivyXmlFile); + Matcher matcher = MODULE_NAME_PATTERN.matcher(ivyXmlContent); + if (matcher.find()) { + moduleName = matcher.group(1); + } + ivyXmlContent = ivyXmlContent.replaceAll("\\btransitive\\s*=\\s*[\"']false[\"']", "transitive=\"true\""); + File transitiveIvyXmlFile = null; + try { + File buildDir = new File(commonBuildDir, "ivy-transitive-resolve"); + if ( ! buildDir.exists() && ! buildDir.mkdirs()) { + throw new BuildException("Could not create temp directory " + buildDir.getPath()); + } + matcher = MODULE_DIRECTORY_PATTERN.matcher(ivyXmlFile.getCanonicalPath()); + if ( ! matcher.matches()) { + throw new BuildException("Unknown ivy.xml module directory: " + ivyXmlFile.getCanonicalPath()); + } + String moduleDirPrefix = matcher.group(1).replaceAll("[/\\\\]", "."); + transitiveIvyXmlFile = new File(buildDir, "transitive." + moduleDirPrefix + ".ivy.xml"); + try (Writer writer = new OutputStreamWriter(new FileOutputStream(transitiveIvyXmlFile), StandardCharsets.UTF_8)) { + writer.write(ivyXmlContent); + } + ResolveReport resolveReport = ivy.resolve(transitiveIvyXmlFile.toURI().toURL(), options); + IvyNodeElement root = IvyNodeElementAdapter.adapt(resolveReport); + for (IvyNodeElement directDependency : root.getDependencies()) { + String coordinate = "/" + directDependency.getOrganization() + "/" + directDependency.getName(); + Versions versions = directDependencies.get(coordinate); + if (null == versions) { + log("ERROR: the following coordinate key does not appear in " + + centralizedVersionsFile.getName() + ": " + coordinate); + success = false; + } else { + versions.directlyReferenced = true; + if (collectConflicts(directDependency, directDependency, moduleName)) { + success = false; + } + } + } + } catch (ParseException | IOException e) { + if (null != transitiveIvyXmlFile) { + log("Exception reading " + transitiveIvyXmlFile.getPath() + ": " + e.toString()); + } + success = false; + } + return success; + } + + /** + * Recursively finds indirect dependencies that have a version conflict with a direct dependency. + * Returns true if one or more conflicts are found, false otherwise + */ + private boolean collectConflicts(IvyNodeElement root, IvyNodeElement parent, String moduleName) { + boolean conflicts = false; + for (IvyNodeElement child : parent.getDependencies()) { + String coordinate = "/" + child.getOrganization() + "/" + child.getName(); + Versions versions = directDependencies.get(coordinate); + if (null != versions) { // Ignore this indirect dependency if it's not also a direct dependency + String indirectVersion = child.getRevision(); + if (isConflict(coordinate, versions.direct, indirectVersion)) { + conflicts = true; + Set moduleNames = versions.conflictLocations.get(root); + if (null == moduleNames) { + moduleNames = new HashSet<>(); + versions.conflictLocations.put(root, moduleNames); + } + moduleNames.add(moduleName); + } + conflicts |= collectConflicts(root, child, moduleName); + } + } + return conflicts; + } + + /** + * Copy-pasted from Ivy's + * org.apache.ivy.plugins.latest.LatestRevisionStrategy + * with minor modifications + */ + private static final Map SPECIAL_MEANINGS; + static { + SPECIAL_MEANINGS = new HashMap<>(); + SPECIAL_MEANINGS.put("dev", -1); + SPECIAL_MEANINGS.put("rc", 1); + SPECIAL_MEANINGS.put("final", 2); + } + + /** + * Copy-pasted from Ivy's + * org.apache.ivy.plugins.latest.LatestRevisionStrategy.MridComparator + * with minor modifications + */ + private static class LatestVersionComparator implements Comparator { + @Override + public int compare(String rev1, String rev2) { + rev1 = rev1.replaceAll("([a-zA-Z])(\\d)", "$1.$2"); + rev1 = rev1.replaceAll("(\\d)([a-zA-Z])", "$1.$2"); + rev2 = rev2.replaceAll("([a-zA-Z])(\\d)", "$1.$2"); + rev2 = rev2.replaceAll("(\\d)([a-zA-Z])", "$1.$2"); + + String[] parts1 = rev1.split("[-._+]"); + String[] parts2 = rev2.split("[-._+]"); + + int i = 0; + for (; i < parts1.length && i < parts2.length; i++) { + if (parts1[i].equals(parts2[i])) { + continue; + } + boolean is1Number = isNumber(parts1[i]); + boolean is2Number = isNumber(parts2[i]); + if (is1Number && !is2Number) { + return 1; + } + if (is2Number && !is1Number) { + return -1; + } + if (is1Number && is2Number) { + return Long.valueOf(parts1[i]).compareTo(Long.valueOf(parts2[i])); + } + // both are strings, we compare them taking into account special meaning + Integer sm1 = SPECIAL_MEANINGS.get(parts1[i].toLowerCase(Locale.ROOT)); + Integer sm2 = SPECIAL_MEANINGS.get(parts2[i].toLowerCase(Locale.ROOT)); + if (sm1 != null) { + sm2 = sm2 == null ? new Integer(0) : sm2; + return sm1.compareTo(sm2); + } + if (sm2 != null) { + return new Integer(0).compareTo(sm2); + } + return parts1[i].compareTo(parts2[i]); + } + if (i < parts1.length) { + return isNumber(parts1[i]) ? 1 : -1; + } + if (i < parts2.length) { + return isNumber(parts2[i]) ? -1 : 1; + } + return 0; + } + + private static final Pattern IS_NUMBER = Pattern.compile("\\d+"); + private static boolean isNumber(String str) { + return IS_NUMBER.matcher(str).matches(); + } + } + private static LatestVersionComparator LATEST_VERSION_COMPARATOR = new LatestVersionComparator(); + + /** + * Returns true if directVersion is less than indirectVersion, and + * coordinate=indirectVersion is not present in ivy-ignore-conflicts.properties. + */ + private boolean isConflict(String coordinate, String directVersion, String indirectVersion) { + boolean isConflict = LATEST_VERSION_COMPARATOR.compare(directVersion, indirectVersion) < 0; + if (isConflict) { + Set ignoredVersions = ignoreConflictVersions.get(coordinate); + if (null != ignoredVersions && ignoredVersions.contains(indirectVersion)) { + isConflict = false; + } + } + return isConflict; + } + + /** + * Returns the number of direct dependencies in conflict with indirect + * dependencies. + */ + private int emitConflicts() { + int conflicts = 0; + StringBuilder builder = new StringBuilder(); + for (Map.Entry directDependency : directDependencies.entrySet()) { + String coordinate = directDependency.getKey(); + Set>> entrySet + = directDependency.getValue().conflictLocations.entrySet(); + if (entrySet.isEmpty()) { + continue; + } + ++conflicts; + Map.Entry> first = entrySet.iterator().next(); + int notPrinted = entrySet.size() - 1; + builder.append("VERSION CONFLICT: transitive dependency in module(s) "); + boolean isFirst = true; + for (String moduleName : first.getValue()) { + if (isFirst) { + isFirst = false; + } else { + builder.append(", "); + } + builder.append(moduleName); + } + builder.append(":\n"); + IvyNodeElement element = first.getKey(); + builder.append('/').append(element.getOrganization()).append('/').append(element.getName()) + .append('=').append(element.getRevision()).append('\n'); + emitConflict(builder, coordinate, first.getKey(), 1); + + if (notPrinted > 0) { + builder.append("... and ").append(notPrinted).append(" more\n"); + } + builder.append("\n"); + } + if (builder.length() > 0) { + log(builder.toString()); + } + return conflicts; + } + + private boolean emitConflict(StringBuilder builder, String conflictCoordinate, IvyNodeElement parent, int depth) { + for (IvyNodeElement child : parent.getDependencies()) { + String indirectCoordinate = "/" + child.getOrganization() + "/" + child.getName(); + if (conflictCoordinate.equals(indirectCoordinate)) { + String directVersion = directDependencies.get(conflictCoordinate).direct; + if (isConflict(conflictCoordinate, directVersion, child.getRevision())) { + for (int i = 0 ; i < depth - 1 ; ++i) { + builder.append(" "); + } + builder.append("+-- "); + builder.append(indirectCoordinate).append("=").append(child.getRevision()); + builder.append(" <<< Conflict (direct=").append(directVersion).append(")\n"); + return true; + } + } else if (hasConflicts(conflictCoordinate, child)) { + for (int i = 0 ; i < depth -1 ; ++i) { + builder.append(" "); + } + builder.append("+-- "); + builder.append(indirectCoordinate).append("=").append(child.getRevision()).append("\n"); + if (emitConflict(builder, conflictCoordinate, child, depth + 1)) { + return true; + } + } + } + return false; + } + + private boolean hasConflicts(String conflictCoordinate, IvyNodeElement parent) { + // the element itself will never be in conflict, since its coordinate is different + for (IvyNodeElement child : parent.getDependencies()) { + String indirectCoordinate = "/" + child.getOrganization() + "/" + child.getName(); + if (conflictCoordinate.equals(indirectCoordinate)) { + if (isConflict(conflictCoordinate, directDependencies.get(conflictCoordinate).direct, child.getRevision())) { + return true; + } + } else if (hasConflicts(conflictCoordinate, child)) { + return true; + } + } + return false; + } + + private String xmlToString(File ivyXmlFile) { + StringWriter writer = new StringWriter(); + try { + StreamSource inputSource = new StreamSource(new FileInputStream(ivyXmlFile.getPath())); + Transformer serializer = TransformerFactory.newInstance().newTransformer(); + serializer.transform(inputSource, new StreamResult(writer)); + } catch (TransformerException | IOException e) { + throw new BuildException("Exception reading " + ivyXmlFile.getPath() + ": " + e.toString(), e); + } + return writer.toString(); + } + + private void setupIvy() { + IvySettings ivySettings = new IvySettings(); + try { + ivySettings.setVariable("common.build.dir", commonBuildDir.getAbsolutePath()); + ivySettings.setVariable("ivy.exclude.types", "source|javadoc"); + ivySettings.setBaseDir(commonBuildDir); + ivySettings.setDefaultConflictManager(new NoConflictManager()); + ivy = Ivy.newInstance(ivySettings); + ivy.configure(ivySettingsFile); + } catch (Exception e) { + throw new BuildException("Exception reading " + ivySettingsFile.getPath() + ": " + e.toString(), e); + } + } + + /** + * Returns true if the "/org/name" coordinate keys in the given + * properties file are lexically sorted and are not duplicates. + */ + private boolean verifySortedCoordinatesPropertiesFile(File coordinatePropertiesFile) { + log("Checking for lexically sorted non-duplicated '/org/name' keys in: " + coordinatePropertiesFile, verboseLevel); + boolean success = true; String line = null; String currentKey = null; String previousKey = null; - try { + try (InputStream stream = new FileInputStream(coordinatePropertiesFile); + Reader reader = new InputStreamReader(stream, StandardCharsets.ISO_8859_1); + BufferedReader bufferedReader = new BufferedReader(reader)) { while (null != (line = readLogicalPropertiesLine(bufferedReader))) { final Matcher keyMatcher = COORDINATE_KEY_PATTERN.matcher(line); - if ( ! keyMatcher.matches()) { + if ( ! keyMatcher.lookingAt()) { continue; // Ignore keys that don't look like "/org/name" } currentKey = keyMatcher.group(1); if (null != previousKey) { int comparison = currentKey.compareTo(previousKey); if (0 == comparison) { - log("DUPLICATE coordinate key '" + currentKey + "' in " + centralizedVersionsFile.getName(), + log("DUPLICATE coordinate key '" + currentKey + "' in " + coordinatePropertiesFile.getName(), Project.MSG_ERR); - failures = true; + success = false; } else if (comparison < 0) { - log("OUT-OF-ORDER coordinate key '" + currentKey + "' in " + centralizedVersionsFile.getName(), + log("OUT-OF-ORDER coordinate key '" + currentKey + "' in " + coordinatePropertiesFile.getName(), Project.MSG_ERR); - failures = true; + success = false; } } - referencedCoordinateKeys.put(currentKey, false); previousKey = currentKey; } } catch (IOException e) { - throw new BuildException("Exception reading centralized versions file: " - + centralizedVersionsFile.getPath(), e); - } finally { - try { reader.close(); } catch (IOException e) { } + throw new BuildException("Exception reading " + coordinatePropertiesFile.getPath() + ": " + e.toString(), e); } - return ! failures; + return success; } /** @@ -294,7 +711,7 @@ public class LibVersionsCheckTask extends Task { for (int pos = 0 ; pos < numChars - 1 ; ++pos) { char ch = leadingWhitespaceStripped.charAt(pos); if (ch == '\\') { - ch = leadingWhitespaceStripped.charAt(++pos); + ch = leadingWhitespaceStripped.charAt(++pos); } output.append(ch); } @@ -360,11 +777,10 @@ public class LibVersionsCheckTask extends Task { + " in " + ivyXmlFile.getPath(), Project.MSG_ERR); fail = true; } - if ( ! referencedCoordinateKeys.containsKey(coordinateKey)) { + if ( ! directDependencies.containsKey(coordinateKey)) { log("MISSING key '" + coordinateKey + "' in " + centralizedVersionsFile.getPath(), Project.MSG_ERR); fail = true; } - referencedCoordinateKeys.put(coordinateKey, true); } } tags.push(localName); diff --git a/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElement.java b/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElement.java new file mode 100644 index 00000000000..8e517d0d4f4 --- /dev/null +++ b/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElement.java @@ -0,0 +1,180 @@ +package org.apache.lucene.validation.ivyde; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.ivy.core.module.id.ModuleRevisionId; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; + +/** + * Assists in the further separation of concerns between the view and the Ivy resolve report. The view looks at the + * IvyNode in a unique way that can lead to expensive operations if we do not achieve this separation. + * + * This class is copied from org/apache/ivyde/eclipse/resolvevisualizer/model/IvyNodeElement.java at + * https://svn.apache.org/repos/asf/ant/ivy/ivyde/trunk/org.apache.ivyde.eclipse.resolvevisualizer/src/ + * + * Changes include: uncommenting generics and converting to diamond operators where appropriate; + * removing unnecessary casts; removing javadoc tags with no description; and adding a hashCode() implementation. + */ +public class IvyNodeElement { + private ModuleRevisionId moduleRevisionId; + private boolean evicted = false; + private int depth = Integer.MAX_VALUE / 10; + private Collection dependencies = new HashSet<>(); + private Collection callers = new HashSet<>(); + private Collection conflicts = new HashSet<>(); + + /** + * The caller configurations that caused this node to be reached in the resolution, grouped by caller. + */ + private MapcallerConfigurationMap = new HashMap<>(); + + /** + * We try to avoid building the list of this nodes deep dependencies by storing them in this cache by depth level. + */ + private IvyNodeElement[] deepDependencyCache; + + @Override + public boolean equals(Object obj) { + if (obj instanceof IvyNodeElement) { + IvyNodeElement elem = (IvyNodeElement) obj; + if (elem.getOrganization().equals(getOrganization()) && elem.getName().equals(getName()) + && elem.getRevision().equals(getRevision())) + return true; + } + return false; + } + + @Override + public int hashCode() { + int result = 1; + result = result * 31 + (null == getOrganization() ? 0 : getOrganization().hashCode()); + result = result * 31 + (null == getName() ? 0 : getName().hashCode()); + result = result * 31 + (null == getRevision() ? 0 : getRevision().hashCode()); + return result; + } + + public IvyNodeElement[] getDependencies() { + return dependencies.toArray(new IvyNodeElement[dependencies.size()]); + } + + /** + * Recursive dependency retrieval + * + * @return The array of nodes that represents a node's immediate and transitive dependencies down to an arbitrary + * depth. + */ + public IvyNodeElement[] getDeepDependencies() { + if (deepDependencyCache == null) { + Collection deepDependencies = getDeepDependencies(this); + deepDependencyCache = (IvyNodeElement[])deepDependencies.toArray(new IvyNodeElement[deepDependencies.size()]); + } + return deepDependencyCache; + } + + /** + * Recursive dependency retrieval + */ + private Collection getDeepDependencies(IvyNodeElement node) { + Collection deepDependencies = new HashSet<>(); + deepDependencies.add(node); + + IvyNodeElement[] directDependencies = node.getDependencies(); + for (int i = 0; i < directDependencies.length; i++) { + deepDependencies.addAll(getDeepDependencies(directDependencies[i])); + } + + return deepDependencies; + } + + /** + * @return An array of configurations by which this module was resolved + */ + public String[] getCallerConfigurations(IvyNodeElement caller) { + return callerConfigurationMap.get(caller); + } + + public void setCallerConfigurations(IvyNodeElement caller, String[] configurations) { + callerConfigurationMap.put(caller, configurations); + } + + public String getOrganization() { + return moduleRevisionId.getOrganisation(); + } + + public String getName() { + return moduleRevisionId.getName(); + } + + public String getRevision() { + return moduleRevisionId.getRevision(); + } + + public boolean isEvicted() { + return evicted; + } + + public void setEvicted(boolean evicted) { + this.evicted = evicted; + } + + public int getDepth() { + return depth; + } + + /** + * Set this node's depth and recursively update the node's children to relative to the new value. + */ + public void setDepth(int depth) { + this.depth = depth; + for (Iterator iter = dependencies.iterator(); iter.hasNext();) { + IvyNodeElement dependency = (IvyNodeElement) iter.next(); + dependency.setDepth(depth + 1); + } + } + + public IvyNodeElement[] getConflicts() { + return conflicts.toArray(new IvyNodeElement[conflicts.size()]); + } + + public void setConflicts(Collection conflicts) { + this.conflicts = conflicts; + } + + public ModuleRevisionId getModuleRevisionId() { + return moduleRevisionId; + } + + public void setModuleRevisionId(ModuleRevisionId moduleRevisionId) { + this.moduleRevisionId = moduleRevisionId; + } + + public void addCaller(IvyNodeElement caller) { + callers.add(caller); + caller.dependencies.add(this); + } + + public IvyNodeElement[] getCallers() { + return callers.toArray(new IvyNodeElement[callers.size()]); + } +} \ No newline at end of file diff --git a/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElementAdapter.java b/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElementAdapter.java new file mode 100644 index 00000000000..d0d551d6434 --- /dev/null +++ b/lucene/tools/src/java/org/apache/lucene/validation/ivyde/IvyNodeElementAdapter.java @@ -0,0 +1,137 @@ +package org.apache.lucene.validation.ivyde; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import org.apache.ivy.core.module.id.ModuleId; +import org.apache.ivy.core.module.id.ModuleRevisionId; +import org.apache.ivy.core.report.ResolveReport; +import org.apache.ivy.core.resolve.IvyNode; +import org.apache.ivy.core.resolve.IvyNodeCallers; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * This class is copied from org/apache/ivyde/eclipse/resolvevisualizer/model/IvyNodeElementAdapter.java at + * https://svn.apache.org/repos/asf/ant/ivy/ivyde/trunk/org.apache.ivyde.eclipse.resolvevisualizer/src/ + * + * Changes include: uncommenting generics and converting to diamond operators where appropriate; + * removing unnecessary casts; and removing javadoc tags with no description. + */ +public class IvyNodeElementAdapter { + /** + * Adapt all dependencies and evictions from the ResolveReport. + * @return the root node adapted from the ResolveReport + */ + public static IvyNodeElement adapt(ResolveReport report) { + Map resolvedNodes = new HashMap<>(); + + IvyNodeElement root = new IvyNodeElement(); + root.setModuleRevisionId(report.getModuleDescriptor().getModuleRevisionId()); + resolvedNodes.put(report.getModuleDescriptor().getModuleRevisionId(), root); + + @SuppressWarnings("unchecked") List dependencies = report.getDependencies(); + + // First pass - build the map of resolved nodes by revision id + for (Iterator iter = dependencies.iterator(); iter.hasNext();) { + IvyNode node = (IvyNode) iter.next(); + if (node.getAllEvictingNodes() != null) { + // Nodes that are evicted as a result of conf inheritance still appear + // as dependencies, but with eviction data. They also appear as evictions. + // We map them as evictions rather than dependencies. + continue; + } + IvyNodeElement nodeElement = new IvyNodeElement(); + nodeElement.setModuleRevisionId(node.getResolvedId()); + resolvedNodes.put(node.getResolvedId(), nodeElement); + } + + // Second pass - establish relationships between the resolved nodes + for (Iterator iter = dependencies.iterator(); iter.hasNext();) { + IvyNode node = (IvyNode) iter.next(); + if (node.getAllEvictingNodes() != null) { + continue; // see note above + } + + IvyNodeElement nodeElement = resolvedNodes.get(node.getResolvedId()); + IvyNodeCallers.Caller[] callers = node.getAllRealCallers(); + for (int i = 0; i < callers.length; i++) { + IvyNodeElement caller = resolvedNodes.get(callers[i].getModuleRevisionId()); + if (caller != null) { + nodeElement.addCaller(caller); + nodeElement.setCallerConfigurations(caller, callers[i].getCallerConfigurations()); + } + } + } + + IvyNode[] evictions = report.getEvictedNodes(); + for (int i = 0; i < evictions.length; i++) { + IvyNode eviction = evictions[i]; + IvyNodeElement evictionElement = new IvyNodeElement(); + evictionElement.setModuleRevisionId(eviction.getResolvedId()); + evictionElement.setEvicted(true); + + IvyNodeCallers.Caller[] callers = eviction.getAllCallers(); + for (int j = 0; j < callers.length; j++) { + IvyNodeElement caller = resolvedNodes.get(callers[j].getModuleRevisionId()); + if (caller != null) { + evictionElement.addCaller(caller); + evictionElement.setCallerConfigurations(caller, callers[j].getCallerConfigurations()); + } + } + } + + // Recursively set depth starting at root + root.setDepth(0); + findConflictsBeneathNode(root); + + return root; + } + + /** + * Derives configuration conflicts that exist between node and all of its descendant dependencies. + */ + private static void findConflictsBeneathNode(IvyNodeElement node) { + // Derive conflicts + Map> moduleRevisionMap = new HashMap<>(); + IvyNodeElement[] deepDependencies = node.getDeepDependencies(); + for (int i = 0; i < deepDependencies.length; i++) { + if (deepDependencies[i].isEvicted()) + continue; + + ModuleId moduleId = deepDependencies[i].getModuleRevisionId().getModuleId(); + if (moduleRevisionMap.containsKey(moduleId)) { + Collection conflicts = moduleRevisionMap.get(moduleId); + conflicts.add(deepDependencies[i]); + for (Iterator iter = conflicts.iterator(); iter.hasNext();) { + IvyNodeElement conflict = (IvyNodeElement) iter.next(); + conflict.setConflicts(conflicts); + } + } else { + List immutableMatchingSet = Arrays.asList(deepDependencies[i]); + moduleRevisionMap.put(moduleId, new HashSet<>(immutableMatchingSet)); + } + } + } +} \ No newline at end of file