Merge pull request #11932 from rmuir/jarhell

Fail startup (and tests) on jar hell

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -179,6 +179,9 @@ public class Bootstrap {
         // install any plugins into classpath
         setupPlugins(environment);
         
+        // look for jar hell
+        JarHell.checkJarHell();
+        
         // install SM after natives, shutdown hooks, etc.
         setupSecurity(settings, environment);
 
@@ -375,7 +378,8 @@ public class Bootstrap {
             return;
         }
 
-        ClassLoader classLoader = ClassLoader.getSystemClassLoader();
+        // note: there's only one classloader here, but Uwe gets upset otherwise.
+        ClassLoader classLoader = Bootstrap.class.getClassLoader();
         Class<?> classLoaderClass = classLoader.getClass();
         Method addURL = null;
         while (!classLoaderClass.equals(Object.class)) {

core/src/main/java/org/elasticsearch/bootstrap/JarHell.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+/** Simple check for duplicate class files across the classpath */
+class JarHell {
+
+    /**
+     * Checks the current classloader for duplicate classes
+     * @throws IllegalStateException if jar hell was found
+     */
+    @SuppressForbidden(reason = "needs JarFile for speed, just reading entries")
+    static void checkJarHell() throws Exception {
+        ClassLoader loader = JarHell.class.getClassLoader();
+        if (loader instanceof URLClassLoader == false) {
+           return;
+        }
+        final Map<String,URL> clazzes = new HashMap<>(32768);
+        Set<String> seenJars = new HashSet<>();
+        for (final URL url : ((URLClassLoader)loader).getURLs()) {
+            String path = url.getPath();
+            if (path.endsWith(".jar")) {
+                if (!seenJars.add(path)) {
+                    continue; // we can't fail because of sheistiness with joda-time
+                }
+                try (JarFile file = new JarFile(url.getPath())) {
+                    Enumeration<JarEntry> elements = file.entries();
+                    while (elements.hasMoreElements()) {
+                        String entry = elements.nextElement().getName();
+                        if (entry.endsWith(".class")) {
+                            // for jar format, the separator is defined as /
+                            entry = entry.replace('/', '.').substring(0, entry.length() - 6);
+                            checkClass(clazzes, entry, url);
+                        }
+                    }
+                }
+            } else {
+                // case for tests: where we have class files in the classpath
+                final Path root = PathUtils.get(url.toURI());
+                final String sep = root.getFileSystem().getSeparator();
+                Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
+                    @Override
+                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                        String entry = root.relativize(file).toString();
+                        if (entry.endsWith(".class")) {
+                            // normalize with the os separator
+                            entry = entry.replace(sep, ".").substring(0,  entry.length() - 6);
+                            checkClass(clazzes, entry, url);
+                        }
+                        return super.visitFile(file, attrs);
+                    }
+                });
+            }
+        }
+    }
+    
+    @SuppressForbidden(reason = "proper use of URL to reduce noise")
+    static void checkClass(Map<String,URL> clazzes, String clazz, URL url) {
+        if (clazz.startsWith("org.apache.log4j")) {
+            return; // go figure, jar hell for what should be System.out.println...
+        }
+        if (clazz.equals("org.joda.time.base.BaseDateTime")) {
+            return; // apparently this is intentional... clean this up
+        }
+        URL previous = clazzes.put(clazz, url);
+        if (previous != null) {
+            throw new IllegalStateException("jar hell!" + System.lineSeparator() +
+                    "class: " + clazz + System.lineSeparator() +
+                    "jar1: " + previous.getPath() + System.lineSeparator() +
+                    "jar2: " + url.getPath());
+        }
+    }
+}

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -81,7 +81,7 @@ final class Security {
      */
     @SuppressForbidden(reason = "proper use of URL")
     static void setCodebaseProperties() {
-        ClassLoader loader = ClassLoader.getSystemClassLoader();
+        ClassLoader loader = Security.class.getClassLoader();
         if (loader instanceof URLClassLoader) {
             for (URL url : ((URLClassLoader)loader).getURLs()) {
                 for (Map.Entry<Pattern,String> e : SPECIAL_JARS.entrySet()) {

core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.bulk;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.elasticsearch.action.get.MultiGetItemResponse;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;

core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -51,6 +51,13 @@ public class BootstrapForTesting {
         // just like bootstrap, initialize natives, then SM
         Bootstrap.initializeNatives(true, true, true);
         
+        // check for jar hell
+        try {
+            JarHell.checkJarHell();
+        } catch (Exception e) {
+            throw new RuntimeException("found jar hell in test classpath", e);
+        }
+
         // make sure java.io.tmpdir exists always (in case code uses it in a static initializer)
         Path javaTmpDir = PathUtils.get(Objects.requireNonNull(System.getProperty("java.io.tmpdir"),
                                                                "please set ${java.io.tmpdir} in pom.xml"));

core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java

@@ -19,10 +19,12 @@
 
 package org.elasticsearch.client.node;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet;
+
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.AbstractClientHeadersTests;
@@ -31,6 +33,7 @@ import org.elasticsearch.client.support.Headers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Collections;
 import java.util.HashMap;
 
 /**
@@ -38,7 +41,7 @@ import java.util.HashMap;
  */
 public class NodeClientHeadersTests extends AbstractClientHeadersTests {
 
-    private static final ActionFilters EMPTY_FILTERS = new ActionFilters(ImmutableSet.of());
+    private static final ActionFilters EMPTY_FILTERS = new ActionFilters(Collections.<ActionFilter>emptySet());
 
     @Override
     protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {

core/src/test/java/org/elasticsearch/common/collect/CopyOnWriteHashMapTests.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.common.collect;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap;
 import org.elasticsearch.test.ElasticsearchTestCase;
 
 import java.util.HashMap;

core/src/test/java/org/elasticsearch/common/lucene/index/FreqTermsEnumTests.java

@@ -19,8 +19,8 @@
 
 package org.elasticsearch.common.lucene.index;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Maps;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 import org.apache.lucene.analysis.core.KeywordAnalyzer;

core/src/test/java/org/elasticsearch/index/store/CorruptedFileTest.java

@@ -18,10 +18,11 @@
  */
 package org.elasticsearch.index.store;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.google.common.base.Charsets;
 import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.IndexFileNames;

core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogTests.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.store;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;

core/src/test/java/org/elasticsearch/test/XContentTestUtils.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.test;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;

core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportTests.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.transport;
 
-import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.ImmutableList;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.action.*;
@@ -380,7 +379,9 @@ public class ContextAndHeaderTransportTests extends ElasticsearchIntegrationTest
 
         @Override
         public Collection<Class<? extends Module>> modules() {
-            return ImmutableList.of(ActionLoggingModule.class);
+            Collection<Class<? extends Module>> classes = new ArrayList<>();
+            classes.add(ActionLoggingModule.class);
+            return classes;
         }
     }
 

plugins/cloud-azure/licenses/stax-api-1.0.1.jar.sha1

@@ -1 +0,0 @@
-49c100caf72d658aca8e58bd74a4ba90fa2b0d70

plugins/cloud-azure/pom.xml

@@ -44,6 +44,12 @@ governing permissions and limitations under the License. -->
             <groupId>com.microsoft.azure</groupId>
             <artifactId>azure-management-compute</artifactId>
             <version>0.7.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>stax</groupId>
+                    <artifactId>stax-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>com.microsoft.azure</groupId>

plugins/cloud-gce/licenses/guava-LICENSE.txt

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

plugins/cloud-gce/licenses/guava-NOTICE.txt

@@ -1 +0,0 @@
- 

plugins/cloud-gce/licenses/guava-jdk5-13.0.jar.sha1

@@ -1 +0,0 @@
-381ae6c8add7c98e9bcddb1e0b898400cd6fc3f7

plugins/cloud-gce/pom.xml

@@ -43,6 +43,12 @@ governing permissions and limitations under the License. -->
           <groupId>com.google.apis</groupId>
           <artifactId>google-api-services-compute</artifactId>
           <version>${google.gce.version}</version>
+          <exclusions>
+              <exclusion>
+                 <groupId>com.google.guava</groupId>
+                 <artifactId>guava-jdk5</artifactId>
+              </exclusion>
+          </exclusions>
         </dependency>
     </dependencies>
 

plugins/pom.xml

@@ -29,11 +29,6 @@
             <artifactId>hamcrest-all</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-test-framework</artifactId>

pom.xml

@@ -163,6 +163,13 @@
                 <groupId>org.apache.lucene</groupId>
                 <artifactId>lucene-test-framework</artifactId>
                 <version>${lucene.maven.version}</version>
+                <!-- we specify our own junit4 version -->
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.carrotsearch.randomizedtesting</groupId>
+                        <artifactId>junit4-ant</artifactId>
+                    </exclusion>
+                </exclusions>
             </dependency>
 
             <dependency>