SOLR-6643: Fix error reporting & logging of low level JVM Errors that occur when loading/reloading a SolrCore

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1650350 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -406,6 +406,9 @@ Bug Fixes
 * SOLR-6880: Harden ZkStateReader to expect that getCollectionLive may return null
   as it's contract states. (Mark Miller, shalin)
 
+* SOLR-6643: Fix error reporting & logging of low level JVM Errors that occur when 
+  loading/reloading a SolrCore (hossman)
+
 Optimizations
 ----------------------
 

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -519,11 +519,15 @@ public class CoreContainer {
 
       return core;
 
-    }
+    } catch (Exception e) {
-    catch (Exception e) {
       coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
       log.error("Error creating core [{}]: {}", dcore.getName(), e.getMessage(), e);
       throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to create core [" + dcore.getName() + "]", e);
+    } catch (Throwable t) {
+      SolrException e = new SolrException(ErrorCode.SERVER_ERROR, "JVM Error creating core [" + dcore.getName() + "]: " + t.getMessage(), t);
+      log.error("Error creating core [{}]: {}", dcore.getName(), t.getMessage(), t);
+      coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
+      throw t;
     }
 
   }

solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -519,41 +519,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     return newInstance(name, expectedType, empty);
   }
 
-  public <T> T newInstance(String cname, Class<T> expectedType, String ... subpackages) {
+  private static final Class[] NO_CLASSES = new Class[0];
-    Class<? extends T> clazz = findClass(cname, expectedType, subpackages);
+  private static final Object[] NO_OBJECTS = new Object[0];
-    if( clazz == null ) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Can not find class: "+cname + " in " + classLoader);
-    }
-    
-    T obj = null;
-    try {
-      obj = clazz.newInstance();
-    } 
-    catch (Exception e) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Error instantiating class: '" + clazz.getName()+"'", e);
-    }
 
-    if (!live) {
+  public <T> T newInstance(String cname, Class<T> expectedType, String ... subpackages) {
-      if( obj instanceof SolrCoreAware ) {
+    return newInstance(cname, expectedType, subpackages, NO_CLASSES, NO_OBJECTS);
-        assertAwareCompatibility( SolrCoreAware.class, obj );
-        waitingForCore.add( (SolrCoreAware)obj );
-      }
-      if (org.apache.solr.util.plugin.ResourceLoaderAware.class.isInstance(obj)) {
-        log.warn("Class [{}] uses org.apache.solr.util.plugin.ResourceLoaderAware " +
-            "which is deprecated. Change to org.apache.lucene.analysis.util.ResourceLoaderAware.", cname);
-      }
-      if( obj instanceof ResourceLoaderAware ) {
-        assertAwareCompatibility( ResourceLoaderAware.class, obj );
-        waitingForResources.add( (ResourceLoaderAware)obj );
-      }
-      if (obj instanceof SolrInfoMBean){
-        //TODO: Assert here?
-        infoMBeans.add((SolrInfoMBean) obj);
-      }
-    }
-    return obj;
   }
 
   public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String ... subpackages) {
@@ -603,8 +573,13 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
 
       Constructor<? extends T> constructor = clazz.getConstructor(params);
       obj = constructor.newInstance(args);
-    }
+
-    catch (Exception e) {
+    } catch (Error err) {
+      log.error("Loading Class " + cName + " ("+clazz.getName() + ") triggered serious java error: "
+                + err.getClass().getName(), err);
+      throw err;
+
+    } catch (Exception e) {
       throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
           "Error instantiating class: '" + clazz.getName()+"'", e);
     }

solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+
+  <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+  <requestHandler name="my_error_handler" class="solr.ThrowErrorOnInitRequestHandler"></requestHandler>
+
+</config>

solr/core/src/test-files/solr/collection1/conf/bad-schema-init-error.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="bad-schema-throws-java-error" version="1.5">
+  <types>
+    <fieldType name="error_ft" class="solr.ThrowErrorOnInitFieldType" />
+ </types>
+
+
+ <fields>
+   <field name="id" type="error_ft" />
+ </fields>
+
+</schema>

solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java

@@ -315,6 +315,103 @@ public class CoreContainerCoreInitFailuresTest extends SolrTestCaseJ4 {
 
   }
 
+  public void testJavaLangErrorFromHandlerOnStartup() throws Exception {
+
+    // reused state
+    Map<String,CoreContainer.CoreLoadFailure> failures = null;
+    Collection<String> cores = null;
+    Exception fail = null;
+
+    init("java_lang_error_handler");
+
+    // start with two collections: 1 ok, and 1 that throws java.lang.Error on startup
+    File solrXml = new File(solrHome, "solr.xml");
+    FileUtils.write(solrXml, BAD_SOLR_XML, IOUtils.UTF_8);
+
+    // our "ok" collection
+    FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-defaults.xml"),
+                       FileUtils.getFile(solrHome, "col_ok", "conf", "solrconfig.xml"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/schema-minimal.xml"),
+                       FileUtils.getFile(solrHome, "col_ok", "conf", "schema.xml"));
+
+    // our "bad" collection
+    ignoreException(Pattern.quote("my_error_handler"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/bad-error-solrconfig.xml"),
+                       FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/schema-minimal.xml"),
+                       FileUtils.getFile(solrHome, "col_bad", "conf", "schema.xml"));
+
+
+    // -----
+    // init the  CoreContainer with the mix of ok/bad cores
+    cc = new CoreContainer(solrHome.getAbsolutePath());
+    cc.load();
+    
+    // check that we have the cores we expect
+    cores = cc.getCoreNames();
+    assertNotNull("core names is null", cores);
+    assertEquals("wrong number of cores", 1, cores.size());
+    assertTrue("col_ok not found", cores.contains("col_ok"));
+    
+    // check that we have the failures we expect
+    failures = cc.getCoreInitFailures();
+    assertNotNull("core failures is a null map", failures);
+    assertEquals("wrong number of core failures", 1, failures.size());
+    fail = failures.get("col_bad").exception;
+    assertNotNull("null failure for test core", fail);
+    assertTrue("init failure doesn't mention root problem: " + fail.getMessage(),
+               0 < fail.getMessage().indexOf("throwing a java.lang.Error"));
+  }
+
+  public void testJavaLangErrorFromSchemaOnStartup() throws Exception {
+
+    // reused state
+    Map<String,CoreContainer.CoreLoadFailure> failures = null;
+    Collection<String> cores = null;
+    Exception fail = null;
+
+    init("java_lang_error_schema");
+
+    // start with two collections: 1 ok, and 1 that throws java.lang.Error on startup
+    File solrXml = new File(solrHome, "solr.xml");
+    FileUtils.write(solrXml, BAD_SOLR_XML, IOUtils.UTF_8);
+
+    // our "ok" collection
+    FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-defaults.xml"),
+                       FileUtils.getFile(solrHome, "col_ok", "conf", "solrconfig.xml"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/schema-minimal.xml"),
+                       FileUtils.getFile(solrHome, "col_ok", "conf", "schema.xml"));
+
+    // our "bad" collection
+    ignoreException(Pattern.quote("error_ft"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-defaults.xml"),
+                       FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"));
+    FileUtils.copyFile(getFile("solr/collection1/conf/bad-schema-init-error.xml"),
+                       FileUtils.getFile(solrHome, "col_bad", "conf", "schema.xml"));
+
+
+    // -----
+    // init the  CoreContainer with the mix of ok/bad cores
+    cc = new CoreContainer(solrHome.getAbsolutePath());
+    cc.load();
+    
+    // check that we have the cores we expect
+    cores = cc.getCoreNames();
+    assertNotNull("core names is null", cores);
+    assertEquals("wrong number of cores", 1, cores.size());
+    assertTrue("col_ok not found", cores.contains("col_ok"));
+    
+    // check that we have the failures we expect
+    failures = cc.getCoreInitFailures();
+    assertNotNull("core failures is a null map", failures);
+    assertEquals("wrong number of core failures", 1, failures.size());
+    fail = failures.get("col_bad").exception;
+    assertNotNull("null failure for test core", fail);
+    assertTrue("init failure doesn't mention root problem: " + fail.getMessage(),
+               0 < fail.getMessage().indexOf("throwing java.lang.Error"));
+
+  }
+
   private long getCoreStartTime(final CoreContainer cc, final String name) {
     try (SolrCore tmp = cc.getCore(name)) {
       return tmp.getStartTime();

solr/core/src/test/org/apache/solr/handler/ThrowErrorOnInitRequestHandler.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import java.io.IOException;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.SolrQueryResponse;
+
+/**
+ * throws a {@link java.lang.Error} on init for testing purposes
+ */
+public class ThrowErrorOnInitRequestHandler extends RequestHandlerBase
+{
+  @Override
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException 
+  {
+    /* NOOP */
+  }
+
+  //////////////////////// SolrInfoMBeans methods //////////////////////
+
+  @Override
+  public String getDescription() {
+    return "throws a java.lang.Error on init for testing purposes";
+  }
+
+  @Override
+  public void init(NamedList args) {
+    throw new Error("Doing my job, throwing a java.lang.Error");
+  }
+}

solr/core/src/test/org/apache/solr/schema/ThrowErrorOnInitFieldType.java

@@ -0,0 +1,30 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Map;
+
+/**
+ * throws an {@link java.lang.Error} on init for testing purposes
+ */
+public class ThrowErrorOnInitFieldType extends TextField {
+
+  protected void init(IndexSchema schema, Map<String,String> args) {
+    throw new Error("Doing my job, throwing java.lang.Error");
+  }
+}