SOLR-6801 Load RequestHandler from blob store

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1646964 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -242,6 +242,8 @@ New Features
 
 * SOLR-6787: API to manage blobs in Solr (Noble Paul)
 
+* SOLR-6801:  Load RequestHandler from blob store (Noble Paul)
+
 Bug Fixes
 ----------------------
 

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -102,6 +102,7 @@ public class CoreContainer {
   protected final CoresLocator coresLocator;
   
   private String hostName;
+  private final JarRepository jarRepository = new JarRepository(this);
   
   private Map<String ,SolrRequestHandler> containerHandlers = new HashMap<>();
 
@@ -790,6 +791,10 @@ public class CoreContainer {
     return core;
   }
 
+  public JarRepository getJarRepository(){
+    return jarRepository;
+  }
+
   // ---------------- CoreContainer request handlers --------------
 
   protected <T> T createHandler(String handlerClass, Class<T> clazz) {

solr/core/src/java/org/apache/solr/core/JarRepository.java

@@ -0,0 +1,176 @@
+package org.apache.solr.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.handler.BlobHandler;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.util.SimplePostTool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+
+/**The purpose of this class is to store the Jars loaded in memory and to keep
+ * only one copy of the Jar in a single node.
+ */
+public class JarRepository {
+  public static Logger log = LoggerFactory.getLogger(JarRepository.class);
+
+  private final CoreContainer coreContainer;
+
+  private Map<String, JarContent> jars = new ConcurrentHashMap<>();
+
+  public JarRepository(CoreContainer coreContainer) {
+    this.coreContainer = coreContainer;
+  }
+
+  /**Returns the contents of a jar and increments a reference count. Please return the same
+   * object to decerease the refcount
+   * @param key it is a combination of blobname and version like blobName/version
+   * @return The reference of a jar
+   */
+  public JarContentRef getJarIncRef(String key) throws IOException {
+    JarContent jar = jars.get(key);
+    if(jar ==null){
+      if(this.coreContainer.isZooKeeperAware()){
+        ClusterState cs = this.coreContainer.getZkController().getZkStateReader().getClusterState();
+        DocCollection coll = cs.getCollectionOrNull(CollectionsHandler.SYSTEM_COLL);
+        if(coll == null) throw new SolrException(SERVICE_UNAVAILABLE,
+            ".system collection not available");
+        Slice slice = coll.getActiveSlices().iterator().next();
+        if(slice == null) throw new SolrException(SERVICE_UNAVAILABLE,
+            ".no active slices for .system collection");
+        Replica replica = slice.getReplicas().iterator().next();
+        if(replica == null) throw new SolrException(SERVICE_UNAVAILABLE,
+            ".no active replica available for .system collection");
+        String url = replica.getStr(BASE_URL_PROP) + "/.system/blob/"+ key+"?wt=filestream";
+
+        HttpClient httpClient = coreContainer.getUpdateShardHandler().getHttpClient();
+        HttpGet httpGet = new HttpGet(url);
+        ByteBuffer b;
+        try {
+          HttpResponse entity =  httpClient.execute(httpGet);
+          int statusCode = entity.getStatusLine().getStatusCode();
+          if(statusCode != 200){
+            throw new SolrException(SolrException.ErrorCode.NOT_FOUND,"no such blob or version available: "+ key);
+          }
+          b = SimplePostTool.inputStreamToByteArray(entity.getEntity().getContent());
+        } finally {
+          httpGet.releaseConnection();
+        }
+        jars.put(key,jar = new JarContent(key,b));
+      } else {
+
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"Jar loading is not supported in non-cloud mode");
+        //todo
+
+      }
+
+    }
+
+    JarContentRef ref = new JarContentRef(jar);
+    synchronized (jar.references) {
+      jar.references.add(ref);
+    }
+    return ref;
+
+  }
+
+
+  /**This is to decrement a ref count
+   * @param ref The reference that is already there. Doing multiple calls with same ref will not matter
+   */
+  public void decrementJarRefCount(JarContentRef ref){
+    if(ref == null) return;
+    synchronized (ref.jar.references) {
+      if (!ref.jar.references.remove(ref)) {
+        log.error("Multiple releases for the same reference");
+      }
+      if (ref.jar.references.isEmpty()) {
+        jars.remove(ref.jar.key);
+      }
+    }
+
+  }
+
+
+  public static class JarContent {
+    private final String key;
+    //TODO move this off-heap
+    private final ByteBuffer buffer;
+    // ref counting mechanism
+    private final Set<JarContentRef> references = new HashSet<>();
+
+    public JarContent(String key, ByteBuffer buffer) {
+      this.key = key;
+      this.buffer = buffer;
+    }
+
+    public ByteBuffer getFileContent(String entryName) throws IOException {
+      ByteArrayInputStream zipContents=new ByteArrayInputStream(buffer.array(),buffer.arrayOffset(),buffer.limit());
+      ZipInputStream zis=new ZipInputStream(zipContents);
+      try {
+        ZipEntry entry;
+        while ((entry=zis.getNextEntry()) != null) {
+          if (entryName == null || entryName.equals(entry.getName())) {
+            SimplePostTool.BAOS out=new SimplePostTool.BAOS();
+            byte[] buffer=new byte[2048];
+            int size;
+            while ((size=zis.read(buffer,0,buffer.length)) != -1) {
+              out.write(buffer,0,size);
+            }
+            out.close();
+            return out.getByteBuffer();
+          }
+        }
+      } finally {
+        zis.closeEntry();
+      }
+      return null;
+    }
+
+  }
+
+  public static class JarContentRef  {
+    public final JarContent jar;
+    private JarContentRef(JarContent jar) {
+      this.jar = jar;
+    }
+  }
+
+}

solr/core/src/java/org/apache/solr/core/RequestHandlers.java

@@ -17,7 +17,19 @@
 
 package org.apache.solr.core;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.ByteBuffer;
+import java.security.BasicPermission;
+import java.security.CodeSource;
+import java.security.Permissions;
+import java.security.ProtectionDomain;
+import java.security.SecureClassLoader;
+import java.security.cert.Certificate;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -25,13 +37,17 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.jar.JarFile;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.util.plugin.PluginInfoInitialized;
@@ -51,6 +67,8 @@ public final class RequestHandlers {
       new ConcurrentHashMap<>() ;
   private final Map<String, SolrRequestHandler> immutableHandlers = Collections.unmodifiableMap(handlers) ;
 
+  public static final boolean disableExternalLib = Boolean.parseBoolean(System.getProperty("disable.external.lib", "false"));
+
   /**
    * Trim the trailing '/' if its there, and convert null to empty string.
    * 
@@ -157,16 +175,21 @@ public final class RequestHandlers {
       try {
         SolrRequestHandler requestHandler;
         String startup = info.attributes.get("startup") ;
-        if( startup != null ) {
+        String lib = info.attributes.get("lib");
+        if(lib != null){
+          requestHandler = new DynamicLazyRequestHandlerWrapper(core);
+        } else if( startup != null ) {
           if( "lazy".equals(startup) ) {
             log.info("adding lazy requestHandler: " + info.className);
-            requestHandler = new LazyRequestHandlerWrapper( core, info.className);
+            requestHandler = new LazyRequestHandlerWrapper( core);
           } else {
             throw new Exception( "Unknown startup value: '"+startup+"' for: "+info.className );
           }
         } else {
           requestHandler = core.createRequestHandler(info.className);
         }
+        if (requestHandler instanceof RequestHandlerBase) ((RequestHandlerBase) requestHandler).setPluginInfo(info);
+
         handlers.put(info,requestHandler);
         SolrRequestHandler old = register(info.name, requestHandler);
         if(old != null) {
@@ -242,25 +265,22 @@ public final class RequestHandlers {
    * 
    * @since solr 1.2
    */
-  public static final class LazyRequestHandlerWrapper implements SolrRequestHandler , PluginInfoInitialized
+  public static class LazyRequestHandlerWrapper implements SolrRequestHandler , AutoCloseable, PluginInfoInitialized
   {
     private final SolrCore core;
-    private String _className;
+    String _className;
-    private SolrRequestHandler _handler;
+    SolrRequestHandler _handler;
-    private PluginInfo _pluginInfo;
+    PluginInfo _pluginInfo;
     
-    public LazyRequestHandlerWrapper( SolrCore core, String className )
+    public LazyRequestHandlerWrapper( SolrCore core)
     {
       this.core = core;
-      _className = className;
       _handler = null; // don't initialize
     }
 
     @Override
     public void init(NamedList args) { }
 
-
-
     /**
      * Wait for the first request before initializing the wrapped handler 
      */
@@ -277,7 +297,12 @@ public final class RequestHandlers {
     {
       if( _handler == null ) {
         try {
-          SolrRequestHandler handler = core.createRequestHandler(_className);
+          SolrRequestHandler handler = createRequestHandler();
+          if (handler instanceof PluginInfoInitialized) {
+            ((PluginInfoInitialized) handler).init(_pluginInfo);
+          } else {
+            handler.init( _pluginInfo.initArgs );
+          }
 
           if (handler instanceof PluginInfoInitialized) {
             ((PluginInfoInitialized) handler).init(_pluginInfo);
@@ -298,6 +323,10 @@ public final class RequestHandlers {
       return _handler;
     }
 
+    protected SolrRequestHandler createRequestHandler() {
+      return core.createRequestHandler(_className);
+    }
+
     public String getHandlerClass()
     {
       return _className;
@@ -356,11 +385,191 @@ public final class RequestHandlers {
       return lst;
     }
 
+    @Override
+    public void close() throws Exception {
+      if(_handler == null) return;
+      if (_handler instanceof AutoCloseable && !(_handler instanceof DynamicLazyRequestHandlerWrapper) ) {
+        ((AutoCloseable) _handler).close();
+      }
+    }
+
     @Override
     public void init(PluginInfo info) {
       _pluginInfo = info;
+      _className = info.className;
+    }
+  }
+
+  public static class DynamicLazyRequestHandlerWrapper extends LazyRequestHandlerWrapper {
+    private String lib;
+    private String key;
+    private String version;
+    private CoreContainer coreContainer;
+    private SolrResourceLoader solrResourceLoader;
+    private MemClassLoader classLoader;
+    private boolean _closed = false;
+    boolean unrecoverable = false;
+    String errMsg = null;
+    private Exception exception;
+
+
+    public DynamicLazyRequestHandlerWrapper(SolrCore core) {
+      super(core);
+      this.coreContainer = core.getCoreDescriptor().getCoreContainer();
+      this.solrResourceLoader = core.getResourceLoader();
 
     }
+
+    @Override
+    public void init(PluginInfo info) {
+      super.init(info);
+      this.lib = _pluginInfo.attributes.get("lib");
+
+      if(disableExternalLib){
+        errMsg = "ERROR external library loading is disabled";
+        unrecoverable = true;
+        _handler = this;
+        log.error(errMsg);
+        return;
+      }
+
+      version = _pluginInfo.attributes.get("version");
+      if (version == null) {
+        errMsg = "ERROR 'lib' attribute must be accompanied with version also";
+        unrecoverable = true;
+        _handler = this;
+        log.error(errMsg);
+        return;
+      }
+      classLoader = new MemClassLoader(this);
+    }
+
+    @Override
+    public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
+      if(unrecoverable) {
+        rsp.add("error", errMsg);
+        if(exception != null) rsp.setException(exception);
+        return;
+      }
+      try {
+        classLoader.checkJarAvailable();
+      } catch (SolrException e) {
+        rsp.add("error", "Jar could not be loaded");
+        rsp.setException(e);
+        return;
+      } catch (IOException e) {
+        unrecoverable = true;
+        errMsg = "Could not load jar";
+        exception = e;
+        handleRequest(req,rsp);
+        return;
+      }
+
+      super.handleRequest(req, rsp);
+    }
+
+    @Override
+    protected SolrRequestHandler createRequestHandler() {
+      try {
+        Class clazz =  classLoader.findClass(_className);
+        Constructor<?>[] cons =  clazz.getConstructors();
+        for (Constructor<?> con : cons) {
+          Class<?>[] types = con.getParameterTypes();
+          if(types.length == 1 && types[0] == SolrCore.class){
+            return SolrRequestHandler.class.cast(con.newInstance(this));
+          }
+        }
+        return (SolrRequestHandler)clazz.newInstance();
+      } catch (Exception e) {
+        unrecoverable = true;
+        errMsg = MessageFormat.format("class {0} could not be loaded ",_className);
+        this.exception = e;
+        return this;
+
+      }
+
+    }
+
+    @Override
+    public void close() throws Exception {
+      super.close();
+      if(_closed) return;
+      classLoader.releaseJar();
+      _closed = true;
+    }
+  }
+
+
+  public static class MemClassLoader extends ClassLoader {
+    private JarRepository.JarContentRef jarContent;
+    private final DynamicLazyRequestHandlerWrapper handlerWrapper;
+    public MemClassLoader(DynamicLazyRequestHandlerWrapper handlerWrapper) {
+      super(handlerWrapper.solrResourceLoader.classLoader);
+      this.handlerWrapper = handlerWrapper;
+
+    }
+
+    boolean checkJarAvailable() throws IOException {
+      if (jarContent != null) return true;
+
+      try {
+        synchronized (this) {
+          jarContent = handlerWrapper.coreContainer.getJarRepository().getJarIncRef(handlerWrapper.lib+"/"+handlerWrapper.version);
+          return true;
+        }
+      } catch (SolrException se) {
+        throw se;
+      }
+
+    }
+
+    @Override
+    protected Class<?> findClass(String name) throws ClassNotFoundException {
+      try {
+        return super.findClass(name);
+      } catch (ClassNotFoundException e) {
+        String path = name.replace('.', '/').concat(".class");
+        ByteBuffer buf = null;
+        try {
+          if(jarContent == null) checkJarAvailable();
+          buf = jarContent.jar.getFileContent(path);
+        } catch (IOException e1) {
+          throw new ClassNotFoundException("class not found "+ name ,e1) ;
+
+        }
+
+        ProtectionDomain defaultDomain = null;
+
+        //using the default protection domain, with no permissions
+        try {
+          defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/.system/blob/"+handlerWrapper.lib), (Certificate[]) null),
+              null);
+        } catch (MalformedURLException e1) {
+          //should not happen
+        }
+        return defineClass(name,buf.array(),buf.arrayOffset(),buf.limit(), defaultDomain);
+      }
+    }
+
+
+    private void releaseJar(){
+      handlerWrapper.coreContainer.getJarRepository().decrementJarRefCount(jarContent);
+    }
+
+  }
+
+  public void close() {
+    for (Map.Entry<String, SolrRequestHandler> e : handlers.entrySet()) {
+      if (e.getValue() instanceof AutoCloseable) {
+        try {
+          ((AutoCloseable) e.getValue()).close();
+        } catch (Exception exp) {
+          log.error("Error closing requestHandler "+e.getKey() , exp);
+        }
+      }
+
+    }
+
   }
 }
 

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -1069,6 +1069,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
       }
     }
 
+    if(reqHandlers != null) reqHandlers.close();
 
 
     try {

solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java

@@ -21,6 +21,7 @@ import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.RequestHandlers;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrInfoMBean;
@@ -54,6 +55,7 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
   private final AtomicLong numTimeouts = new AtomicLong();
   private final Timer requestTimes = new Timer();
   private final long handlerStart = System.currentTimeMillis();
+  private PluginInfo pluginInfo;
 
   /**
    * Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
@@ -233,6 +235,18 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
     return handler;
   }
 
+  /**
+   *
+   * @param pluginInfo
+   */
+  public void setPluginInfo(PluginInfo pluginInfo){
+    if(pluginInfo==null) this.pluginInfo = pluginInfo;
+  }
+
+  public PluginInfo getPluginInfo(){
+    return  pluginInfo;
+  }
+
 
   @Override
   public NamedList<Object> getStatistics() {

solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java

@@ -121,7 +121,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
   @Override
   public void init(FilterConfig config) throws ServletException
   {
-    log.info("SolrDispatchFilter.init()");
+    log.info("SolrDispatchFilter.init()" + this.getClass().getClassLoader());
 
     try {
       // web.xml configuration

solr/core/src/test/org/apache/solr/core/BlobStoreTestRequestHandler.java

@@ -0,0 +1,33 @@
+package org.apache.solr.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+
+import org.apache.solr.handler.DumpRequestHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+
+public class BlobStoreTestRequestHandler extends DumpRequestHandler{
+  @Override
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+    super.handleRequestBody(req, rsp);
+    rsp.add("classloader", this.getClass().getClassLoader().getClass().getName());
+  }
+}

solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java

@@ -0,0 +1,125 @@
+package org.apache.solr.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.handler.TestBlobHandler;
+import org.apache.solr.util.RESTfulServerProvider;
+import org.apache.solr.util.RestTestHarness;
+import org.apache.solr.util.SimplePostTool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
+  static final Logger log =  LoggerFactory.getLogger(TestDynamicLoading.class);
+  private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
+
+  private void setupHarnesses() {
+    for (final SolrServer client : clients) {
+      RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
+        @Override
+        public String getBaseURL() {
+          return ((HttpSolrServer)client).getBaseURL();
+        }
+      });
+      restTestHarnesses.add(harness);
+    }
+  }
+
+
+
+  @Override
+  public void doTest() throws Exception {
+
+   setupHarnesses();
+   dynamicLoading();
+
+
+
+  }
+
+  private void dynamicLoading() throws Exception {
+    String payload = "{\n" +
+        "'create-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandler' , 'lib':'test','version':'1'}\n" +
+        "}";
+    RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+    TestSolrConfigHandler.runConfigCommand(client,"/config?wt=json",payload);
+    TestSolrConfigHandler.testForResponseElement(client,
+        null,
+        "/config/overlay?wt=json",
+        null,
+        Arrays.asList("overlay", "requestHandler", "/test1", "lib"),
+        "test",10);
+
+    Map map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
+
+    assertNotNull(map = (Map) map.get("error"));
+    assertEquals(".system collection not available", map.get("msg"));
+
+    HttpSolrServer server = (HttpSolrServer) clients.get(random().nextInt(clients.size()));
+    String baseURL = server.getBaseURL();
+    baseURL = baseURL.substring(0, baseURL.lastIndexOf('/'));
+    TestBlobHandler.createSysColl(new HttpSolrServer(baseURL,server.getHttpClient()));
+    map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
+
+    assertNotNull(map = (Map) map.get("error"));
+    assertEquals("no such blob or version available: test/1", map.get("msg"));
+    ByteBuffer jar = generateZip( TestDynamicLoading.class,BlobStoreTestRequestHandler.class);
+    TestBlobHandler.postAndCheck(cloudClient, baseURL, jar,1);
+
+//    Thread.sleep(100);
+    map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
+
+    assertEquals(RequestHandlers.MemClassLoader.class.getName(), map.get("classloader"));
+
+
+  }
+
+
+  public static ByteBuffer generateZip(Class... classes) throws IOException {
+    ZipOutputStream zipOut = null;
+    SimplePostTool.BAOS bos = new SimplePostTool.BAOS();
+    zipOut = new ZipOutputStream(bos);
+    zipOut.setLevel(ZipOutputStream.DEFLATED);
+    for (Class c : classes) {
+      String path = c.getName().replace('.', '/').concat(".class");
+      ZipEntry entry = new ZipEntry(path);
+      ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path));
+      zipOut.putNextEntry(entry);
+      zipOut.write(b.array(), 0, b.limit());
+      zipOut.closeEntry();
+    }
+    zipOut.close();
+    return bos.getByteBuffer();
+  }
+
+
+
+}

solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java

@@ -21,22 +21,19 @@ package org.apache.solr.core;
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.io.FileUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.util.EntityUtils;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.impl.CloudSolrServer;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
 import org.apache.solr.util.RestTestBase;
 import org.apache.solr.util.RestTestHarness;
@@ -142,7 +139,7 @@ public class TestSolrConfigHandler extends RestTestBase {
     reqhandlertests(restTestHarness, null,null);
   }
 
-  private static void runConfigCommand(RestTestHarness harness, String uri,  String payload) throws IOException {
+  public static void runConfigCommand(RestTestHarness harness, String uri,  String payload) throws IOException {
     String response = harness.post(uri, SolrTestCaseJ4.json(payload));
     Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
     assertNull(response,  map.get("errors"));
@@ -155,53 +152,34 @@ public class TestSolrConfigHandler extends RestTestBase {
         "}";
     runConfigCommand(writeHarness,"/config?wt=json", payload);
 
-    boolean success = false;
+    testForResponseElement(writeHarness,
-    long startTime = System.nanoTime();
+        testServerBaseUrl,
-    long maxTimeoutSeconds = 10;
+        "/config/overlay?wt=json",
-    while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+        cloudSolrServer,
-      String uri = "/config/overlay?wt=json";
+        Arrays.asList("overlay", "requestHandler", "/x", "startup"),
-      Map m = testServerBaseUrl ==null?  getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
+        "lazy",
-      if("lazy".equals( ConfigOverlay.getObjectByPath(m,  true, Arrays.asList("overlay", "requestHandler", "/x","startup")))) {
+        10);
-        Map map = getRespMap("/x?wt=json",writeHarness);
-        if(map.containsKey("params")) {
-          success = true;
-          break;
-        }
-      }
-      Thread.sleep(100);
-
-    }
-
-    assertTrue( "Could not register requestHandler  ", success);
 
     payload = "{\n" +
         "'update-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy' , 'a':'b'}\n" +
         "}";
     runConfigCommand(writeHarness,"/config?wt=json", payload);
 
-    success = false;
+    testForResponseElement(writeHarness,
-    startTime = System.nanoTime();
+        testServerBaseUrl,
-    maxTimeoutSeconds = 10;
+        "/config/overlay?wt=json",
-    while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+        cloudSolrServer,
-      String uri = "/config/overlay?wt=json";
+        Arrays.asList("overlay", "requestHandler", "/x", "a"),
-      Map m = testServerBaseUrl ==null?  getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
+        "b",
-      if("b".equals( ConfigOverlay.getObjectByPath(m,  true, Arrays.asList("overlay", "requestHandler", "/x","a")))) {
+        10);
-          success = true;
-          break;
-      }
-      Thread.sleep(100);
-
-    }
-
-    assertTrue( "Could not update requestHandler  ", success);
 
     payload = "{\n" +
         "'delete-requesthandler' : '/x'" +
         "}";
     runConfigCommand(writeHarness,"/config?wt=json", payload);
-    success = false;
+    boolean success = false;
-    startTime = System.nanoTime();
+    long startTime = System.nanoTime();
-    maxTimeoutSeconds = 10;
+    int maxTimeoutSeconds = 10;
     while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
       String uri = "/config/overlay?wt=json";
       Map m = testServerBaseUrl ==null?  getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
@@ -216,6 +194,33 @@ public class TestSolrConfigHandler extends RestTestBase {
 
   }
 
+  public static void testForResponseElement(RestTestHarness harness,
+                                            String testServerBaseUrl,
+                                            String uri,
+                                            CloudSolrServer cloudSolrServer,List<String> jsonPath,
+                                            String expected,
+                                            long maxTimeoutSeconds ) throws Exception {
+
+    boolean success = false;
+    long startTime = System.nanoTime();
+    while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
+      Map m = testServerBaseUrl ==null?  getRespMap(uri,harness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl + uri, cloudSolrServer) ;
+      if(Objects.equals(expected,ConfigOverlay.getObjectByPath(m, true, jsonPath))) {
+        success = true;
+        break;
+        /*Map map = getRespMap("/x?wt=json",harness);
+        if(map.containsKey("params")) {
+          success = true;
+          break;
+        }*/
+      }
+      Thread.sleep(100);
+
+    }
+
+    assertTrue( "Could not add/change requestHandler  ", success);
+  }
+
 
   public static Map getRespMap(String path, RestTestHarness restHarness) throws Exception {
     String response = restHarness.query(path);

solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java

@@ -17,13 +17,7 @@ package org.apache.solr.handler;
  * limitations under the License.
  */
 
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -31,7 +25,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.io.Closeables;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -41,6 +34,7 @@ import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.util.EntityUtils;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.response.CollectionAdminResponse;
@@ -57,7 +51,7 @@ import org.slf4j.LoggerFactory;
 import static org.apache.solr.core.ConfigOverlay.getObjectByPath;
 
 public class TestBlobHandler extends AbstractFullDistribZkTestBase {
-  static final Logger log =  LoggerFactory.getLogger(TestSolrConfigHandlerConcurrent.class);
+  static final Logger log =  LoggerFactory.getLogger(TestBlobHandler.class);
 
   private void doBlobHandlerTest() throws Exception {
     SolrServer server = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0)));
@@ -87,8 +81,8 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
     for (int i = 0; i < bytarr.length; i++) bytarr[i]= (byte) (i % 127);
     byte[] bytarr2  = new byte[2048];
     for (int i = 0; i < bytarr2.length; i++) bytarr2[i]= (byte) (i % 127);
-    postAndCheck(baseUrl, bytarr, 1);
+    postAndCheck(cloudClient, baseUrl, ByteBuffer.wrap( bytarr), 1);
-    postAndCheck(baseUrl, bytarr2, 2);
+    postAndCheck(cloudClient, baseUrl, ByteBuffer.wrap( bytarr2), 2);
 
     url = baseUrl + "/.system/blob/test/1";
     map = TestSolrConfigHandlerConcurrent.getAsMap(url,cloudClient);
@@ -102,6 +96,17 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
 
   }
 
+  public static  void createSysColl(SolrServer server) throws SolrServerException, IOException {
+    CollectionAdminResponse response1;
+    CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create();
+    createCollectionRequest.setCollectionName(".system");
+    createCollectionRequest.setNumShards(1);
+    createCollectionRequest.setReplicationFactor(2);
+    response1 = createCollectionRequest.process(server);
+    assertEquals(0, response1.getStatus());
+    assertTrue(response1.isSuccess());
+  }
+
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
@@ -112,8 +117,8 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
     DirectUpdateHandler2.commitOnClose = true;
   }
 
-  private void postAndCheck(String baseUrl, byte[] bytes, int count) throws Exception {
+  public static void postAndCheck(CloudSolrServer cloudClient, String baseUrl, ByteBuffer bytes, int count) throws Exception {
-    postData(baseUrl, bytes);
+    postData(cloudClient, baseUrl, bytes);
     String url;
     Map map;
     List l;
@@ -132,7 +137,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
       l = (List) ConfigOverlay.getObjectByPath(map, false, Arrays.asList("response", "docs"));
       assertNotNull(l);
       map = (Map) l.get(0);
-      assertEquals("" + bytes.length, String.valueOf(map.get("size")));
+      assertEquals("" + bytes.limit(), String.valueOf(map.get("size")));
       break;
     }
   }
@@ -155,14 +160,14 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
 
   }
 
-  private String postData(String baseUrl, byte[] bytarr) throws IOException {
+  public static String postData(CloudSolrServer cloudClient, String baseUrl, ByteBuffer bytarr) throws IOException {
     HttpPost httpPost = null;
     HttpEntity entity;
     String response;
     try {
       httpPost = new HttpPost(baseUrl+"/.system/blob/test");
       httpPost.setHeader("Content-Type","application/octet-stream");
-      httpPost.setEntity(new ByteArrayEntity(bytarr));
+      httpPost.setEntity(new ByteArrayEntity(bytarr.array(), bytarr.arrayOffset(), bytarr.limit()));
       entity = cloudClient.getLbServer().getHttpClient().execute(httpPost).getEntity();
       return EntityUtils.toString(entity, StandardCharsets.UTF_8);
     } finally {