mirror of https://github.com/apache/lucene.git
SOLR-6801 Load RequestHandler from blob store
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1646964 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul
parent
d4f03f72ae
commit
c2c72f5f02
|
|
@ -242,6 +242,8 @@ New Features
|
|||
|
||||
* SOLR-6787: API to manage blobs in Solr (Noble Paul)
|
||||
|
||||
* SOLR-6801: Load RequestHandler from blob store (Noble Paul)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ public class CoreContainer {
|
|||
protected final CoresLocator coresLocator;
|
||||
|
||||
private String hostName;
|
||||
private final JarRepository jarRepository = new JarRepository(this);
|
||||
|
||||
private Map<String ,SolrRequestHandler> containerHandlers = new HashMap<>();
|
||||
|
||||
|
|
@ -790,6 +791,10 @@ public class CoreContainer {
|
|||
return core;
|
||||
}
|
||||
|
||||
public JarRepository getJarRepository(){
|
||||
return jarRepository;
|
||||
}
|
||||
|
||||
// ---------------- CoreContainer request handlers --------------
|
||||
|
||||
protected <T> T createHandler(String handlerClass, Class<T> clazz) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
package org.apache.solr.core;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.handler.BlobHandler;
|
||||
import org.apache.solr.handler.admin.CollectionsHandler;
|
||||
import org.apache.solr.util.SimplePostTool;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
|
||||
|
||||
/**The purpose of this class is to store the Jars loaded in memory and to keep
|
||||
* only one copy of the Jar in a single node.
|
||||
*/
|
||||
public class JarRepository {
|
||||
public static Logger log = LoggerFactory.getLogger(JarRepository.class);
|
||||
|
||||
private final CoreContainer coreContainer;
|
||||
|
||||
private Map<String, JarContent> jars = new ConcurrentHashMap<>();
|
||||
|
||||
public JarRepository(CoreContainer coreContainer) {
|
||||
this.coreContainer = coreContainer;
|
||||
}
|
||||
|
||||
/**Returns the contents of a jar and increments a reference count. Please return the same
|
||||
* object to decerease the refcount
|
||||
* @param key it is a combination of blobname and version like blobName/version
|
||||
* @return The reference of a jar
|
||||
*/
|
||||
public JarContentRef getJarIncRef(String key) throws IOException {
|
||||
JarContent jar = jars.get(key);
|
||||
if(jar ==null){
|
||||
if(this.coreContainer.isZooKeeperAware()){
|
||||
ClusterState cs = this.coreContainer.getZkController().getZkStateReader().getClusterState();
|
||||
DocCollection coll = cs.getCollectionOrNull(CollectionsHandler.SYSTEM_COLL);
|
||||
if(coll == null) throw new SolrException(SERVICE_UNAVAILABLE,
|
||||
".system collection not available");
|
||||
Slice slice = coll.getActiveSlices().iterator().next();
|
||||
if(slice == null) throw new SolrException(SERVICE_UNAVAILABLE,
|
||||
".no active slices for .system collection");
|
||||
Replica replica = slice.getReplicas().iterator().next();
|
||||
if(replica == null) throw new SolrException(SERVICE_UNAVAILABLE,
|
||||
".no active replica available for .system collection");
|
||||
String url = replica.getStr(BASE_URL_PROP) + "/.system/blob/"+ key+"?wt=filestream";
|
||||
|
||||
HttpClient httpClient = coreContainer.getUpdateShardHandler().getHttpClient();
|
||||
HttpGet httpGet = new HttpGet(url);
|
||||
ByteBuffer b;
|
||||
try {
|
||||
HttpResponse entity = httpClient.execute(httpGet);
|
||||
int statusCode = entity.getStatusLine().getStatusCode();
|
||||
if(statusCode != 200){
|
||||
throw new SolrException(SolrException.ErrorCode.NOT_FOUND,"no such blob or version available: "+ key);
|
||||
}
|
||||
b = SimplePostTool.inputStreamToByteArray(entity.getEntity().getContent());
|
||||
} finally {
|
||||
httpGet.releaseConnection();
|
||||
}
|
||||
jars.put(key,jar = new JarContent(key,b));
|
||||
} else {
|
||||
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"Jar loading is not supported in non-cloud mode");
|
||||
//todo
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
JarContentRef ref = new JarContentRef(jar);
|
||||
synchronized (jar.references) {
|
||||
jar.references.add(ref);
|
||||
}
|
||||
return ref;
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**This is to decrement a ref count
|
||||
* @param ref The reference that is already there. Doing multiple calls with same ref will not matter
|
||||
*/
|
||||
public void decrementJarRefCount(JarContentRef ref){
|
||||
if(ref == null) return;
|
||||
synchronized (ref.jar.references) {
|
||||
if (!ref.jar.references.remove(ref)) {
|
||||
log.error("Multiple releases for the same reference");
|
||||
}
|
||||
if (ref.jar.references.isEmpty()) {
|
||||
jars.remove(ref.jar.key);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static class JarContent {
|
||||
private final String key;
|
||||
//TODO move this off-heap
|
||||
private final ByteBuffer buffer;
|
||||
// ref counting mechanism
|
||||
private final Set<JarContentRef> references = new HashSet<>();
|
||||
|
||||
public JarContent(String key, ByteBuffer buffer) {
|
||||
this.key = key;
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
public ByteBuffer getFileContent(String entryName) throws IOException {
|
||||
ByteArrayInputStream zipContents=new ByteArrayInputStream(buffer.array(),buffer.arrayOffset(),buffer.limit());
|
||||
ZipInputStream zis=new ZipInputStream(zipContents);
|
||||
try {
|
||||
ZipEntry entry;
|
||||
while ((entry=zis.getNextEntry()) != null) {
|
||||
if (entryName == null || entryName.equals(entry.getName())) {
|
||||
SimplePostTool.BAOS out=new SimplePostTool.BAOS();
|
||||
byte[] buffer=new byte[2048];
|
||||
int size;
|
||||
while ((size=zis.read(buffer,0,buffer.length)) != -1) {
|
||||
out.write(buffer,0,size);
|
||||
}
|
||||
out.close();
|
||||
return out.getByteBuffer();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
zis.closeEntry();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class JarContentRef {
|
||||
public final JarContent jar;
|
||||
private JarContentRef(JarContent jar) {
|
||||
this.jar = jar;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,7 +17,19 @@
|
|||
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.security.BasicPermission;
|
||||
import java.security.CodeSource;
|
||||
import java.security.Permissions;
|
||||
import java.security.ProtectionDomain;
|
||||
import java.security.SecureClassLoader;
|
||||
import java.security.cert.Certificate;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -25,13 +37,17 @@ import java.util.LinkedHashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.jar.JarFile;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.handler.RequestHandlerBase;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequestBase;
|
||||
import org.apache.solr.request.SolrRequestHandler;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.util.plugin.PluginInfoInitialized;
|
||||
|
|
@ -51,6 +67,8 @@ public final class RequestHandlers {
|
|||
new ConcurrentHashMap<>() ;
|
||||
private final Map<String, SolrRequestHandler> immutableHandlers = Collections.unmodifiableMap(handlers) ;
|
||||
|
||||
public static final boolean disableExternalLib = Boolean.parseBoolean(System.getProperty("disable.external.lib", "false"));
|
||||
|
||||
/**
|
||||
* Trim the trailing '/' if its there, and convert null to empty string.
|
||||
*
|
||||
|
|
@ -157,16 +175,21 @@ public final class RequestHandlers {
|
|||
try {
|
||||
SolrRequestHandler requestHandler;
|
||||
String startup = info.attributes.get("startup") ;
|
||||
if( startup != null ) {
|
||||
String lib = info.attributes.get("lib");
|
||||
if(lib != null){
|
||||
requestHandler = new DynamicLazyRequestHandlerWrapper(core);
|
||||
} else if( startup != null ) {
|
||||
if( "lazy".equals(startup) ) {
|
||||
log.info("adding lazy requestHandler: " + info.className);
|
||||
requestHandler = new LazyRequestHandlerWrapper( core, info.className);
|
||||
requestHandler = new LazyRequestHandlerWrapper( core);
|
||||
} else {
|
||||
throw new Exception( "Unknown startup value: '"+startup+"' for: "+info.className );
|
||||
}
|
||||
} else {
|
||||
requestHandler = core.createRequestHandler(info.className);
|
||||
}
|
||||
if (requestHandler instanceof RequestHandlerBase) ((RequestHandlerBase) requestHandler).setPluginInfo(info);
|
||||
|
||||
handlers.put(info,requestHandler);
|
||||
SolrRequestHandler old = register(info.name, requestHandler);
|
||||
if(old != null) {
|
||||
|
|
@ -242,25 +265,22 @@ public final class RequestHandlers {
|
|||
*
|
||||
* @since solr 1.2
|
||||
*/
|
||||
public static final class LazyRequestHandlerWrapper implements SolrRequestHandler , PluginInfoInitialized
|
||||
public static class LazyRequestHandlerWrapper implements SolrRequestHandler , AutoCloseable, PluginInfoInitialized
|
||||
{
|
||||
private final SolrCore core;
|
||||
private String _className;
|
||||
private SolrRequestHandler _handler;
|
||||
private PluginInfo _pluginInfo;
|
||||
String _className;
|
||||
SolrRequestHandler _handler;
|
||||
PluginInfo _pluginInfo;
|
||||
|
||||
public LazyRequestHandlerWrapper( SolrCore core, String className )
|
||||
public LazyRequestHandlerWrapper( SolrCore core)
|
||||
{
|
||||
this.core = core;
|
||||
_className = className;
|
||||
_handler = null; // don't initialize
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(NamedList args) { }
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Wait for the first request before initializing the wrapped handler
|
||||
*/
|
||||
|
|
@ -277,7 +297,12 @@ public final class RequestHandlers {
|
|||
{
|
||||
if( _handler == null ) {
|
||||
try {
|
||||
SolrRequestHandler handler = core.createRequestHandler(_className);
|
||||
SolrRequestHandler handler = createRequestHandler();
|
||||
if (handler instanceof PluginInfoInitialized) {
|
||||
((PluginInfoInitialized) handler).init(_pluginInfo);
|
||||
} else {
|
||||
handler.init( _pluginInfo.initArgs );
|
||||
}
|
||||
|
||||
if (handler instanceof PluginInfoInitialized) {
|
||||
((PluginInfoInitialized) handler).init(_pluginInfo);
|
||||
|
|
@ -298,6 +323,10 @@ public final class RequestHandlers {
|
|||
return _handler;
|
||||
}
|
||||
|
||||
protected SolrRequestHandler createRequestHandler() {
|
||||
return core.createRequestHandler(_className);
|
||||
}
|
||||
|
||||
public String getHandlerClass()
|
||||
{
|
||||
return _className;
|
||||
|
|
@ -356,11 +385,191 @@ public final class RequestHandlers {
|
|||
return lst;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
if(_handler == null) return;
|
||||
if (_handler instanceof AutoCloseable && !(_handler instanceof DynamicLazyRequestHandlerWrapper) ) {
|
||||
((AutoCloseable) _handler).close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(PluginInfo info) {
|
||||
_pluginInfo = info;
|
||||
_className = info.className;
|
||||
}
|
||||
}
|
||||
|
||||
public static class DynamicLazyRequestHandlerWrapper extends LazyRequestHandlerWrapper {
|
||||
private String lib;
|
||||
private String key;
|
||||
private String version;
|
||||
private CoreContainer coreContainer;
|
||||
private SolrResourceLoader solrResourceLoader;
|
||||
private MemClassLoader classLoader;
|
||||
private boolean _closed = false;
|
||||
boolean unrecoverable = false;
|
||||
String errMsg = null;
|
||||
private Exception exception;
|
||||
|
||||
|
||||
public DynamicLazyRequestHandlerWrapper(SolrCore core) {
|
||||
super(core);
|
||||
this.coreContainer = core.getCoreDescriptor().getCoreContainer();
|
||||
this.solrResourceLoader = core.getResourceLoader();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(PluginInfo info) {
|
||||
super.init(info);
|
||||
this.lib = _pluginInfo.attributes.get("lib");
|
||||
|
||||
if(disableExternalLib){
|
||||
errMsg = "ERROR external library loading is disabled";
|
||||
unrecoverable = true;
|
||||
_handler = this;
|
||||
log.error(errMsg);
|
||||
return;
|
||||
}
|
||||
|
||||
version = _pluginInfo.attributes.get("version");
|
||||
if (version == null) {
|
||||
errMsg = "ERROR 'lib' attribute must be accompanied with version also";
|
||||
unrecoverable = true;
|
||||
_handler = this;
|
||||
log.error(errMsg);
|
||||
return;
|
||||
}
|
||||
classLoader = new MemClassLoader(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
if(unrecoverable) {
|
||||
rsp.add("error", errMsg);
|
||||
if(exception != null) rsp.setException(exception);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
classLoader.checkJarAvailable();
|
||||
} catch (SolrException e) {
|
||||
rsp.add("error", "Jar could not be loaded");
|
||||
rsp.setException(e);
|
||||
return;
|
||||
} catch (IOException e) {
|
||||
unrecoverable = true;
|
||||
errMsg = "Could not load jar";
|
||||
exception = e;
|
||||
handleRequest(req,rsp);
|
||||
return;
|
||||
}
|
||||
|
||||
super.handleRequest(req, rsp);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SolrRequestHandler createRequestHandler() {
|
||||
try {
|
||||
Class clazz = classLoader.findClass(_className);
|
||||
Constructor<?>[] cons = clazz.getConstructors();
|
||||
for (Constructor<?> con : cons) {
|
||||
Class<?>[] types = con.getParameterTypes();
|
||||
if(types.length == 1 && types[0] == SolrCore.class){
|
||||
return SolrRequestHandler.class.cast(con.newInstance(this));
|
||||
}
|
||||
}
|
||||
return (SolrRequestHandler)clazz.newInstance();
|
||||
} catch (Exception e) {
|
||||
unrecoverable = true;
|
||||
errMsg = MessageFormat.format("class {0} could not be loaded ",_className);
|
||||
this.exception = e;
|
||||
return this;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
super.close();
|
||||
if(_closed) return;
|
||||
classLoader.releaseJar();
|
||||
_closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static class MemClassLoader extends ClassLoader {
|
||||
private JarRepository.JarContentRef jarContent;
|
||||
private final DynamicLazyRequestHandlerWrapper handlerWrapper;
|
||||
public MemClassLoader(DynamicLazyRequestHandlerWrapper handlerWrapper) {
|
||||
super(handlerWrapper.solrResourceLoader.classLoader);
|
||||
this.handlerWrapper = handlerWrapper;
|
||||
|
||||
}
|
||||
|
||||
boolean checkJarAvailable() throws IOException {
|
||||
if (jarContent != null) return true;
|
||||
|
||||
try {
|
||||
synchronized (this) {
|
||||
jarContent = handlerWrapper.coreContainer.getJarRepository().getJarIncRef(handlerWrapper.lib+"/"+handlerWrapper.version);
|
||||
return true;
|
||||
}
|
||||
} catch (SolrException se) {
|
||||
throw se;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<?> findClass(String name) throws ClassNotFoundException {
|
||||
try {
|
||||
return super.findClass(name);
|
||||
} catch (ClassNotFoundException e) {
|
||||
String path = name.replace('.', '/').concat(".class");
|
||||
ByteBuffer buf = null;
|
||||
try {
|
||||
if(jarContent == null) checkJarAvailable();
|
||||
buf = jarContent.jar.getFileContent(path);
|
||||
} catch (IOException e1) {
|
||||
throw new ClassNotFoundException("class not found "+ name ,e1) ;
|
||||
|
||||
}
|
||||
|
||||
ProtectionDomain defaultDomain = null;
|
||||
|
||||
//using the default protection domain, with no permissions
|
||||
try {
|
||||
defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/.system/blob/"+handlerWrapper.lib), (Certificate[]) null),
|
||||
null);
|
||||
} catch (MalformedURLException e1) {
|
||||
//should not happen
|
||||
}
|
||||
return defineClass(name,buf.array(),buf.arrayOffset(),buf.limit(), defaultDomain);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void releaseJar(){
|
||||
handlerWrapper.coreContainer.getJarRepository().decrementJarRefCount(jarContent);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void close() {
|
||||
for (Map.Entry<String, SolrRequestHandler> e : handlers.entrySet()) {
|
||||
if (e.getValue() instanceof AutoCloseable) {
|
||||
try {
|
||||
((AutoCloseable) e.getValue()).close();
|
||||
} catch (Exception exp) {
|
||||
log.error("Error closing requestHandler "+e.getKey() , exp);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1069,6 +1069,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
|||
}
|
||||
}
|
||||
|
||||
if(reqHandlers != null) reqHandlers.close();
|
||||
|
||||
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import org.apache.solr.common.SolrException;
|
|||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.core.PluginInfo;
|
||||
import org.apache.solr.core.RequestHandlers;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.core.SolrInfoMBean;
|
||||
|
|
@ -54,6 +55,7 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
|
|||
private final AtomicLong numTimeouts = new AtomicLong();
|
||||
private final Timer requestTimes = new Timer();
|
||||
private final long handlerStart = System.currentTimeMillis();
|
||||
private PluginInfo pluginInfo;
|
||||
|
||||
/**
|
||||
* Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
|
||||
|
|
@ -233,6 +235,18 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
|
|||
return handler;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pluginInfo
|
||||
*/
|
||||
public void setPluginInfo(PluginInfo pluginInfo){
|
||||
if(pluginInfo==null) this.pluginInfo = pluginInfo;
|
||||
}
|
||||
|
||||
public PluginInfo getPluginInfo(){
|
||||
return pluginInfo;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public NamedList<Object> getStatistics() {
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
|
|||
@Override
|
||||
public void init(FilterConfig config) throws ServletException
|
||||
{
|
||||
log.info("SolrDispatchFilter.init()");
|
||||
log.info("SolrDispatchFilter.init()" + this.getClass().getClassLoader());
|
||||
|
||||
try {
|
||||
// web.xml configuration
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
package org.apache.solr.core;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.solr.handler.DumpRequestHandler;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
|
||||
public class BlobStoreTestRequestHandler extends DumpRequestHandler{
|
||||
@Override
|
||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
|
||||
super.handleRequestBody(req, rsp);
|
||||
rsp.add("classloader", this.getClass().getClassLoader().getClass().getName());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
package org.apache.solr.core;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.handler.TestBlobHandler;
|
||||
import org.apache.solr.util.RESTfulServerProvider;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
import org.apache.solr.util.SimplePostTool;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
|
||||
static final Logger log = LoggerFactory.getLogger(TestDynamicLoading.class);
|
||||
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
|
||||
|
||||
private void setupHarnesses() {
|
||||
for (final SolrServer client : clients) {
|
||||
RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
|
||||
@Override
|
||||
public String getBaseURL() {
|
||||
return ((HttpSolrServer)client).getBaseURL();
|
||||
}
|
||||
});
|
||||
restTestHarnesses.add(harness);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public void doTest() throws Exception {
|
||||
|
||||
setupHarnesses();
|
||||
dynamicLoading();
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
private void dynamicLoading() throws Exception {
|
||||
String payload = "{\n" +
|
||||
"'create-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandler' , 'lib':'test','version':'1'}\n" +
|
||||
"}";
|
||||
RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
|
||||
TestSolrConfigHandler.runConfigCommand(client,"/config?wt=json",payload);
|
||||
TestSolrConfigHandler.testForResponseElement(client,
|
||||
null,
|
||||
"/config/overlay?wt=json",
|
||||
null,
|
||||
Arrays.asList("overlay", "requestHandler", "/test1", "lib"),
|
||||
"test",10);
|
||||
|
||||
Map map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
|
||||
|
||||
assertNotNull(map = (Map) map.get("error"));
|
||||
assertEquals(".system collection not available", map.get("msg"));
|
||||
|
||||
HttpSolrServer server = (HttpSolrServer) clients.get(random().nextInt(clients.size()));
|
||||
String baseURL = server.getBaseURL();
|
||||
baseURL = baseURL.substring(0, baseURL.lastIndexOf('/'));
|
||||
TestBlobHandler.createSysColl(new HttpSolrServer(baseURL,server.getHttpClient()));
|
||||
map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
|
||||
|
||||
assertNotNull(map = (Map) map.get("error"));
|
||||
assertEquals("no such blob or version available: test/1", map.get("msg"));
|
||||
ByteBuffer jar = generateZip( TestDynamicLoading.class,BlobStoreTestRequestHandler.class);
|
||||
TestBlobHandler.postAndCheck(cloudClient, baseURL, jar,1);
|
||||
|
||||
// Thread.sleep(100);
|
||||
map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
|
||||
|
||||
assertEquals(RequestHandlers.MemClassLoader.class.getName(), map.get("classloader"));
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static ByteBuffer generateZip(Class... classes) throws IOException {
|
||||
ZipOutputStream zipOut = null;
|
||||
SimplePostTool.BAOS bos = new SimplePostTool.BAOS();
|
||||
zipOut = new ZipOutputStream(bos);
|
||||
zipOut.setLevel(ZipOutputStream.DEFLATED);
|
||||
for (Class c : classes) {
|
||||
String path = c.getName().replace('.', '/').concat(".class");
|
||||
ZipEntry entry = new ZipEntry(path);
|
||||
ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path));
|
||||
zipOut.putNextEntry(entry);
|
||||
zipOut.write(b.array(), 0, b.limit());
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
zipOut.close();
|
||||
return bos.getByteBuffer();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -21,22 +21,19 @@ package org.apache.solr.core;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
|
||||
import org.apache.solr.util.RestTestBase;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
|
|
@ -142,7 +139,7 @@ public class TestSolrConfigHandler extends RestTestBase {
|
|||
reqhandlertests(restTestHarness, null,null);
|
||||
}
|
||||
|
||||
private static void runConfigCommand(RestTestHarness harness, String uri, String payload) throws IOException {
|
||||
public static void runConfigCommand(RestTestHarness harness, String uri, String payload) throws IOException {
|
||||
String response = harness.post(uri, SolrTestCaseJ4.json(payload));
|
||||
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
|
||||
assertNull(response, map.get("errors"));
|
||||
|
|
@ -155,53 +152,34 @@ public class TestSolrConfigHandler extends RestTestBase {
|
|||
"}";
|
||||
runConfigCommand(writeHarness,"/config?wt=json", payload);
|
||||
|
||||
boolean success = false;
|
||||
long startTime = System.nanoTime();
|
||||
long maxTimeoutSeconds = 10;
|
||||
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
|
||||
String uri = "/config/overlay?wt=json";
|
||||
Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
|
||||
if("lazy".equals( ConfigOverlay.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x","startup")))) {
|
||||
Map map = getRespMap("/x?wt=json",writeHarness);
|
||||
if(map.containsKey("params")) {
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Thread.sleep(100);
|
||||
|
||||
}
|
||||
|
||||
assertTrue( "Could not register requestHandler ", success);
|
||||
testForResponseElement(writeHarness,
|
||||
testServerBaseUrl,
|
||||
"/config/overlay?wt=json",
|
||||
cloudSolrServer,
|
||||
Arrays.asList("overlay", "requestHandler", "/x", "startup"),
|
||||
"lazy",
|
||||
10);
|
||||
|
||||
payload = "{\n" +
|
||||
"'update-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy' , 'a':'b'}\n" +
|
||||
"}";
|
||||
runConfigCommand(writeHarness,"/config?wt=json", payload);
|
||||
|
||||
success = false;
|
||||
startTime = System.nanoTime();
|
||||
maxTimeoutSeconds = 10;
|
||||
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
|
||||
String uri = "/config/overlay?wt=json";
|
||||
Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
|
||||
if("b".equals( ConfigOverlay.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x","a")))) {
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
Thread.sleep(100);
|
||||
|
||||
}
|
||||
|
||||
assertTrue( "Could not update requestHandler ", success);
|
||||
testForResponseElement(writeHarness,
|
||||
testServerBaseUrl,
|
||||
"/config/overlay?wt=json",
|
||||
cloudSolrServer,
|
||||
Arrays.asList("overlay", "requestHandler", "/x", "a"),
|
||||
"b",
|
||||
10);
|
||||
|
||||
payload = "{\n" +
|
||||
"'delete-requesthandler' : '/x'" +
|
||||
"}";
|
||||
runConfigCommand(writeHarness,"/config?wt=json", payload);
|
||||
success = false;
|
||||
startTime = System.nanoTime();
|
||||
maxTimeoutSeconds = 10;
|
||||
boolean success = false;
|
||||
long startTime = System.nanoTime();
|
||||
int maxTimeoutSeconds = 10;
|
||||
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
|
||||
String uri = "/config/overlay?wt=json";
|
||||
Map m = testServerBaseUrl ==null? getRespMap(uri,writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl+uri ,cloudSolrServer) ;
|
||||
|
|
@ -216,6 +194,33 @@ public class TestSolrConfigHandler extends RestTestBase {
|
|||
|
||||
}
|
||||
|
||||
public static void testForResponseElement(RestTestHarness harness,
|
||||
String testServerBaseUrl,
|
||||
String uri,
|
||||
CloudSolrServer cloudSolrServer,List<String> jsonPath,
|
||||
String expected,
|
||||
long maxTimeoutSeconds ) throws Exception {
|
||||
|
||||
boolean success = false;
|
||||
long startTime = System.nanoTime();
|
||||
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
|
||||
Map m = testServerBaseUrl ==null? getRespMap(uri,harness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl + uri, cloudSolrServer) ;
|
||||
if(Objects.equals(expected,ConfigOverlay.getObjectByPath(m, true, jsonPath))) {
|
||||
success = true;
|
||||
break;
|
||||
/*Map map = getRespMap("/x?wt=json",harness);
|
||||
if(map.containsKey("params")) {
|
||||
success = true;
|
||||
break;
|
||||
}*/
|
||||
}
|
||||
Thread.sleep(100);
|
||||
|
||||
}
|
||||
|
||||
assertTrue( "Could not add/change requestHandler ", success);
|
||||
}
|
||||
|
||||
|
||||
public static Map getRespMap(String path, RestTestHarness restHarness) throws Exception {
|
||||
String response = restHarness.query(path);
|
||||
|
|
|
|||
|
|
@ -17,13 +17,7 @@ package org.apache.solr.handler;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
|
|
@ -31,7 +25,6 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.google.common.io.Closeables;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
|
|
@ -41,6 +34,7 @@ import org.apache.http.entity.ByteArrayEntity;
|
|||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
||||
|
|
@ -57,7 +51,7 @@ import org.slf4j.LoggerFactory;
|
|||
import static org.apache.solr.core.ConfigOverlay.getObjectByPath;
|
||||
|
||||
public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
||||
static final Logger log = LoggerFactory.getLogger(TestSolrConfigHandlerConcurrent.class);
|
||||
static final Logger log = LoggerFactory.getLogger(TestBlobHandler.class);
|
||||
|
||||
private void doBlobHandlerTest() throws Exception {
|
||||
SolrServer server = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0)));
|
||||
|
|
@ -87,8 +81,8 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
|||
for (int i = 0; i < bytarr.length; i++) bytarr[i]= (byte) (i % 127);
|
||||
byte[] bytarr2 = new byte[2048];
|
||||
for (int i = 0; i < bytarr2.length; i++) bytarr2[i]= (byte) (i % 127);
|
||||
postAndCheck(baseUrl, bytarr, 1);
|
||||
postAndCheck(baseUrl, bytarr2, 2);
|
||||
postAndCheck(cloudClient, baseUrl, ByteBuffer.wrap( bytarr), 1);
|
||||
postAndCheck(cloudClient, baseUrl, ByteBuffer.wrap( bytarr2), 2);
|
||||
|
||||
url = baseUrl + "/.system/blob/test/1";
|
||||
map = TestSolrConfigHandlerConcurrent.getAsMap(url,cloudClient);
|
||||
|
|
@ -102,6 +96,17 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
|||
|
||||
}
|
||||
|
||||
public static void createSysColl(SolrServer server) throws SolrServerException, IOException {
|
||||
CollectionAdminResponse response1;
|
||||
CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create();
|
||||
createCollectionRequest.setCollectionName(".system");
|
||||
createCollectionRequest.setNumShards(1);
|
||||
createCollectionRequest.setReplicationFactor(2);
|
||||
response1 = createCollectionRequest.process(server);
|
||||
assertEquals(0, response1.getStatus());
|
||||
assertTrue(response1.isSuccess());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
|
|
@ -112,8 +117,8 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
|||
DirectUpdateHandler2.commitOnClose = true;
|
||||
}
|
||||
|
||||
private void postAndCheck(String baseUrl, byte[] bytes, int count) throws Exception {
|
||||
postData(baseUrl, bytes);
|
||||
public static void postAndCheck(CloudSolrServer cloudClient, String baseUrl, ByteBuffer bytes, int count) throws Exception {
|
||||
postData(cloudClient, baseUrl, bytes);
|
||||
String url;
|
||||
Map map;
|
||||
List l;
|
||||
|
|
@ -132,7 +137,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
|||
l = (List) ConfigOverlay.getObjectByPath(map, false, Arrays.asList("response", "docs"));
|
||||
assertNotNull(l);
|
||||
map = (Map) l.get(0);
|
||||
assertEquals("" + bytes.length, String.valueOf(map.get("size")));
|
||||
assertEquals("" + bytes.limit(), String.valueOf(map.get("size")));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -155,14 +160,14 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
|
|||
|
||||
}
|
||||
|
||||
private String postData(String baseUrl, byte[] bytarr) throws IOException {
|
||||
public static String postData(CloudSolrServer cloudClient, String baseUrl, ByteBuffer bytarr) throws IOException {
|
||||
HttpPost httpPost = null;
|
||||
HttpEntity entity;
|
||||
String response;
|
||||
try {
|
||||
httpPost = new HttpPost(baseUrl+"/.system/blob/test");
|
||||
httpPost.setHeader("Content-Type","application/octet-stream");
|
||||
httpPost.setEntity(new ByteArrayEntity(bytarr));
|
||||
httpPost.setEntity(new ByteArrayEntity(bytarr.array(), bytarr.arrayOffset(), bytarr.limit()));
|
||||
entity = cloudClient.getLbServer().getHttpClient().execute(httpPost).getEntity();
|
||||
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
|
||||
} finally {
|
||||
|
|
|
|||
Loading…
Reference in New Issue