diff --git a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java deleted file mode 100644 index 03e4de295df..00000000000 --- a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.core; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.security.CodeSource; -import java.security.ProtectionDomain; -import java.security.cert.Certificate; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.lucene.analysis.util.ResourceLoader; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.util.StrUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class MemClassLoader extends ClassLoader implements AutoCloseable, ResourceLoader { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private boolean allJarsLoaded = false; - private final SolrResourceLoader parentLoader; - private List libs = new ArrayList<>(); - @SuppressWarnings("rawtypes") - private Map classCache = new HashMap<>(); - private List errors = new ArrayList<>(); - - - public MemClassLoader(List libs, SolrResourceLoader resourceLoader) { - this.parentLoader = resourceLoader; - this.libs = libs; - } - - synchronized void loadRemoteJars() { - if (allJarsLoaded) return; - int count = 0; - for (PluginBag.RuntimeLib lib : libs) { - if (lib.getUrl() != null) { - try { - lib.loadJar(); - lib.verify(); - } catch (Exception e) { - log.error("Error loading runtime library", e); - } - count++; - } - } - if (count == libs.size()) allJarsLoaded = true; - } - - public synchronized void loadJars() { - if (allJarsLoaded) return; - - for (PluginBag.RuntimeLib lib : libs) { - try { - lib.loadJar(); - lib.verify(); - } catch (Exception exception) { - errors.add(exception.getMessage()); - if (exception instanceof SolrException) throw (SolrException) exception; - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Atleast one runtimeLib could not be loaded", exception); - } - } - allJarsLoaded = true; - } - - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - if(!allJarsLoaded ) loadJars(); - try { - return parentLoader.findClass(name, Object.class); - } catch (Exception e) { - return loadFromRuntimeLibs(name); - } - } - - @SuppressWarnings({"rawtypes"}) - private synchronized Class loadFromRuntimeLibs(String name) throws ClassNotFoundException { - Class result = classCache.get(name); - if(result != null) - return result; - AtomicReference jarName = new AtomicReference<>(); - ByteBuffer buf = null; - try { - buf = getByteBuffer(name, jarName); - } catch (Exception e) { - throw new ClassNotFoundException("class could not be loaded " + name + (errors.isEmpty()? "": "Some dynamic libraries could not be loaded: "+ StrUtils.join(errors, '|')), e); - } - if (buf == null) throw new ClassNotFoundException("Class not found :" + name); - ProtectionDomain defaultDomain = null; - //using the default protection domain, with no permissions - try { - defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + jarName.get()), (Certificate[]) null), - null); - } catch (MalformedURLException mue) { - throw new ClassNotFoundException("Unexpected exception ", mue); - //should not happen - } - log.info("Defining_class {} from runtime jar {} ", name, jarName); - - result = defineClass(name, buf.array(), buf.arrayOffset(), buf.limit(), defaultDomain); - classCache.put(name, result); - return result; - } - - private ByteBuffer getByteBuffer(String name, AtomicReference jarName) throws Exception { - if (!allJarsLoaded) { - loadJars(); - - } - - String path = name.replace('.', '/').concat(".class"); - ByteBuffer buf = null; - for (PluginBag.RuntimeLib lib : libs) { - try { - buf = lib.getFileContent(path); - if (buf != null) { - jarName.set(lib.getName()); - break; - } - } catch (Exception exp) { - throw new ClassNotFoundException("Unable to load class :" + name, exp); - } - } - - return buf; - } - - @Override - public void close() { - for (PluginBag.RuntimeLib lib : libs) { - try { - lib.close(); - } catch (Exception e) { - log.error("Error closing lib {}", lib.getName(), e); - } - } - } - - @Override - public InputStream openResource(String resource) throws IOException { - AtomicReference jarName = new AtomicReference<>(); - try { - ByteBuffer buf = getByteBuffer(resource, jarName); - if (buf == null) throw new IOException("Resource could not be found " + resource); - } catch (Exception e) { - throw new IOException("Resource could not be found " + resource, e); - } - return null; - } - - @Override - public Class findClass(String cname, Class expectedType) { - if(!allJarsLoaded ) loadJars(); - try { - return findClass(cname).asSubclass(expectedType); - } catch (Exception e) { - if (e instanceof SolrException) { - throw (SolrException) e; - } else { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "error loading class " + cname, e); - } - } - - } - - @Override - public T newInstance(String cname, Class expectedType) { - try { - return findClass(cname, expectedType).getConstructor().newInstance(); - } catch (SolrException e) { - throw e; - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "error instantiating class :" + cname, e); - } - } - - -} diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index d2442a130cc..e868ada52a6 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -16,11 +16,8 @@ */ package org.apache.solr.core; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -31,25 +28,18 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag; import org.apache.solr.api.ApiSupport; -import org.apache.solr.cloud.CloudUtil; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.StrUtils; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.update.processor.UpdateRequestProcessorChain; -import org.apache.solr.update.processor.UpdateRequestProcessorFactory; -import org.apache.solr.util.CryptoKeys; -import org.apache.solr.util.SimplePostTool; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; @@ -58,7 +48,6 @@ import org.slf4j.LoggerFactory; import static java.util.Collections.singletonMap; import static org.apache.solr.api.ApiBag.HANDLER_NAME; -import static org.apache.solr.common.params.CommonParams.NAME; /** * This manages the lifecycle of a set of plugin of the same type . @@ -130,22 +119,11 @@ public class PluginBag implements AutoCloseable { @SuppressWarnings({"unchecked"}) public PluginHolder createPlugin(PluginInfo info) { - if ("true".equals(String.valueOf(info.attributes.get("runtimeLib")))) { - if (log.isDebugEnabled()) { - log.debug(" {} : '{}' created with runtimeLib=true ", meta.getCleanTag(), info.name); - } - LazyPluginHolder holder = new LazyPluginHolder<>(meta, info, core, RuntimeLib.isEnabled() ? - core.getMemClassLoader() : - core.getResourceLoader(), true); - - return meta.clazz == UpdateRequestProcessorFactory.class ? - (PluginHolder) new UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder(holder) : - holder; - } else if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) { + if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) { if (log.isDebugEnabled()) { log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name); } - return new LazyPluginHolder(meta, info, core, core.getResourceLoader(), false); + return new LazyPluginHolder(meta, info, core, core.getResourceLoader()); } else { if (info.pkgName != null) { PackagePluginHolder holder = new PackagePluginHolder<>(info, core, meta); @@ -433,22 +411,13 @@ public class PluginBag implements AutoCloseable { protected SolrException solrException; private final SolrCore core; protected ResourceLoader resourceLoader; - private final boolean isRuntimeLib; - LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader, boolean isRuntimeLib) { + LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader) { super(pluginInfo); this.pluginMeta = pluginMeta; - this.isRuntimeLib = isRuntimeLib; this.core = core; this.resourceLoader = loader; - if (loader instanceof MemClassLoader) { - if (!RuntimeLib.isEnabled()) { - String s = "runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true"; - log.warn(s); - solrException = new SolrException(SolrException.ErrorCode.SERVER_ERROR, s); - } - } } @Override @@ -474,24 +443,14 @@ public class PluginBag implements AutoCloseable { if (log.isInfoEnabled()) { log.info("Going to create a new {} with {} ", pluginMeta.getCleanTag(), pluginInfo); } - if (resourceLoader instanceof MemClassLoader) { - MemClassLoader loader = (MemClassLoader) resourceLoader; - loader.loadJars(); - } + @SuppressWarnings({"unchecked"}) Class clazz = (Class) pluginMeta.clazz; T localInst = null; try { localInst = SolrCore.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader); } catch (SolrException e) { - if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) { - throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()), - e.getMessage() + ". runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true", - e.getCause()); - } throw e; - - } initInstance(localInst, pluginInfo); if (localInst instanceof SolrCoreAware) { @@ -511,165 +470,6 @@ public class PluginBag implements AutoCloseable { } } - /** - * This represents a Runtime Jar. A jar requires two details , name and version - */ - public static class RuntimeLib implements PluginInfoInitialized, AutoCloseable { - private String name, version, sig, sha512, url; - private BlobRepository.BlobContentRef jarContent; - private final CoreContainer coreContainer; - private boolean verified = false; - - @Override - public void init(PluginInfo info) { - name = info.attributes.get(NAME); - url = info.attributes.get("url"); - sig = info.attributes.get("sig"); - if(url == null) { - Object v = info.attributes.get("version"); - if (name == null || v == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version"); - } - version = String.valueOf(v); - } else { - sha512 = info.attributes.get("sha512"); - if(sha512 == null){ - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute"); - } - ByteBuffer buf = null; - buf = coreContainer.getBlobRepository().fetchFromUrl(name, url); - - String digest = BlobRepository.sha512Digest(buf); - if(!sha512.equals(digest)) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest) ); - } - log.info("dynamic library verified {}, sha512: {}", url, sha512); - } - } - - public RuntimeLib(SolrCore core) { - coreContainer = core.getCoreContainer(); - } - - public String getUrl(){ - return url; - } - - @SuppressWarnings({"unchecked"}) - void loadJar() { - if (jarContent != null) return; - synchronized (this) { - if (jarContent != null) return; - - jarContent = url == null? - coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version): - coreContainer.getBlobRepository().getBlobIncRef(name, null,url,sha512); - - } - } - - public static boolean isEnabled() { - return Boolean.getBoolean("enable.runtime.lib"); - } - - public String getName() { - return name; - } - - public String getVersion() { - return version; - } - - public String getSig() { - return sig; - - } - - public ByteBuffer getFileContent(String entryName) throws IOException { - if (jarContent == null) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name ); - return getFileContent(jarContent.blob, entryName); - - } - - public ByteBuffer getFileContent(BlobRepository.BlobContent blobContent, String entryName) throws IOException { - ByteBuffer buff = blobContent.get(); - ByteArrayInputStream zipContents = new ByteArrayInputStream(buff.array(), buff.arrayOffset(), buff.limit()); - ZipInputStream zis = new ZipInputStream(zipContents); - try { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (entryName == null || entryName.equals(entry.getName())) { - SimplePostTool.BAOS out = new SimplePostTool.BAOS(); - byte[] buffer = new byte[2048]; - int size; - while ((size = zis.read(buffer, 0, buffer.length)) != -1) { - out.write(buffer, 0, size); - } - out.close(); - return out.getByteBuffer(); - } - } - } finally { - zis.closeEntry(); - } - return null; - } - - - @Override - public void close() { - if (jarContent != null) coreContainer.getBlobRepository().decrementBlobRefCount(jarContent); - } - - public static List getLibObjects(SolrCore core, List libs) { - List l = new ArrayList<>(libs.size()); - for (PluginInfo lib : libs) { - RuntimeLib rtl = new RuntimeLib(core); - try { - rtl.init(lib); - } catch (Exception e) { - log.error("error loading runtime library", e); - } - l.add(rtl); - } - return l; - } - - public void verify() throws Exception { - if (verified) return; - if (jarContent == null) { - log.error("Calling verify before loading the jar"); - return; - } - - if (!coreContainer.isZooKeeperAware()) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Signing jar is possible only in cloud"); - Map keys = CloudUtil.getTrustedKeys(coreContainer.getZkController().getZkClient(), "exe"); - if (keys.isEmpty()) { - if (sig == null) { - verified = true; - return; - } else { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No public keys are available in ZK to verify signature for runtime lib " + name); - } - } else if (sig == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString("runtimelib {0} should be signed with one of the keys in ZK /keys/exe ", name)); - } - - try { - String matchedKey = new CryptoKeys(keys).verify(sig, jarContent.blob.get()); - if (matchedKey == null) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No key matched signature for jar : " + name + " version: " + version); - log.info("Jar {} signed with {} successfully verified", name, matchedKey); - } catch (Exception e) { - if (e instanceof SolrException) throw e; - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error verifying key ", e); - } - } - } - - public Api v2lookup(String path, String method, Map parts) { if (apiBag == null) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "this should not happen, looking up for v2 API at the wrong place"); diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 886179906f3..6cc7c6f4dfa 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -347,7 +347,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { // and even then -- only if there is a single SpellCheckComponent // because of queryConverter.setIndexAnalyzer .add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS)) - .add(new SolrPluginInfo(PluginBag.RuntimeLib.class, "runtimeLib", REQUIRE_NAME, MULTI_OK)) // this is hackish, since it picks up all SolrEventListeners, // regardless of when/how/why they are used (or even if they are // declared outside of the appropriate context) but there's no nice diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index dc6ef6edad6..a4a6fcfde7b 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -225,7 +225,6 @@ public final class SolrCore implements SolrInfoBean, Closeable { private final RecoveryStrategy.Builder recoveryStrategyBuilder; private IndexReaderFactory indexReaderFactory; private final Codec codec; - private final MemClassLoader memClassLoader; //singleton listener for all packages used in schema private final PackageListeningClassLoader schemaPluginsLoader; @@ -1003,10 +1002,6 @@ public final class SolrCore implements SolrInfoBean, Closeable { this.solrDelPolicy = initDeletionPolicy(delPolicy); this.codec = initCodec(solrConfig, this.schema); - - memClassLoader = new MemClassLoader( - PluginBag.RuntimeLib.getLibObjects(this, solrConfig.getPluginInfos(PluginBag.RuntimeLib.class.getName())), - getResourceLoader()); initIndex(prev != null, reload); initWriters(); @@ -1036,9 +1031,6 @@ public final class SolrCore implements SolrInfoBean, Closeable { // Initialize the RestManager restManager = initRestManager(); - // at this point we can load jars loaded from remote urls. - memClassLoader.loadRemoteJars(); - // Finally tell anyone who wants to know resourceLoader.inform(resourceLoader); resourceLoader.inform(this); // last call before the latch is released. @@ -1611,14 +1603,6 @@ public final class SolrCore implements SolrInfoBean, Closeable { valueSourceParsers.close(); transformerFactories.close(); - if (memClassLoader != null) { - try { - memClassLoader.close(); - } catch (Exception e) { - } - } - - try { if (null != updateHandler) { updateHandler.close(); @@ -2789,9 +2773,6 @@ public final class SolrCore implements SolrInfoBean, Closeable { }; } - public MemClassLoader getMemClassLoader() { - return memClassLoader; - } public interface RawWriter { default String getContentType() { diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index b94064ad6bd..4940af29bfa 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -63,7 +63,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigOverlay; -import org.apache.solr.core.PluginBag; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.RequestParams; import org.apache.solr.core.SolrConfig; @@ -572,21 +571,6 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa op.getMap(PluginInfo.INVARIANTS, null); op.getMap(PluginInfo.APPENDS, null); if (op.hasError()) return overlay; - if (info.clazz == PluginBag.RuntimeLib.class) { - if (!PluginBag.RuntimeLib.isEnabled()) { - op.addError("Solr not started with -Denable.runtime.lib=true"); - return overlay; - } - try { - try (PluginBag.RuntimeLib rtl = new PluginBag.RuntimeLib(req.getCore())) { - rtl.init(new PluginInfo(info.tag, op.getDataMap())); - } - } catch (Exception e) { - op.addError(e.getMessage()); - log.error("can't load this plugin ", e); - return overlay; - } - } if (!verifyClass(op, clz, info.clazz)) return overlay; if (pluginExists(info, overlay, name)) { if (isCeate) { diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index d6046166cf8..0877c54d216 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -138,7 +138,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware, streamFactory.withFunctionName(pluginInfo.name, () -> holder.getClazz()); } else { - Class clazz = core.getMemClassLoader().findClass(pluginInfo.className, Expressible.class); + Class clazz = core.getResourceLoader().findClass(pluginInfo.className, Expressible.class); streamFactory.withFunctionName(pluginInfo.name, clazz); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index 86742fada2d..a3c8a55856f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -77,7 +77,7 @@ import org.apache.solr.common.util.Base64; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetProperties; -import org.apache.solr.core.TestDynamicLoading; +import org.apache.solr.core.TestSolrConfigHandler; import org.apache.solr.security.BasicAuthIntegrationTest; import org.apache.solr.servlet.SolrDispatchFilter; import org.apache.solr.util.ExternalPaths; @@ -485,7 +485,7 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 { private long uploadConfigSet(String configSetName, String suffix, String username, String password, SolrZkClient zkClient) throws IOException { // Read zipped sample config - ByteBuffer sampleZippedConfig = TestDynamicLoading + ByteBuffer sampleZippedConfig = TestSolrConfigHandler .getFileContent( createTempZipFile("solr/configsets/upload/"+configSetName), false); @@ -509,7 +509,7 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 { File zipFile = new File(solrCluster.getBaseDir().toFile().getAbsolutePath() + File.separator + TestUtil.randomSimpleString(random(), 6, 8) + ".zip"); - File directory = TestDynamicLoading.getFile(directoryPath); + File directory = SolrTestCaseJ4.getFile(directoryPath); if (log.isInfoEnabled()) { log.info("Directory: {}", directory.getAbsolutePath()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java deleted file mode 100644 index 321e208c2b6..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Map; - -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.common.LinkedHashMapWriter; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.MemClassLoader; -import org.apache.solr.core.TestDynamicLoading; -import org.apache.solr.core.TestSolrConfigHandler; -import org.apache.solr.handler.TestBlobHandler; -import org.apache.solr.util.CryptoKeys; -import org.apache.solr.util.RestTestHarness; -import org.apache.zookeeper.CreateMode; -import org.junit.Test; - -import static java.util.Arrays.asList; -import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues; - -public class TestCryptoKeys extends AbstractFullDistribZkTestBase { - - public TestCryptoKeys() { - super(); - sliceCount = 1; - } - - @Test - public void test() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - setupRestTestHarnesses(); - String pk1sig = "G8LEW7uJ1is81Aqqfl3Sld3qDtOxPuVFeTLJHFJWecgDvUkmJNFXmf7nkHOVlXnDWahp1vqZf0W02VHXg37lBw=="; - String pk2sig = "pCyBQycB/0YvLVZfKLDIIqG1tFwM/awqzkp2QNpO7R3ThTqmmrj11wEJFDRLkY79efuFuQPHt40EE7jrOKoj9jLNELsfEqvU3jw9sZKiDONY+rV9Bj9QPeW8Pgt+F9Y1"; - String wrongKeySig = "xTk2hTipfpb+J5s4x3YZGOXkmHWtnJz05Vvd8RTm/Q1fbQVszR7vMk6dQ1URxX08fcg4HvxOo8g9bG2TSMOGjg=="; - String result = null; - CryptoKeys cryptoKeys = null; - SolrZkClient zk = getCommonCloudSolrClient().getZkStateReader().getZkClient(); - cryptoKeys = new CryptoKeys(CloudUtil.getTrustedKeys(zk, "exe")); - ByteBuffer samplefile = ByteBuffer.wrap(readFile("cryptokeys/samplefile.bin")); - //there are no keys yet created in ZK - - result = cryptoKeys.verify( pk1sig,samplefile); - assertNull(result); - - zk.makePath("/keys/exe", true); - zk.create("/keys/exe/pubk1.der", readFile("cryptokeys/pubk1.der"), CreateMode.PERSISTENT, true); - zk.create("/keys/exe/pubk2.der", readFile("cryptokeys/pubk2.der"), CreateMode.PERSISTENT, true); - Map trustedKeys = CloudUtil.getTrustedKeys(zk, "exe"); - - cryptoKeys = new CryptoKeys(trustedKeys); - result = cryptoKeys.verify(pk2sig, samplefile); - assertEquals("pubk2.der", result); - - - result = cryptoKeys.verify(pk1sig, samplefile); - assertEquals("pubk1.der", result); - - try { - result = cryptoKeys.verify(wrongKeySig,samplefile); - assertNull(result); - } catch (Exception e) { - //pass - } - try { - result = cryptoKeys.verify( "SGVsbG8gV29ybGQhCg==", samplefile); - assertNull(result); - } catch (Exception e) { - //pass - } - - - HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size())); - String baseURL = randomClient.getBaseURL(); - baseURL = baseURL.substring(0, baseURL.lastIndexOf('/')); - - TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient())); - waitForRecoveriesToFinish(".system", true); - - ByteBuffer jar = TestDynamicLoading.getFileContent("runtimecode/runtimelibs.jar.bin"); - String blobName = "signedjar"; - TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 1); - - String payload = "{\n" + - "'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler' , 'runtimeLib':true }" + - "}"; - RestTestHarness client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "requestHandler", "/runtime", "class"), - "org.apache.solr.core.RuntimeLibReqHandler", 10); - - - payload = "{\n" + - "'add-runtimelib' : { 'name' : 'signedjar' ,'version':1}\n" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "version"), - 1l, 10); - - @SuppressWarnings({"rawtypes"}) - LinkedHashMapWriter map = TestSolrConfigHandler.getRespMap("/runtime", client); - String s = map._getStr( "error/msg",null); - assertNotNull(map.toString(), s); - assertTrue(map.toString(), s.contains("should be signed with one of the keys in ZK /keys/exe")); - - String wrongSig = "QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw=="; - - payload = "{\n" + - "'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw=='}\n" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "sig"), - wrongSig, 10); - - map = TestSolrConfigHandler.getRespMap("/runtime", client); - s = (String) Utils.getObjectByPath(map, false, Arrays.asList("error", "msg")); - assertNotNull(map.toString(), s);//No key matched signature for jar - assertTrue(map.toString(), s.contains("No key matched signature for jar")); - - String rightSig = "YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q=="; - - payload = "{\n" + - "'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q=='}\n" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "sig"), - rightSig, 10); - - map = TestSolrConfigHandler.testForResponseElement(client, - null, - "/runtime", - null, - Arrays.asList("class"), - "org.apache.solr.core.RuntimeLibReqHandler", 10); - compareValues(map, MemClassLoader.class.getName(), asList("loader")); - - rightSig = "VJPMTxDf8Km3IBj2B5HWkIOqeM/o+HHNobOYCNA3WjrEVfOMZbMMqS1Lo7uLUUp//RZwOGkOhrUhuPNY1z2CGEIKX2/m8VGH64L14d52oSvFiwhoTDDuuyjW1TFGu35D"; - payload = "{\n" + - "'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'VJPMTxDf8Km3IBj2B5HWkIOqeM/o+HHNobOYCNA3WjrEVfOMZbMMqS1Lo7uLUUp//RZwOGkOhrUhuPNY1z2CGEIKX2/m8VGH64L14d52oSvFiwhoTDDuuyjW1TFGu35D'}\n" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "sig"), - rightSig, 10); - - map = TestSolrConfigHandler.testForResponseElement(client, - null, - "/runtime", - null, - Arrays.asList("class"), - "org.apache.solr.core.RuntimeLibReqHandler", 10); - compareValues(map, MemClassLoader.class.getName(), asList("loader")); - } - - - private byte[] readFile(String fname) throws IOException { - byte[] buf = null; - try (FileInputStream fis = new FileInputStream(getFile(fname))) { - buf = new byte[fis.available()]; - fis.read(buf); - } - return buf; - } - - -} diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java index 16dad975258..18922f01699 100644 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java +++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java @@ -131,7 +131,7 @@ public class BlobRepositoryMockingTest { @Test public void testGetBlobIncrRefByUrl() throws Exception{ when(mockContainer.isZooKeeperAware()).thenReturn(true); - filecontent = TestDynamicLoading.getFileContent("runtimecode/runtimelibs_v2.jar.bin"); + filecontent = TestSolrConfigHandler.getFileContent("runtimecode/runtimelibs_v2.jar.bin"); url = "http://localhost:8080/myjar/location.jar"; @SuppressWarnings({"rawtypes"}) BlobRepository.BlobContentRef ref = repository.getBlobIncRef( "filefoo",null,url, diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java deleted file mode 100644 index d9a3bf47fbb..00000000000 --- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.core; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.cloud.AbstractFullDistribZkTestBase; -import org.apache.solr.handler.TestBlobHandler; -import org.apache.solr.util.RestTestHarness; -import org.apache.solr.util.SimplePostTool; -import org.junit.BeforeClass; -import org.junit.Test; - -import static java.util.Arrays.asList; -import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues; - -public class TestDynamicLoading extends AbstractFullDistribZkTestBase { - - @BeforeClass - public static void enableRuntimeLib() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - } - - @Test - // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") - //17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 - public void testDynamicLoading() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - setupRestTestHarnesses(); - - String blobName = "colltest"; - boolean success = false; - - - HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size())); - String baseURL = randomClient.getBaseURL(); - baseURL = baseURL.substring(0, baseURL.lastIndexOf('/')); - String payload = "{\n" + - "'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" + - "}"; - RestTestHarness client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "version"), - 1l, 10); - - - payload = "{\n" + - "'create-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandler' ,registerPath: '/solr,/v2', 'runtimeLib' : true }\n" + - "}"; - - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client,"/config",payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "requestHandler", "/test1", "class"), - "org.apache.solr.core.BlobStoreTestRequestHandler",10); - - @SuppressWarnings({"rawtypes"}) - Map map = TestSolrConfigHandler.getRespMap("/test1", client); - - assertNotNull(map.toString(), map = (Map) map.get("error")); - assertTrue(map.toString(), map.get("msg").toString().contains(".system collection not available")); - - - TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient())); - waitForRecoveriesToFinish(".system", true); - - map = TestSolrConfigHandler.getRespMap("/test1", client); - - - assertNotNull(map = (Map) map.get("error")); - assertTrue("full output " + map, map.get("msg").toString().contains("no such blob or version available: colltest/1" )); - payload = " {\n" + - " 'set' : {'watched': {" + - " 'x':'X val',\n" + - " 'y': 'Y val'}\n" + - " }\n" + - " }"; - - TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload); - TestSolrConfigHandler.testForResponseElement( - client, - null, - "/config/params", - cloudClient, - Arrays.asList("response", "params", "watched", "x"), - "X val", - 10); - - - - - for(int i=0;i<100;i++) { - map = TestSolrConfigHandler.getRespMap("/test1", client); - if("X val".equals(map.get("x"))){ - success = true; - break; - } - Thread.sleep(100); - } - ByteBuffer jar = null; - -// jar = persistZip("/tmp/runtimelibs.jar.bin", TestDynamicLoading.class, RuntimeLibReqHandler.class, RuntimeLibResponseWriter.class, RuntimeLibSearchComponent.class); -// if(true) return; - - jar = getFileContent("runtimecode/runtimelibs.jar.bin"); - TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 1); - - payload = "{\n" + - "'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler' , 'runtimeLib':true }," + - "'create-searchcomponent' : { 'name' : 'get', 'class': 'org.apache.solr.core.RuntimeLibSearchComponent' , 'runtimeLib':true }," + - "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'org.apache.solr.core.RuntimeLibResponseWriter' , 'runtimeLib':true }" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - - @SuppressWarnings({"rawtypes"}) - Map result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "requestHandler", "/runtime", "class"), - "org.apache.solr.core.RuntimeLibReqHandler", 10); - compareValues(result, "org.apache.solr.core.RuntimeLibResponseWriter", asList("overlay", "queryResponseWriter", "json1", "class")); - compareValues(result, "org.apache.solr.core.RuntimeLibSearchComponent", asList("overlay", "searchComponent", "get", "class")); - - result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/runtime", - null, - Arrays.asList("class"), - "org.apache.solr.core.RuntimeLibReqHandler", 10); - compareValues(result, MemClassLoader.class.getName(), asList( "loader")); - - result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/runtime?wt=json1", - null, - Arrays.asList("wt"), - "org.apache.solr.core.RuntimeLibResponseWriter", 10); - compareValues(result, MemClassLoader.class.getName(), asList( "loader")); - - result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/get?abc=xyz", - null, - Arrays.asList("get"), - "org.apache.solr.core.RuntimeLibSearchComponent", 10); - compareValues(result, MemClassLoader.class.getName(), asList( "loader")); - - jar = getFileContent("runtimecode/runtimelibs_v2.jar.bin"); - TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 2); - payload = "{\n" + - "'update-runtimelib' : { 'name' : 'colltest' ,'version':2}\n" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "version"), - 2l, 10); - - result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/get?abc=xyz", - null, - Arrays.asList("Version"), - "2", 10); - - - payload = " {\n" + - " 'set' : {'watched': {" + - " 'x':'X val',\n" + - " 'y': 'Y val'}\n" + - " }\n" + - " }"; - - TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload); - TestSolrConfigHandler.testForResponseElement( - client, - null, - "/config/params", - cloudClient, - Arrays.asList("response", "params", "watched", "x"), - "X val", - 10); - result = TestSolrConfigHandler.testForResponseElement( - client, - null, - "/test1", - cloudClient, - Arrays.asList("x"), - "X val", - 10); - - payload = " {\n" + - " 'set' : {'watched': {" + - " 'x':'X val changed',\n" + - " 'y': 'Y val'}\n" + - " }\n" + - " }"; - - TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload); - result = TestSolrConfigHandler.testForResponseElement( - client, - null, - "/test1", - cloudClient, - Arrays.asList("x"), - "X val changed", - 10); - } - - public static ByteBuffer getFileContent(String f) throws IOException { - return getFileContent(f, true); - } - /** - * @param loadFromClassPath if true, it will look in the classpath to find the file, - * otherwise load from absolute filesystem path. - */ - public static ByteBuffer getFileContent(String f, boolean loadFromClassPath) throws IOException { - ByteBuffer jar; - File file = loadFromClassPath ? getFile(f): new File(f); - try (FileInputStream fis = new FileInputStream(file)) { - byte[] buf = new byte[fis.available()]; - fis.read(buf); - jar = ByteBuffer.wrap(buf); - } - return jar; - } - - public static ByteBuffer persistZip(String loc, - @SuppressWarnings({"rawtypes"})Class... classes) throws IOException { - ByteBuffer jar = generateZip(classes); - try (FileOutputStream fos = new FileOutputStream(loc)){ - fos.write(jar.array(), 0, jar.limit()); - fos.flush(); - } - return jar; - } - - - public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException { - SimplePostTool.BAOS bos = new SimplePostTool.BAOS(); - try (ZipOutputStream zipOut = new ZipOutputStream(bos)) { - zipOut.setLevel(ZipOutputStream.DEFLATED); - for (@SuppressWarnings({"rawtypes"})Class c : classes) { - String path = c.getName().replace('.', '/').concat(".class"); - ZipEntry entry = new ZipEntry(path); - ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path)); - zipOut.putNextEntry(entry); - zipOut.write(b.array(), 0, b.limit()); - zipOut.closeEntry(); - } - } - return bos.getByteBuffer(); - } - -} diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java deleted file mode 100644 index b172d524430..00000000000 --- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.core; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Map; - -import com.google.common.collect.ImmutableMap; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.AbstractFullDistribZkTestBase; -import org.apache.solr.common.util.Pair; -import org.apache.solr.util.RestTestHarness; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.AbstractHandler; -import org.junit.BeforeClass; - -import static java.util.Arrays.asList; -import static org.apache.solr.core.TestDynamicLoading.getFileContent; -import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues; - -@SolrTestCaseJ4.SuppressSSL -public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase { - - @BeforeClass - public static void enableRuntimeLib() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - } - - public static Pair runHttpServer(Map jars) throws Exception { - final Server server = new Server(); - final ServerConnector connector = new ServerConnector(server); - server.setConnectors(new Connector[] { connector }); - server.setHandler(new AbstractHandler() { - @Override - public void handle(String s, Request request, HttpServletRequest req, HttpServletResponse rsp) - throws IOException { - ByteBuffer b = (ByteBuffer) jars.get(s); - if (b != null) { - rsp.getOutputStream().write(b.array(), 0, b.limit()); - rsp.setContentType("application/octet-stream"); - rsp.setStatus(HttpServletResponse.SC_OK); - request.setHandled(true); - } - } - }); - server.start(); - return new Pair<>(server, connector.getLocalPort()); - } - - public void testDynamicLoadingUrl() throws Exception { - setupRestTestHarnesses(); - Pair pair = runHttpServer(ImmutableMap.of("/jar1.jar", getFileContent("runtimecode/runtimelibs.jar.bin"))); - Integer port = pair.second(); - - try { - String payload = "{\n" + - "'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" + - " 'sha512':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" + - "}"; - RestTestHarness client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommandExpectFailure(client, "/config", payload, "Invalid jar"); - - - payload = "{\n" + - "'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" + - " 'sha512':'d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", "urljar", "sha512"), - "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", 120); - - payload = "{\n" + - "'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}" + - "}"; - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "requestHandler", "/runtime", "class"), - "org.apache.solr.core.RuntimeLibReqHandler", 120); - - @SuppressWarnings({"rawtypes"}) - Map result = TestSolrConfigHandler.testForResponseElement(client, - null, - "/runtime", - null, - Arrays.asList("class"), - "org.apache.solr.core.RuntimeLibReqHandler", 120); - compareValues(result, MemClassLoader.class.getName(), asList("loader")); - } finally { - pair.first().stop(); - - } - - - } -} - diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicURP.java b/solr/core/src/test/org/apache/solr/core/TestDynamicURP.java deleted file mode 100644 index ac37e28d421..00000000000 --- a/solr/core/src/test/org/apache/solr/core/TestDynamicURP.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.core; - -import static java.util.Collections.singletonMap; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; -import static org.apache.solr.core.TestDynamicLoading.getFileContent; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.handler.TestBlobHandler; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestDynamicURP extends SolrCloudTestCase { - - - private static final String COLLECTION = "testUrpColl"; - - @BeforeClass - public static void setupCluster() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - configureCluster(3) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient(); - String path = ZkStateReader.CONFIGS_ZKNODE + "/conf/solrconfig.xml"; - byte[] data = zkClient.getData(path, null, null, true); - - String solrconfigStr = new String(data, StandardCharsets.UTF_8); - zkClient.setData(path, solrconfigStr.replace("", - "\n" + - " \n" + - " \n" + - "\n" + - " \n" + - "").getBytes(StandardCharsets.UTF_8), true ); - - - CollectionAdminRequest.createCollection(COLLECTION, "conf", 3, 1).process(cluster.getSolrClient()); - waitForState("", COLLECTION, clusterShape(3, 3)); - } - - - - @Test - public void testUrp() throws Exception { - - ByteBuffer jar = getFileContent("runtimecode/runtimeurp.jar.bin"); - - String blobName = "urptest"; - TestBlobHandler.postAndCheck(cluster.getSolrClient(), cluster.getRandomJetty(random()).getBaseUrl().toString(), - blobName, jar, 1); - - new V2Request.Builder("/c/" + COLLECTION + "/config") - .withPayload(singletonMap("add-runtimelib", (MapWriter) ew1 -> ew1 - .put("name", blobName) - .put("version", "1"))) - .withMethod(POST) - .build() - .process(cluster.getSolrClient()); - TestSolrConfigHandler.testForResponseElement(null, - cluster.getRandomJetty(random()).getBaseUrl().toString(), - "/"+COLLECTION+"/config/overlay", - cluster.getSolrClient(), - Arrays.asList("overlay", "runtimeLib", blobName, "version") - ,"1",10); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", "123"); - doc.addField("name_s", "Test URP"); - new UpdateRequest() - .add(doc) - .commit(cluster.getSolrClient(), COLLECTION); - QueryResponse result = cluster.getSolrClient().query(COLLECTION, new SolrQuery("id:123")); - assertEquals(1, result.getResults().getNumFound()); - Object time_s = result.getResults().get(0).getFirstValue("time_s"); - assertNotNull(time_s); - - - - } - -} diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index 6aa48a5467f..ead344c316d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -16,10 +16,9 @@ */ package org.apache.solr.core; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; +import java.io.*; import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +26,8 @@ import java.util.Objects; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import com.google.common.collect.ImmutableList; import org.apache.commons.io.FileUtils; @@ -45,6 +46,7 @@ import org.apache.solr.search.SolrCache; import org.apache.solr.util.RESTfulServerProvider; import org.apache.solr.util.RestTestBase; import org.apache.solr.util.RestTestHarness; +import org.apache.solr.util.SimplePostTool; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.After; import org.junit.Before; @@ -67,8 +69,53 @@ public class TestSolrConfigHandler extends RestTestBase { private static final String collection = "collection1"; private static final String confDir = collection + "/conf"; + public static ByteBuffer getFileContent(String f) throws IOException { + return getFileContent(f, true); + } - @Before + /** + * @param loadFromClassPath if true, it will look in the classpath to find the file, + * otherwise load from absolute filesystem path. + */ + public static ByteBuffer getFileContent(String f, boolean loadFromClassPath) throws IOException { + ByteBuffer jar; + File file = loadFromClassPath ? getFile(f): new File(f); + try (FileInputStream fis = new FileInputStream(file)) { + byte[] buf = new byte[fis.available()]; + fis.read(buf); + jar = ByteBuffer.wrap(buf); + } + return jar; + } + + public static ByteBuffer persistZip(String loc, + @SuppressWarnings({"rawtypes"})Class... classes) throws IOException { + ByteBuffer jar = generateZip(classes); + try (FileOutputStream fos = new FileOutputStream(loc)){ + fos.write(jar.array(), 0, jar.limit()); + fos.flush(); + } + return jar; + } + + public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException { + SimplePostTool.BAOS bos = new SimplePostTool.BAOS(); + try (ZipOutputStream zipOut = new ZipOutputStream(bos)) { + zipOut.setLevel(ZipOutputStream.DEFLATED); + for (@SuppressWarnings({"rawtypes"})Class c : classes) { + String path = c.getName().replace('.', '/').concat(".class"); + ZipEntry entry = new ZipEntry(path); + ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path)); + zipOut.putNextEntry(entry); + zipOut.write(b.array(), 0, b.limit()); + zipOut.closeEntry(); + } + } + return bos.getByteBuffer(); + } + + + @Before public void before() throws Exception { tmpSolrHome = createTempDir().toFile(); tmpConfDir = new File(tmpSolrHome, confDir); diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java index f8811561ca0..f6930d064df 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java @@ -53,7 +53,7 @@ import java.util.concurrent.Callable; import java.util.function.Predicate; import static org.apache.solr.common.util.Utils.JAVABINCONSUMER; -import static org.apache.solr.core.TestDynamicLoading.getFileContent; +import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.hamcrest.CoreMatchers.containsString; @LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG") diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 2257cf3022d..1566759764c 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -68,7 +68,7 @@ import org.junit.Test; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.core.TestDynamicLoading.getFileContent; +import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.apache.solr.filestore.TestDistribPackageStore.readFile; import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey; import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile; diff --git a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java b/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java deleted file mode 100644 index 6cee3d9c1d8..00000000000 --- a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.update.processor; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.update.AddUpdateCommand; - -public class RuntimeUrp extends SimpleUpdateProcessorFactory { - @Override - protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) { - UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessorChain(req.getParams()); - List names = new ArrayList<>(); - for (UpdateRequestProcessorFactory p : processorChain.getProcessors()) { - if (p instanceof UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) { - p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).getDelegate(); - } - names.add(p.getClass().getSimpleName()); - } - cmd.solrDoc.addField("processors_s", StrUtils.join(names,'>')); - } -} diff --git a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java b/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java deleted file mode 100644 index 45bb41c07f5..00000000000 --- a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.update.processor; - -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractFullDistribZkTestBase; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.core.TestDynamicLoading; -import org.apache.solr.core.TestSolrConfigHandler; -import org.apache.solr.handler.TestBlobHandler; -import org.apache.solr.util.RestTestHarness; -import org.apache.solr.util.SimplePostTool; -import org.junit.Test; - -public class TestNamedUpdateProcessors extends AbstractFullDistribZkTestBase { - - @Test - //17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 - public void test() throws Exception { - System.setProperty("enable.runtime.lib", "true"); - setupRestTestHarnesses(); - - String blobName = "colltest"; - - HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size())); - String baseURL = randomClient.getBaseURL(); - - final String solrClientUrl = baseURL.substring(0, baseURL.lastIndexOf('/')); - TestBlobHandler.createSystemCollection(getHttpSolrClient(solrClientUrl, randomClient.getHttpClient())); - waitForRecoveriesToFinish(".system", true); - - TestBlobHandler.postAndCheck(cloudClient, baseURL.substring(0, baseURL.lastIndexOf('/')), blobName, TestDynamicLoading.generateZip(RuntimeUrp.class), 1); - - String payload = "{\n" + - "'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" + - "}"; - RestTestHarness client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - TestSolrConfigHandler.testForResponseElement(client, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "runtimeLib", blobName, "version"), - 1l, 10); - - payload = "{\n" + - "'create-updateprocessor' : { 'name' : 'firstFld', 'class': 'solr.FirstFieldValueUpdateProcessorFactory', 'fieldName':'test_s'}, \n" + - "'create-updateprocessor' : { 'name' : 'test', 'class': 'org.apache.solr.update.processor.RuntimeUrp', 'runtimeLib':true }, \n" + - "'create-updateprocessor' : { 'name' : 'maxFld', 'class': 'solr.MaxFieldValueUpdateProcessorFactory', 'fieldName':'mul_s'} \n" + - "}"; - - client = randomRestTestHarness(); - TestSolrConfigHandler.runConfigCommand(client, "/config", payload); - forAllRestTestHarnesses(restTestHarness -> { - try { - TestSolrConfigHandler.testForResponseElement(restTestHarness, - null, - "/config/overlay", - null, - Arrays.asList("overlay", "updateProcessor", "firstFld", "fieldName"), - "test_s", 10); - } catch (Exception ex) { - fail("Caught exception: "+ex); - } - }); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", "123"); - doc.addField("test_s", Arrays.asList("one", "two")); - doc.addField("mul_s", Arrays.asList("aaa", "bbb")); - randomClient.add(doc); - randomClient.commit(true, true); - QueryResponse result = randomClient.query(new SolrQuery("id:123")); - assertEquals(2, ((Collection) result.getResults().get(0).getFieldValues("test_s")).size()); - assertEquals(2, ((Collection) result.getResults().get(0).getFieldValues("mul_s")).size()); - doc = new SolrInputDocument(); - doc.addField("id", "456"); - doc.addField("test_s", Arrays.asList("three", "four")); - doc.addField("mul_s", Arrays.asList("aaa", "bbb")); - UpdateRequest ur = new UpdateRequest(); - ur.add(doc).setParam("processor", "firstFld,maxFld,test"); - randomClient.request(ur); - randomClient.commit(true, true); - result = randomClient.query(new SolrQuery("id:456")); - SolrDocument d = result.getResults().get(0); - assertEquals(1, d.getFieldValues("test_s").size()); - assertEquals(1, d.getFieldValues("mul_s").size()); - assertEquals("three", d.getFieldValues("test_s").iterator().next()); - assertEquals("bbb", d.getFieldValues("mul_s").iterator().next()); - String processors = (String) d.getFirstValue("processors_s"); - assertNotNull(processors); - assertEquals(StrUtils.splitSmart(processors, '>'), - Arrays.asList("FirstFieldValueUpdateProcessorFactory", "MaxFieldValueUpdateProcessorFactory", "RuntimeUrp", "LogUpdateProcessorFactory", "DistributedUpdateProcessorFactory", "RunUpdateProcessorFactory")); - - - } - - public static ByteBuffer getFileContent(String f) throws IOException { - ByteBuffer jar; - try (FileInputStream fis = new FileInputStream(getFile(f))) { - byte[] buf = new byte[fis.available()]; - fis.read(buf); - jar = ByteBuffer.wrap(buf); - } - return jar; - } - - public static ByteBuffer persistZip(String loc, - @SuppressWarnings({"rawtypes"})Class... classes) throws IOException { - ByteBuffer jar = generateZip(classes); - try (FileOutputStream fos = new FileOutputStream(loc)) { - fos.write(jar.array(), 0, jar.limit()); - fos.flush(); - } - return jar; - } - - - public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException { - SimplePostTool.BAOS bos = new SimplePostTool.BAOS(); - try (ZipOutputStream zipOut = new ZipOutputStream(bos)) { - zipOut.setLevel(ZipOutputStream.DEFLATED); - for (@SuppressWarnings({"rawtypes"})Class c : classes) { - String path = c.getName().replace('.', '/').concat(".class"); - ZipEntry entry = new ZipEntry(path); - ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path)); - zipOut.putNextEntry(entry); - zipOut.write(b.array(), 0, b.limit()); - zipOut.closeEntry(); - } - } - return bos.getByteBuffer(); - } - -}