Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14654 Remove plugin loading from .system collection (for 9.0) (#1677)

This commit is contained in:
Noble Paul 2020-08-06 16:32:32 +10:00 committed by GitHub
parent 6752111be8
commit 6f2f1c3de2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 62 additions and 1395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
solr/core/src/java/org/apache/solr/core/MemClassLoader.java
View File

@ -1,203 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.security.CodeSource;
import java.security.ProtectionDomain;
import java.security.cert.Certificate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.StrUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MemClassLoader extends ClassLoader implements AutoCloseable, ResourceLoader {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private boolean allJarsLoaded = false;
private final SolrResourceLoader parentLoader;
private List<PluginBag.RuntimeLib> libs = new ArrayList<>();
@SuppressWarnings("rawtypes")
private Map<String, Class> classCache = new HashMap<>();
private List<String> errors = new ArrayList<>();
public MemClassLoader(List<PluginBag.RuntimeLib> libs, SolrResourceLoader resourceLoader) {
this.parentLoader = resourceLoader;
this.libs = libs;
}
synchronized void loadRemoteJars() {
if (allJarsLoaded) return;
int count = 0;
for (PluginBag.RuntimeLib lib : libs) {
if (lib.getUrl() != null) {
try {
lib.loadJar();
lib.verify();
} catch (Exception e) {
log.error("Error loading runtime library", e);
}
count++;
}
}
if (count == libs.size()) allJarsLoaded = true;
}
public synchronized void loadJars() {
if (allJarsLoaded) return;
for (PluginBag.RuntimeLib lib : libs) {
try {
lib.loadJar();
lib.verify();
} catch (Exception exception) {
errors.add(exception.getMessage());
if (exception instanceof SolrException) throw (SolrException) exception;
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Atleast one runtimeLib could not be loaded", exception);
}
}
allJarsLoaded = true;
}
@Override
protected Class<?> findClass(String name) throws ClassNotFoundException {
if(!allJarsLoaded ) loadJars();
try {
return parentLoader.findClass(name, Object.class);
} catch (Exception e) {
return loadFromRuntimeLibs(name);
}
}
@SuppressWarnings({"rawtypes"})
private synchronized Class<?> loadFromRuntimeLibs(String name) throws ClassNotFoundException {
Class result = classCache.get(name);
if(result != null)
return result;
AtomicReference<String> jarName = new AtomicReference<>();
ByteBuffer buf = null;
try {
buf = getByteBuffer(name, jarName);
} catch (Exception e) {
throw new ClassNotFoundException("class could not be loaded " + name + (errors.isEmpty()? "": "Some dynamic libraries could not be loaded: "+ StrUtils.join(errors, '|')), e);
}
if (buf == null) throw new ClassNotFoundException("Class not found :" + name);
ProtectionDomain defaultDomain = null;
//using the default protection domain, with no permissions
try {
defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + jarName.get()), (Certificate[]) null),
null);
} catch (MalformedURLException mue) {
throw new ClassNotFoundException("Unexpected exception ", mue);
//should not happen
}
log.info("Defining_class {} from runtime jar {} ", name, jarName);
result = defineClass(name, buf.array(), buf.arrayOffset(), buf.limit(), defaultDomain);
classCache.put(name, result);
return result;
}
private ByteBuffer getByteBuffer(String name, AtomicReference<String> jarName) throws Exception {
if (!allJarsLoaded) {
loadJars();
}
String path = name.replace('.', '/').concat(".class");
ByteBuffer buf = null;
for (PluginBag.RuntimeLib lib : libs) {
try {
buf = lib.getFileContent(path);
if (buf != null) {
jarName.set(lib.getName());
break;
}
} catch (Exception exp) {
throw new ClassNotFoundException("Unable to load class :" + name, exp);
}
}
return buf;
}
@Override
public void close() {
for (PluginBag.RuntimeLib lib : libs) {
try {
lib.close();
} catch (Exception e) {
log.error("Error closing lib {}", lib.getName(), e);
}
}
}
@Override
public InputStream openResource(String resource) throws IOException {
AtomicReference<String> jarName = new AtomicReference<>();
try {
ByteBuffer buf = getByteBuffer(resource, jarName);
if (buf == null) throw new IOException("Resource could not be found " + resource);
} catch (Exception e) {
throw new IOException("Resource could not be found " + resource, e);
}
return null;
}
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
if(!allJarsLoaded ) loadJars();
try {
return findClass(cname).asSubclass(expectedType);
} catch (Exception e) {
if (e instanceof SolrException) {
throw (SolrException) e;
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "error loading class " + cname, e);
}
}
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
try {
return findClass(cname, expectedType).getConstructor().newInstance();
} catch (SolrException e) {
throw e;
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "error instantiating class :" + cname, e);
}
}
}

208
solr/core/src/java/org/apache/solr/core/PluginBag.java
View File

@ -16,11 +16,8 @@
*/
package org.apache.solr.core;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@ -31,25 +28,18 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.CryptoKeys;
import org.apache.solr.util.SimplePostTool;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
@ -58,7 +48,6 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonMap;
import static org.apache.solr.api.ApiBag.HANDLER_NAME;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
* This manages the lifecycle of a set of plugin of the same type .
@ -130,22 +119,11 @@ public class PluginBag<T> implements AutoCloseable {
@SuppressWarnings({"unchecked"})
public PluginHolder<T> createPlugin(PluginInfo info) {
if ("true".equals(String.valueOf(info.attributes.get("runtimeLib")))) {
if (log.isDebugEnabled()) {
log.debug(" {} : '{}' created with runtimeLib=true ", meta.getCleanTag(), info.name);
}
LazyPluginHolder<T> holder = new LazyPluginHolder<>(meta, info, core, RuntimeLib.isEnabled() ?
core.getMemClassLoader() :
core.getResourceLoader(), true);
return meta.clazz == UpdateRequestProcessorFactory.class ?
(PluginHolder<T>) new UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder(holder) :
holder;
} else if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) {
if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) {
if (log.isDebugEnabled()) {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
}
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader());
} else {
if (info.pkgName != null) {
PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core, meta);
@ -433,22 +411,13 @@ public class PluginBag<T> implements AutoCloseable {
protected SolrException solrException;
private final SolrCore core;
protected ResourceLoader resourceLoader;
private final boolean isRuntimeLib;
LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader, boolean isRuntimeLib) {
LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader) {
super(pluginInfo);
this.pluginMeta = pluginMeta;
this.isRuntimeLib = isRuntimeLib;
this.core = core;
this.resourceLoader = loader;
if (loader instanceof MemClassLoader) {
if (!RuntimeLib.isEnabled()) {
String s = "runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true";
log.warn(s);
solrException = new SolrException(SolrException.ErrorCode.SERVER_ERROR, s);
}
}
}
@Override
@ -474,24 +443,14 @@ public class PluginBag<T> implements AutoCloseable {
if (log.isInfoEnabled()) {
log.info("Going to create a new {} with {} ", pluginMeta.getCleanTag(), pluginInfo);
}
if (resourceLoader instanceof MemClassLoader) {
MemClassLoader loader = (MemClassLoader) resourceLoader;
loader.loadJars();
}
@SuppressWarnings({"unchecked"})
Class<T> clazz = (Class<T>) pluginMeta.clazz;
T localInst = null;
try {
localInst = SolrCore.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader);
} catch (SolrException e) {
if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) {
throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()),
e.getMessage() + ". runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true",
e.getCause());
}
throw e;
}
initInstance(localInst, pluginInfo);
if (localInst instanceof SolrCoreAware) {
@ -511,165 +470,6 @@ public class PluginBag<T> implements AutoCloseable {
}
}
/**
* This represents a Runtime Jar. A jar requires two details , name and version
*/
public static class RuntimeLib implements PluginInfoInitialized, AutoCloseable {
private String name, version, sig, sha512, url;
private BlobRepository.BlobContentRef<ByteBuffer> jarContent;
private final CoreContainer coreContainer;
private boolean verified = false;
@Override
public void init(PluginInfo info) {
name = info.attributes.get(NAME);
url = info.attributes.get("url");
sig = info.attributes.get("sig");
if(url == null) {
Object v = info.attributes.get("version");
if (name == null || v == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version");
}
version = String.valueOf(v);
} else {
sha512 = info.attributes.get("sha512");
if(sha512 == null){
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute");
}
ByteBuffer buf = null;
buf = coreContainer.getBlobRepository().fetchFromUrl(name, url);
String digest = BlobRepository.sha512Digest(buf);
if(!sha512.equals(digest)) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest) );
}
log.info("dynamic library verified {}, sha512: {}", url, sha512);
}
}
public RuntimeLib(SolrCore core) {
coreContainer = core.getCoreContainer();
}
public String getUrl(){
return url;
}
@SuppressWarnings({"unchecked"})
void loadJar() {
if (jarContent != null) return;
synchronized (this) {
if (jarContent != null) return;
jarContent = url == null?
coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version):
coreContainer.getBlobRepository().getBlobIncRef(name, null,url,sha512);
}
}
public static boolean isEnabled() {
return Boolean.getBoolean("enable.runtime.lib");
}
public String getName() {
return name;
}
public String getVersion() {
return version;
}
public String getSig() {
return sig;
}
public ByteBuffer getFileContent(String entryName) throws IOException {
if (jarContent == null)
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name );
return getFileContent(jarContent.blob, entryName);
}
public ByteBuffer getFileContent(BlobRepository.BlobContent<ByteBuffer> blobContent, String entryName) throws IOException {
ByteBuffer buff = blobContent.get();
ByteArrayInputStream zipContents = new ByteArrayInputStream(buff.array(), buff.arrayOffset(), buff.limit());
ZipInputStream zis = new ZipInputStream(zipContents);
try {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
if (entryName == null || entryName.equals(entry.getName())) {
SimplePostTool.BAOS out = new SimplePostTool.BAOS();
byte[] buffer = new byte[2048];
int size;
while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
out.write(buffer, 0, size);
}
out.close();
return out.getByteBuffer();
}
}
} finally {
zis.closeEntry();
}
return null;
}
@Override
public void close() {
if (jarContent != null) coreContainer.getBlobRepository().decrementBlobRefCount(jarContent);
}
public static List<RuntimeLib> getLibObjects(SolrCore core, List<PluginInfo> libs) {
List<RuntimeLib> l = new ArrayList<>(libs.size());
for (PluginInfo lib : libs) {
RuntimeLib rtl = new RuntimeLib(core);
try {
rtl.init(lib);
} catch (Exception e) {
log.error("error loading runtime library", e);
}
l.add(rtl);
}
return l;
}
public void verify() throws Exception {
if (verified) return;
if (jarContent == null) {
log.error("Calling verify before loading the jar");
return;
}
if (!coreContainer.isZooKeeperAware())
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Signing jar is possible only in cloud");
Map<String, byte[]> keys = CloudUtil.getTrustedKeys(coreContainer.getZkController().getZkClient(), "exe");
if (keys.isEmpty()) {
if (sig == null) {
verified = true;
return;
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No public keys are available in ZK to verify signature for runtime lib " + name);
}
} else if (sig == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString("runtimelib {0} should be signed with one of the keys in ZK /keys/exe ", name));
}
try {
String matchedKey = new CryptoKeys(keys).verify(sig, jarContent.blob.get());
if (matchedKey == null)
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No key matched signature for jar : " + name + " version: " + version);
log.info("Jar {} signed with {} successfully verified", name, matchedKey);
} catch (Exception e) {
if (e instanceof SolrException) throw e;
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error verifying key ", e);
}
}
}
public Api v2lookup(String path, String method, Map<String, String> parts) {
if (apiBag == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "this should not happen, looking up for v2 API at the wrong place");

1
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -347,7 +347,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
// and even then -- only if there is a single SpellCheckComponent
// because of queryConverter.setIndexAnalyzer
.add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS))
.add(new SolrPluginInfo(PluginBag.RuntimeLib.class, "runtimeLib", REQUIRE_NAME, MULTI_OK))
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice

19
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -225,7 +225,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final RecoveryStrategy.Builder recoveryStrategyBuilder;
private IndexReaderFactory indexReaderFactory;
private final Codec codec;
private final MemClassLoader memClassLoader;
//singleton listener for all packages used in schema
private final PackageListeningClassLoader schemaPluginsLoader;
@ -1003,10 +1002,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
this.solrDelPolicy = initDeletionPolicy(delPolicy);
this.codec = initCodec(solrConfig, this.schema);
memClassLoader = new MemClassLoader(
PluginBag.RuntimeLib.getLibObjects(this, solrConfig.getPluginInfos(PluginBag.RuntimeLib.class.getName())),
getResourceLoader());
initIndex(prev != null, reload);
initWriters();
@ -1036,9 +1031,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
// Initialize the RestManager
restManager = initRestManager();
// at this point we can load jars loaded from remote urls.
memClassLoader.loadRemoteJars();
// Finally tell anyone who wants to know
resourceLoader.inform(resourceLoader);
resourceLoader.inform(this); // last call before the latch is released.
@ -1611,14 +1603,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
valueSourceParsers.close();
transformerFactories.close();
if (memClassLoader != null) {
try {
memClassLoader.close();
} catch (Exception e) {
}
}
try {
if (null != updateHandler) {
updateHandler.close();
@ -2789,9 +2773,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
};
}
public MemClassLoader getMemClassLoader() {
return memClassLoader;
}
public interface RawWriter {
default String getContentType() {

16
solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
View File

@ -63,7 +63,6 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigOverlay;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
@ -572,21 +571,6 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
if (info.clazz == PluginBag.RuntimeLib.class) {
if (!PluginBag.RuntimeLib.isEnabled()) {
op.addError("Solr not started with -Denable.runtime.lib=true");
return overlay;
}
try {
try (PluginBag.RuntimeLib rtl = new PluginBag.RuntimeLib(req.getCore())) {
rtl.init(new PluginInfo(info.tag, op.getDataMap()));
}
} catch (Exception e) {
op.addError(e.getMessage());
log.error("can't load this plugin ", e);
return overlay;
}
}
if (!verifyClass(op, clz, info.clazz)) return overlay;
if (pluginExists(info, overlay, name)) {
if (isCeate) {

2
solr/core/src/java/org/apache/solr/handler/StreamHandler.java
View File

@ -138,7 +138,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
streamFactory.withFunctionName(pluginInfo.name,
() -> holder.getClazz());
} else {
Class<? extends Expressible> clazz = core.getMemClassLoader().findClass(pluginInfo.className, Expressible.class);
Class<? extends Expressible> clazz = core.getResourceLoader().findClass(pluginInfo.className, Expressible.class);
streamFactory.withFunctionName(pluginInfo.name, clazz);
}
}

6
solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
View File

@ -77,7 +77,7 @@ import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetProperties;
import org.apache.solr.core.TestDynamicLoading;
import org.apache.solr.core.TestSolrConfigHandler;
import org.apache.solr.security.BasicAuthIntegrationTest;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.util.ExternalPaths;
@ -485,7 +485,7 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
private long uploadConfigSet(String configSetName, String suffix, String username, String password,
SolrZkClient zkClient) throws IOException {
// Read zipped sample config
ByteBuffer sampleZippedConfig = TestDynamicLoading
ByteBuffer sampleZippedConfig = TestSolrConfigHandler
.getFileContent(
createTempZipFile("solr/configsets/upload/"+configSetName), false);
@ -509,7 +509,7 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
File zipFile = new File(solrCluster.getBaseDir().toFile().getAbsolutePath() +
File.separator + TestUtil.randomSimpleString(random(), 6, 8) + ".zip");
File directory = TestDynamicLoading.getFile(directoryPath);
File directory = SolrTestCaseJ4.getFile(directoryPath);
if (log.isInfoEnabled()) {
log.info("Directory: {}", directory.getAbsolutePath());
}

209
solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
View File

@ -1,209 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Map;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.MemClassLoader;
import org.apache.solr.core.TestDynamicLoading;
import org.apache.solr.core.TestSolrConfigHandler;
import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.util.CryptoKeys;
import org.apache.solr.util.RestTestHarness;
import org.apache.zookeeper.CreateMode;
import org.junit.Test;
import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
public TestCryptoKeys() {
super();
sliceCount = 1;
}
@Test
public void test() throws Exception {
System.setProperty("enable.runtime.lib", "true");
setupRestTestHarnesses();
String pk1sig = "G8LEW7uJ1is81Aqqfl3Sld3qDtOxPuVFeTLJHFJWecgDvUkmJNFXmf7nkHOVlXnDWahp1vqZf0W02VHXg37lBw==";
String pk2sig = "pCyBQycB/0YvLVZfKLDIIqG1tFwM/awqzkp2QNpO7R3ThTqmmrj11wEJFDRLkY79efuFuQPHt40EE7jrOKoj9jLNELsfEqvU3jw9sZKiDONY+rV9Bj9QPeW8Pgt+F9Y1";
String wrongKeySig = "xTk2hTipfpb+J5s4x3YZGOXkmHWtnJz05Vvd8RTm/Q1fbQVszR7vMk6dQ1URxX08fcg4HvxOo8g9bG2TSMOGjg==";
String result = null;
CryptoKeys cryptoKeys = null;
SolrZkClient zk = getCommonCloudSolrClient().getZkStateReader().getZkClient();
cryptoKeys = new CryptoKeys(CloudUtil.getTrustedKeys(zk, "exe"));
ByteBuffer samplefile = ByteBuffer.wrap(readFile("cryptokeys/samplefile.bin"));
//there are no keys yet created in ZK
result = cryptoKeys.verify( pk1sig,samplefile);
assertNull(result);
zk.makePath("/keys/exe", true);
zk.create("/keys/exe/pubk1.der", readFile("cryptokeys/pubk1.der"), CreateMode.PERSISTENT, true);
zk.create("/keys/exe/pubk2.der", readFile("cryptokeys/pubk2.der"), CreateMode.PERSISTENT, true);
Map<String, byte[]> trustedKeys = CloudUtil.getTrustedKeys(zk, "exe");
cryptoKeys = new CryptoKeys(trustedKeys);
result = cryptoKeys.verify(pk2sig, samplefile);
assertEquals("pubk2.der", result);
result = cryptoKeys.verify(pk1sig, samplefile);
assertEquals("pubk1.der", result);
try {
result = cryptoKeys.verify(wrongKeySig,samplefile);
assertNull(result);
} catch (Exception e) {
//pass
}
try {
result = cryptoKeys.verify( "SGVsbG8gV29ybGQhCg==", samplefile);
assertNull(result);
} catch (Exception e) {
//pass
}
HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size()));
String baseURL = randomClient.getBaseURL();
baseURL = baseURL.substring(0, baseURL.lastIndexOf('/'));
TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient()));
waitForRecoveriesToFinish(".system", true);
ByteBuffer jar = TestDynamicLoading.getFileContent("runtimecode/runtimelibs.jar.bin");
String blobName = "signedjar";
TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 1);
String payload = "{\n" +
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler' , 'runtimeLib':true }" +
"}";
RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "requestHandler", "/runtime", "class"),
"org.apache.solr.core.RuntimeLibReqHandler", 10);
payload = "{\n" +
"'add-runtimelib' : { 'name' : 'signedjar' ,'version':1}\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "version"),
1l, 10);
@SuppressWarnings({"rawtypes"})
LinkedHashMapWriter map = TestSolrConfigHandler.getRespMap("/runtime", client);
String s = map._getStr( "error/msg",null);
assertNotNull(map.toString(), s);
assertTrue(map.toString(), s.contains("should be signed with one of the keys in ZK /keys/exe"));
String wrongSig = "QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw==";
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw=='}\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "sig"),
wrongSig, 10);
map = TestSolrConfigHandler.getRespMap("/runtime", client);
s = (String) Utils.getObjectByPath(map, false, Arrays.asList("error", "msg"));
assertNotNull(map.toString(), s);//No key matched signature for jar
assertTrue(map.toString(), s.contains("No key matched signature for jar"));
String rightSig = "YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q==";
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q=='}\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "sig"),
rightSig, 10);
map = TestSolrConfigHandler.testForResponseElement(client,
null,
"/runtime",
null,
Arrays.asList("class"),
"org.apache.solr.core.RuntimeLibReqHandler", 10);
compareValues(map, MemClassLoader.class.getName(), asList("loader"));
rightSig = "VJPMTxDf8Km3IBj2B5HWkIOqeM/o+HHNobOYCNA3WjrEVfOMZbMMqS1Lo7uLUUp//RZwOGkOhrUhuPNY1z2CGEIKX2/m8VGH64L14d52oSvFiwhoTDDuuyjW1TFGu35D";
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'VJPMTxDf8Km3IBj2B5HWkIOqeM/o+HHNobOYCNA3WjrEVfOMZbMMqS1Lo7uLUUp//RZwOGkOhrUhuPNY1z2CGEIKX2/m8VGH64L14d52oSvFiwhoTDDuuyjW1TFGu35D'}\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "sig"),
rightSig, 10);
map = TestSolrConfigHandler.testForResponseElement(client,
null,
"/runtime",
null,
Arrays.asList("class"),
"org.apache.solr.core.RuntimeLibReqHandler", 10);
compareValues(map, MemClassLoader.class.getName(), asList("loader"));
}
private byte[] readFile(String fname) throws IOException {
byte[] buf = null;
try (FileInputStream fis = new FileInputStream(getFile(fname))) {
buf = new byte[fis.available()];
fis.read(buf);
}
return buf;
}
}

2
solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java
View File

@ -131,7 +131,7 @@ public class BlobRepositoryMockingTest {
@Test
public void testGetBlobIncrRefByUrl() throws Exception{
when(mockContainer.isZooKeeperAware()).thenReturn(true);
filecontent = TestDynamicLoading.getFileContent("runtimecode/runtimelibs_v2.jar.bin");
filecontent = TestSolrConfigHandler.getFileContent("runtimecode/runtimelibs_v2.jar.bin");
url = "http://localhost:8080/myjar/location.jar";
@SuppressWarnings({"rawtypes"})
BlobRepository.BlobContentRef ref = repository.getBlobIncRef( "filefoo",null,url,

290
solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
View File

@ -1,290 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.util.RestTestHarness;
import org.apache.solr.util.SimplePostTool;
import org.junit.BeforeClass;
import org.junit.Test;
import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
@BeforeClass
public static void enableRuntimeLib() throws Exception {
System.setProperty("enable.runtime.lib", "true");
}
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
//17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
public void testDynamicLoading() throws Exception {
System.setProperty("enable.runtime.lib", "true");
setupRestTestHarnesses();
String blobName = "colltest";
boolean success = false;
HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size()));
String baseURL = randomClient.getBaseURL();
baseURL = baseURL.substring(0, baseURL.lastIndexOf('/'));
String payload = "{\n" +
"'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" +
"}";
RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "version"),
1l, 10);
payload = "{\n" +
"'create-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandler' ,registerPath: '/solr,/v2', 'runtimeLib' : true }\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client,"/config",payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "requestHandler", "/test1", "class"),
"org.apache.solr.core.BlobStoreTestRequestHandler",10);
@SuppressWarnings({"rawtypes"})
Map map = TestSolrConfigHandler.getRespMap("/test1", client);
assertNotNull(map.toString(), map = (Map) map.get("error"));
assertTrue(map.toString(), map.get("msg").toString().contains(".system collection not available"));
TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient()));
waitForRecoveriesToFinish(".system", true);
map = TestSolrConfigHandler.getRespMap("/test1", client);
assertNotNull(map = (Map) map.get("error"));
assertTrue("full output " + map, map.get("msg").toString().contains("no such blob or version available: colltest/1" ));
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val',\n" +
" 'y': 'Y val'}\n" +
" }\n" +
" }";
TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload);
TestSolrConfigHandler.testForResponseElement(
client,
null,
"/config/params",
cloudClient,
Arrays.asList("response", "params", "watched", "x"),
"X val",
10);
for(int i=0;i<100;i++) {
map = TestSolrConfigHandler.getRespMap("/test1", client);
if("X val".equals(map.get("x"))){
success = true;
break;
}
Thread.sleep(100);
}
ByteBuffer jar = null;
// jar = persistZip("/tmp/runtimelibs.jar.bin", TestDynamicLoading.class, RuntimeLibReqHandler.class, RuntimeLibResponseWriter.class, RuntimeLibSearchComponent.class);
// if(true) return;
jar = getFileContent("runtimecode/runtimelibs.jar.bin");
TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 1);
payload = "{\n" +
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler' , 'runtimeLib':true }," +
"'create-searchcomponent' : { 'name' : 'get', 'class': 'org.apache.solr.core.RuntimeLibSearchComponent' , 'runtimeLib':true }," +
"'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'org.apache.solr.core.RuntimeLibResponseWriter' , 'runtimeLib':true }" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
@SuppressWarnings({"rawtypes"})
Map result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "requestHandler", "/runtime", "class"),
"org.apache.solr.core.RuntimeLibReqHandler", 10);
compareValues(result, "org.apache.solr.core.RuntimeLibResponseWriter", asList("overlay", "queryResponseWriter", "json1", "class"));
compareValues(result, "org.apache.solr.core.RuntimeLibSearchComponent", asList("overlay", "searchComponent", "get", "class"));
result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/runtime",
null,
Arrays.asList("class"),
"org.apache.solr.core.RuntimeLibReqHandler", 10);
compareValues(result, MemClassLoader.class.getName(), asList( "loader"));
result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/runtime?wt=json1",
null,
Arrays.asList("wt"),
"org.apache.solr.core.RuntimeLibResponseWriter", 10);
compareValues(result, MemClassLoader.class.getName(), asList( "loader"));
result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/get?abc=xyz",
null,
Arrays.asList("get"),
"org.apache.solr.core.RuntimeLibSearchComponent", 10);
compareValues(result, MemClassLoader.class.getName(), asList( "loader"));
jar = getFileContent("runtimecode/runtimelibs_v2.jar.bin");
TestBlobHandler.postAndCheck(cloudClient, baseURL, blobName, jar, 2);
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'colltest' ,'version':2}\n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "version"),
2l, 10);
result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/get?abc=xyz",
null,
Arrays.asList("Version"),
"2", 10);
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val',\n" +
" 'y': 'Y val'}\n" +
" }\n" +
" }";
TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload);
TestSolrConfigHandler.testForResponseElement(
client,
null,
"/config/params",
cloudClient,
Arrays.asList("response", "params", "watched", "x"),
"X val",
10);
result = TestSolrConfigHandler.testForResponseElement(
client,
null,
"/test1",
cloudClient,
Arrays.asList("x"),
"X val",
10);
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val changed',\n" +
" 'y': 'Y val'}\n" +
" }\n" +
" }";
TestSolrConfigHandler.runConfigCommand(client,"/config/params",payload);
result = TestSolrConfigHandler.testForResponseElement(
client,
null,
"/test1",
cloudClient,
Arrays.asList("x"),
"X val changed",
10);
}
public static ByteBuffer getFileContent(String f) throws IOException {
return getFileContent(f, true);
}
/**
* @param loadFromClassPath if true, it will look in the classpath to find the file,
* otherwise load from absolute filesystem path.
*/
public static ByteBuffer getFileContent(String f, boolean loadFromClassPath) throws IOException {
ByteBuffer jar;
File file = loadFromClassPath ? getFile(f): new File(f);
try (FileInputStream fis = new FileInputStream(file)) {
byte[] buf = new byte[fis.available()];
fis.read(buf);
jar = ByteBuffer.wrap(buf);
}
return jar;
}
public static ByteBuffer persistZip(String loc,
@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
ByteBuffer jar = generateZip(classes);
try (FileOutputStream fos = new FileOutputStream(loc)){
fos.write(jar.array(), 0, jar.limit());
fos.flush();
}
return jar;
}
public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
SimplePostTool.BAOS bos = new SimplePostTool.BAOS();
try (ZipOutputStream zipOut = new ZipOutputStream(bos)) {
zipOut.setLevel(ZipOutputStream.DEFLATED);
for (@SuppressWarnings({"rawtypes"})Class c : classes) {
String path = c.getName().replace('.', '/').concat(".class");
ZipEntry entry = new ZipEntry(path);
ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path));
zipOut.putNextEntry(entry);
zipOut.write(b.array(), 0, b.limit());
zipOut.closeEntry();
}
}
return bos.getByteBuffer();
}
}

128
solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java
View File

@ -1,128 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.util.Pair;
import org.apache.solr.util.RestTestHarness;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.BeforeClass;
import static java.util.Arrays.asList;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
@SolrTestCaseJ4.SuppressSSL
public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase {
@BeforeClass
public static void enableRuntimeLib() throws Exception {
System.setProperty("enable.runtime.lib", "true");
}
public static Pair<Server, Integer> runHttpServer(Map<String, Object> jars) throws Exception {
final Server server = new Server();
final ServerConnector connector = new ServerConnector(server);
server.setConnectors(new Connector[] { connector });
server.setHandler(new AbstractHandler() {
@Override
public void handle(String s, Request request, HttpServletRequest req, HttpServletResponse rsp)
throws IOException {
ByteBuffer b = (ByteBuffer) jars.get(s);
if (b != null) {
rsp.getOutputStream().write(b.array(), 0, b.limit());
rsp.setContentType("application/octet-stream");
rsp.setStatus(HttpServletResponse.SC_OK);
request.setHandled(true);
}
}
});
server.start();
return new Pair<>(server, connector.getLocalPort());
}
public void testDynamicLoadingUrl() throws Exception {
setupRestTestHarnesses();
Pair<Server, Integer> pair = runHttpServer(ImmutableMap.of("/jar1.jar", getFileContent("runtimecode/runtimelibs.jar.bin")));
Integer port = pair.second();
try {
String payload = "{\n" +
"'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" +
" 'sha512':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
"}";
RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommandExpectFailure(client, "/config", payload, "Invalid jar");
payload = "{\n" +
"'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" +
" 'sha512':'d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", "urljar", "sha512"),
"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", 120);
payload = "{\n" +
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "requestHandler", "/runtime", "class"),
"org.apache.solr.core.RuntimeLibReqHandler", 120);
@SuppressWarnings({"rawtypes"})
Map result = TestSolrConfigHandler.testForResponseElement(client,
null,
"/runtime",
null,
Arrays.asList("class"),
"org.apache.solr.core.RuntimeLibReqHandler", 120);
compareValues(result, MemClassLoader.class.getName(), asList("loader"));
} finally {
pair.first().stop();
}
}
}

111
solr/core/src/test/org/apache/solr/core/TestDynamicURP.java
View File

@ -1,111 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import static java.util.Collections.singletonMap;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.handler.TestBlobHandler;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestDynamicURP extends SolrCloudTestCase {
private static final String COLLECTION = "testUrpColl";
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("enable.runtime.lib", "true");
configureCluster(3)
.addConfig("conf", configset("cloud-minimal"))
.configure();
SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
String path = ZkStateReader.CONFIGS_ZKNODE + "/conf/solrconfig.xml";
byte[] data = zkClient.getData(path, null, null, true);
String solrconfigStr = new String(data, StandardCharsets.UTF_8);
zkClient.setData(path, solrconfigStr.replace("</config>",
"<updateRequestProcessorChain name=\"test_urp\" processor=\"testURP\" default=\"true\">\n" +
" <processor class=\"solr.RunUpdateProcessorFactory\"/>\n" +
" </updateRequestProcessorChain>\n" +
"\n" +
" <updateProcessor class=\"runtimecode.TestURP\" name=\"testURP\" runtimeLib=\"true\"></updateProcessor>\n" +
"</config>").getBytes(StandardCharsets.UTF_8), true );
CollectionAdminRequest.createCollection(COLLECTION, "conf", 3, 1).process(cluster.getSolrClient());
waitForState("", COLLECTION, clusterShape(3, 3));
}
@Test
public void testUrp() throws Exception {
ByteBuffer jar = getFileContent("runtimecode/runtimeurp.jar.bin");
String blobName = "urptest";
TestBlobHandler.postAndCheck(cluster.getSolrClient(), cluster.getRandomJetty(random()).getBaseUrl().toString(),
blobName, jar, 1);
new V2Request.Builder("/c/" + COLLECTION + "/config")
.withPayload(singletonMap("add-runtimelib", (MapWriter) ew1 -> ew1
.put("name", blobName)
.put("version", "1")))
.withMethod(POST)
.build()
.process(cluster.getSolrClient());
TestSolrConfigHandler.testForResponseElement(null,
cluster.getRandomJetty(random()).getBaseUrl().toString(),
"/"+COLLECTION+"/config/overlay",
cluster.getSolrClient(),
Arrays.asList("overlay", "runtimeLib", blobName, "version")
,"1",10);
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "123");
doc.addField("name_s", "Test URP");
new UpdateRequest()
.add(doc)
.commit(cluster.getSolrClient(), COLLECTION);
QueryResponse result = cluster.getSolrClient().query(COLLECTION, new SolrQuery("id:123"));
assertEquals(1, result.getResults().getNumFound());
Object time_s = result.getResults().get(0).getFirstValue("time_s");
assertNotNull(time_s);
}
}

55
solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
View File

@ -16,10 +16,9 @@
*/
package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.io.*;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -27,6 +26,8 @@ import java.util.Objects;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
@ -45,6 +46,7 @@ import org.apache.solr.search.SolrCache;
import org.apache.solr.util.RESTfulServerProvider;
import org.apache.solr.util.RestTestBase;
import org.apache.solr.util.RestTestHarness;
import org.apache.solr.util.SimplePostTool;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.After;
import org.junit.Before;
@ -67,8 +69,53 @@ public class TestSolrConfigHandler extends RestTestBase {
private static final String collection = "collection1";
private static final String confDir = collection + "/conf";
public static ByteBuffer getFileContent(String f) throws IOException {
return getFileContent(f, true);
}
@Before
/**
* @param loadFromClassPath if true, it will look in the classpath to find the file,
* otherwise load from absolute filesystem path.
*/
public static ByteBuffer getFileContent(String f, boolean loadFromClassPath) throws IOException {
ByteBuffer jar;
File file = loadFromClassPath ? getFile(f): new File(f);
try (FileInputStream fis = new FileInputStream(file)) {
byte[] buf = new byte[fis.available()];
fis.read(buf);
jar = ByteBuffer.wrap(buf);
}
return jar;
}
public static ByteBuffer persistZip(String loc,
@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
ByteBuffer jar = generateZip(classes);
try (FileOutputStream fos = new FileOutputStream(loc)){
fos.write(jar.array(), 0, jar.limit());
fos.flush();
}
return jar;
}
public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
SimplePostTool.BAOS bos = new SimplePostTool.BAOS();
try (ZipOutputStream zipOut = new ZipOutputStream(bos)) {
zipOut.setLevel(ZipOutputStream.DEFLATED);
for (@SuppressWarnings({"rawtypes"})Class c : classes) {
String path = c.getName().replace('.', '/').concat(".class");
ZipEntry entry = new ZipEntry(path);
ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path));
zipOut.putNextEntry(entry);
zipOut.write(b.array(), 0, b.limit());
zipOut.closeEntry();
}
}
return bos.getByteBuffer();
}
@Before
public void before() throws Exception {
tmpSolrHome = createTempDir().toFile();
tmpConfDir = new File(tmpSolrHome, confDir);

2
solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
View File

@ -53,7 +53,7 @@ import java.util.concurrent.Callable;
import java.util.function.Predicate;
import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG")

2
solr/core/src/test/org/apache/solr/pkg/TestPackages.java
View File

@ -68,7 +68,7 @@ import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;

40
solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update.processor;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
public class RuntimeUrp extends SimpleUpdateProcessorFactory {
@Override
protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) {
UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessorChain(req.getParams());
List<String> names = new ArrayList<>();
for (UpdateRequestProcessorFactory p : processorChain.getProcessors()) {
if (p instanceof UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) {
p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).getDelegate();
}
names.add(p.getClass().getSimpleName());
}
cmd.solrDoc.addField("processors_s", StrUtils.join(names,'>'));
}
}

163
solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java
View File

@ -1,163 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update.processor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collection;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.TestDynamicLoading;
import org.apache.solr.core.TestSolrConfigHandler;
import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.util.RestTestHarness;
import org.apache.solr.util.SimplePostTool;
import org.junit.Test;
public class TestNamedUpdateProcessors extends AbstractFullDistribZkTestBase {
@Test
//17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
public void test() throws Exception {
System.setProperty("enable.runtime.lib", "true");
setupRestTestHarnesses();
String blobName = "colltest";
HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size()));
String baseURL = randomClient.getBaseURL();
final String solrClientUrl = baseURL.substring(0, baseURL.lastIndexOf('/'));
TestBlobHandler.createSystemCollection(getHttpSolrClient(solrClientUrl, randomClient.getHttpClient()));
waitForRecoveriesToFinish(".system", true);
TestBlobHandler.postAndCheck(cloudClient, baseURL.substring(0, baseURL.lastIndexOf('/')), blobName, TestDynamicLoading.generateZip(RuntimeUrp.class), 1);
String payload = "{\n" +
"'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" +
"}";
RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "runtimeLib", blobName, "version"),
1l, 10);
payload = "{\n" +
"'create-updateprocessor' : { 'name' : 'firstFld', 'class': 'solr.FirstFieldValueUpdateProcessorFactory', 'fieldName':'test_s'}, \n" +
"'create-updateprocessor' : { 'name' : 'test', 'class': 'org.apache.solr.update.processor.RuntimeUrp', 'runtimeLib':true }, \n" +
"'create-updateprocessor' : { 'name' : 'maxFld', 'class': 'solr.MaxFieldValueUpdateProcessorFactory', 'fieldName':'mul_s'} \n" +
"}";
client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
forAllRestTestHarnesses(restTestHarness -> {
try {
TestSolrConfigHandler.testForResponseElement(restTestHarness,
null,
"/config/overlay",
null,
Arrays.asList("overlay", "updateProcessor", "firstFld", "fieldName"),
"test_s", 10);
} catch (Exception ex) {
fail("Caught exception: "+ex);
}
});
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "123");
doc.addField("test_s", Arrays.asList("one", "two"));
doc.addField("mul_s", Arrays.asList("aaa", "bbb"));
randomClient.add(doc);
randomClient.commit(true, true);
QueryResponse result = randomClient.query(new SolrQuery("id:123"));
assertEquals(2, ((Collection) result.getResults().get(0).getFieldValues("test_s")).size());
assertEquals(2, ((Collection) result.getResults().get(0).getFieldValues("mul_s")).size());
doc = new SolrInputDocument();
doc.addField("id", "456");
doc.addField("test_s", Arrays.asList("three", "four"));
doc.addField("mul_s", Arrays.asList("aaa", "bbb"));
UpdateRequest ur = new UpdateRequest();
ur.add(doc).setParam("processor", "firstFld,maxFld,test");
randomClient.request(ur);
randomClient.commit(true, true);
result = randomClient.query(new SolrQuery("id:456"));
SolrDocument d = result.getResults().get(0);
assertEquals(1, d.getFieldValues("test_s").size());
assertEquals(1, d.getFieldValues("mul_s").size());
assertEquals("three", d.getFieldValues("test_s").iterator().next());
assertEquals("bbb", d.getFieldValues("mul_s").iterator().next());
String processors = (String) d.getFirstValue("processors_s");
assertNotNull(processors);
assertEquals(StrUtils.splitSmart(processors, '>'),
Arrays.asList("FirstFieldValueUpdateProcessorFactory", "MaxFieldValueUpdateProcessorFactory", "RuntimeUrp", "LogUpdateProcessorFactory", "DistributedUpdateProcessorFactory", "RunUpdateProcessorFactory"));
}
public static ByteBuffer getFileContent(String f) throws IOException {
ByteBuffer jar;
try (FileInputStream fis = new FileInputStream(getFile(f))) {
byte[] buf = new byte[fis.available()];
fis.read(buf);
jar = ByteBuffer.wrap(buf);
}
return jar;
}
public static ByteBuffer persistZip(String loc,
@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
ByteBuffer jar = generateZip(classes);
try (FileOutputStream fos = new FileOutputStream(loc)) {
fos.write(jar.array(), 0, jar.limit());
fos.flush();
}
return jar;
}
public static ByteBuffer generateZip(@SuppressWarnings({"rawtypes"})Class... classes) throws IOException {
SimplePostTool.BAOS bos = new SimplePostTool.BAOS();
try (ZipOutputStream zipOut = new ZipOutputStream(bos)) {
zipOut.setLevel(ZipOutputStream.DEFLATED);
for (@SuppressWarnings({"rawtypes"})Class c : classes) {
String path = c.getName().replace('.', '/').concat(".class");
ZipEntry entry = new ZipEntry(path);
ByteBuffer b = SimplePostTool.inputStreamToByteArray(c.getClassLoader().getResourceAsStream(path));
zipOut.putNextEntry(entry);
zipOut.write(b.array(), 0, b.limit());
zipOut.closeEntry();
}
}
return bos.getByteBuffer();
}
}